diff --git a/include/api/callback/callback.h b/include/api/callback/callback.h index 3332f8199be..40c3908efe7 100644 --- a/include/api/callback/callback.h +++ b/include/api/callback/callback.h @@ -31,6 +31,7 @@ class CallbackImpl; using GraphPoint = std::pair; +/// The TrainCallBackData class defines a set of parameters for training callbacks. struct TrainCallBackData { TrainCallBackData(bool train_mode, int epoch, int step, Model *model): train_mode_(train_mode), epoch_(epoch), step_(step), model_(model) {} @@ -41,6 +42,7 @@ struct TrainCallBackData { Model *model_; /**< pointer to the Model object */ }; +/// The CallbackRetValue class represents whether to continue looping in training. enum CallbackRetValue : uint32_t { kContinue = 0, kStopTraining = 1, @@ -48,6 +50,8 @@ enum CallbackRetValue : uint32_t { kUnknownRetValue = 0xFFFFFFFF }; +/// TrainCallBack is the training callback class in MindSpore Lite. +/// In MindSpore, the callback function is actually not a function but a class. class TrainCallBack { public: virtual ~TrainCallBack() = default; @@ -88,6 +92,8 @@ class TrainCallBack { virtual void StepEnd(const TrainCallBackData &cb_data) {} protected: + /// Due to TrainCallBack being used for implementing callbacks during the training process, the Model class and ModelImpl class need to access its private methods. + /// This is achieved through the use of friend classes. friend class Model; friend class ModelImpl; CallbackImpl* callback_impl_ = nullptr; diff --git a/include/api/callback/ckpt_saver.h b/include/api/callback/ckpt_saver.h index e673c624224..447e17bb994 100644 --- a/include/api/callback/ckpt_saver.h +++ b/include/api/callback/ckpt_saver.h @@ -20,20 +20,30 @@ #include #include #include +/// The "callback.h" provides the class TrainCallBack, which is responsible for generating its subclass CkptSaver. #include "include/api/callback/callback.h" +/// The "dual_abi_helper.h" provides the StringToChar method, which converts data of type std::string& to std::vector& . #include "include/api/dual_abi_helper.h" namespace mindspore { +/// \brief The CkptSaver class is the model file saving class of MindSpore Lite for training. +/// The CkptSaver class is a subclass of TrainCallBack. class CkptSaver: public TrainCallBack { public: inline CkptSaver(int save_every_n, const std::string &filename_prefix); virtual ~CkptSaver(); private: + /// \brief The private constructor of CkptSaver is used for initialization by the public constructor of CkptSaver. + /// \param save_every_n The step size of the callback. + /// \param filename_prefix The prefix of a file name. CkptSaver(int save_every_n, const std::vector &filename_prefix); }; +/// \brief The public constructor use a private constructor to initialize CkptSaver to ensure data security. +/// \param save_every_n The step size of the callback. +/// \param filename_prefix The prefix of a file name. CkptSaver::CkptSaver(int save_every_n, const std::string &filename_prefix) : CkptSaver(save_every_n, StringToChar(filename_prefix)) {} diff --git a/include/api/callback/loss_monitor.h b/include/api/callback/loss_monitor.h index 9e0a8247e37..6c423ccb759 100644 --- a/include/api/callback/loss_monitor.h +++ b/include/api/callback/loss_monitor.h @@ -23,10 +23,20 @@ namespace mindspore { +/// \brief LossMonitor is a class for training loss functions in MindSpore Lite. +/// LossMonitor is a subclass of TrainCallBack, and the inheritance method is public inheritance. class LossMonitor: public TrainCallBack { public: + + /// \brief The single-parameter constructor of LossMonitor is decorated with the "explicit" keyword to ensure that it cannot be implicitly converted. + /// \param print_every_n_steps is the callback step size for displaying the loss function value Loss. explicit LossMonitor(int print_every_n_steps = INT_MAX); + + /// \brief Virtual Destructor of LossMonitor. virtual ~LossMonitor(); + + /// @brief The GetLossPoints method is used to obtain training loss data. + /// @return a vector containing GraphPoint data representing the training loss data. const std::vector &GetLossPoints(); }; } // namespace mindspore diff --git a/include/api/callback/lr_scheduler.h b/include/api/callback/lr_scheduler.h index 2eddc66b44a..63ec8e700d9 100644 --- a/include/api/callback/lr_scheduler.h +++ b/include/api/callback/lr_scheduler.h @@ -34,6 +34,8 @@ int MultiplicativeLRLambda(float *lr, int epoch, void *multiplication); /// \brief Multiply the LR by a factor of gamma every step_size int StepLRLambda(float *lr, int epoch, void *step_size); + +/// \brief StepLRLambda is a struct that defines a set of parameters for training learning rate. struct StepLRLambda { StepLRLambda(int step, float g) : step_size(step), gamma(g) {} @@ -41,8 +43,14 @@ struct StepLRLambda { float gamma; // LR decay factor }; +/// \brief LRScheduler is the training learning rate scheduling class of MindSpore Lite. class LRScheduler: public TrainCallBack { public: + + /// \brief Constructor of LRScheduler. + /// \param lambda_func The lambda_func is a function wrapper template that wraps a function. It is used to adjust the learning rate during training. + /// \param lr_cb_data Data parameter during the callback process. + /// \param step is the callback step size for learning rate changes. The default value is 1. explicit LRScheduler(LR_Lambda lambda_func, void *lr_cb_data = nullptr, int step = 1); virtual ~LRScheduler(); }; diff --git a/include/api/callback/time_monitor.h b/include/api/callback/time_monitor.h index 7e857849f8a..63f6b55db8e 100644 --- a/include/api/callback/time_monitor.h +++ b/include/api/callback/time_monitor.h @@ -24,10 +24,18 @@ namespace mindspore { +/// \brief TimeMonitor is a training time monitoring class in MindSpore Lite. class TimeMonitor: public TrainCallBack { public: virtual ~TimeMonitor() = default; + + /// \brief The EpochBegin method is called before each iteration, overriding the method of the parent class. + /// \param cb_data include a set of parameters for training callbacks. void EpochBegin(const TrainCallBackData &cb_data) override; + + /// \brief The EpochEnd method is called after each round of iteration, overriding the method of the parent class. + /// \param cb_data include a set of parameters for training callbacks. + /// \return CallbackRetValue, indicating whether to continue looping in training. CallbackRetValue EpochEnd(const TrainCallBackData &cb_data) override; }; } // namespace mindspore diff --git a/include/api/callback/train_accuracy.h b/include/api/callback/train_accuracy.h index 5838dfd9c1b..1f11424ac04 100644 --- a/include/api/callback/train_accuracy.h +++ b/include/api/callback/train_accuracy.h @@ -26,13 +26,23 @@ namespace mindspore { +/// \brief TrainAccuracy is a learning rate scheduling class in MindSpore Lite for training. class TrainAccuracy: public TrainCallBack { public: + + /// \brief Constructor of TrainAccuracy. + /// \param print_every_n is the callback step size. + /// \param accuracy_metrics is the accuracy metric, with a default value of METRICS_CLASSIFICATION indicating 0. + /// \param input_indexes is the index of input. + /// \param output_indexes is the index of output. explicit TrainAccuracy(int print_every_n = INT_MAX, int accuracy_metrics = METRICS_CLASSIFICATION, const std::vector &input_indexes = {1}, const std::vector &output_indexes = {0}); virtual ~TrainAccuracy(); + + /// \brief The GetAccuracyPoints method is used to obtain the training accuracy. + /// \return Return a vector containing GraphPoint, i.e. training accuracy data. const std::vector &GetAccuracyPoints(); }; } // namespace mindspore diff --git a/include/api/cell.h b/include/api/cell.h index e0813467666..2c29f5c3f3d 100644 --- a/include/api/cell.h +++ b/include/api/cell.h @@ -29,6 +29,8 @@ class Context; using Input = InputAndOutput; using Output = InputAndOutput; + +/// \brief Container base class. class MS_API CellBase { public: CellBase() = default; @@ -39,13 +41,20 @@ class MS_API CellBase { std::vector operator()(const std::vector &inputs) const; }; +/// \brief Container class. +/// \brief A cell is the base class for all neural networks. A cell can be a single neural network unit or a unit that forms a network. template class MS_API Cell : public CellBase { public: virtual ~Cell() = default; + /// \brief The Clone method is used to create a copy of itself. + /// \return a pointer to a copy. std::shared_ptr Clone() const override { return std::make_shared(static_cast(*this)); } }; +/// \brief Graph container class. +/// Use GraphCell to run the computation graph loaded from MindIR. +/// In GRAPH_MODE (static graph mode), Cell will be compiled into a computational graph, while in PYNATIVE_MODE (dynamic graph mode), it serves as the foundational module for neural networks. class MS_API GraphCell final : public Cell { public: class GraphImpl; @@ -53,38 +62,60 @@ class MS_API GraphCell final : public Cell { GraphCell() = default; ~GraphCell() override = default; + /// \brief Constructor of GraphCell. + /// \param A pointer to a compiled graph loaded from MindIR. explicit GraphCell(const Graph &); explicit GraphCell(Graph &&); explicit GraphCell(const std::shared_ptr &); + /// \brief Create graph impl for device target(set the Graph object and the executor context). + /// \param context is the environment variable during execution. void SetContext(const std::shared_ptr &context); + + /// \brief The GetGraph method is used to obtain a pointer to the graph object. + /// \return a std::shared_ptr & pointer of Graph. const std::shared_ptr &GetGraph() const { return graph_; } + + /// \brief A method to set the Graph object with member graph_ and to run the Graph with parameters inputs and outputs. + /// \param inputs is a pointer of the input MStensor for the GraphCell. + /// \param outputs is a pointer of the output MStensor for the GraphCell. + /// \return the status of Running. Status Run(const std::vector &inputs, std::vector *outputs) override; + + /// \brief Get the input MStensor for the GraphCell. + /// \return the input MStensor for the GraphCell. std::vector GetInputs(); + + /// \brief Get the output MStensor for the GraphCell. + /// \return the output MStensor for the GraphCell. std::vector GetOutputs(); + + /// \brief Load the object into the device of device_id. + /// \param device_id is the target device ID to perform operation. + /// \return a Status object of the StatusCode class, and you can use its public functions StatusCode or ToString to obtain the specific error code and error message. Status Load(uint32_t device_id); private: - friend class Model; + friend class Model; /// The Model class can call all the methods of the Cell::GraphCell class. - std::shared_ptr graph_; - std::shared_ptr executor_; + std::shared_ptr graph_; /// graph_ is a pointer to a Graph object. + std::shared_ptr executor_; /// executor_ is a pointer to the GraphImpl object. }; class MS_API InputAndOutput { public: InputAndOutput(); ~InputAndOutput() = default; - + InputAndOutput(const std::shared_ptr &, const std::vector &, int32_t index); int32_t GetIndex() const { return index_; } void SetIndex(int32_t index) { index_ = index; } private: - std::shared_ptr cell_; - std::vector prev_; - int32_t index_; + std::shared_ptr cell_; /// cell_ is a pointer to a CellBase object. + std::vector prev_; /// prev_ is a pointer to an InputAndOutput object. + int32_t index_;/// The index of the current InputAndOutput object. }; } // namespace mindspore #endif // MINDSPORE_INCLUDE_API_CELL_H diff --git a/include/api/graph.h b/include/api/graph.h index f25a6217f32..91d2d1a8f82 100644 --- a/include/api/graph.h +++ b/include/api/graph.h @@ -27,20 +27,30 @@ namespace mindspore { class MS_API Graph { public: class GraphData; + /// \brief The default constructor of Graph will initialize the private member graph_data_ with nullptr. Graph(); explicit Graph(const std::shared_ptr &graph_data); explicit Graph(std::shared_ptr &&graph_data); explicit Graph(std::nullptr_t); ~Graph(); + /// \brief The ModelType method is used to obtain the model type. + /// \return Model Types. + /// Due to the possibility that the Graph object(this pointer) may be constant, and C++ rules dictate that only pointers to constants can be used to store the address of constant objects, the const keyword is used to modify the this pointer. enum ModelType ModelType() const; + + /// \brief Overloaded operator method for checking if it is a null pointer. + /// Due to the possibility that the Graph object(this pointer) may be constant, and C++ rules dictate that only pointers to constants can be used to store the address of constant objects, the const keyword is used to modify the this pointer. bool operator==(std::nullptr_t) const; + + /// \brief Operator Overloading Method for checking if it is a non-null pointer. + /// Due to the possibility that the Graph object(this pointer) may be constant, and C++ rules dictate that only pointers to constants can be used to store the address of constant objects, the const keyword is used to modify the this pointer. bool operator!=(std::nullptr_t) const; private: friend class GraphCell; friend class ModelImpl; - std::shared_ptr graph_data_; + std::shared_ptr graph_data_; /// Pointer to graphical model. }; } // namespace mindspore #endif // MINDSPORE_INCLUDE_API_GRAPH_H diff --git a/include/api/metrics/accuracy.h b/include/api/metrics/accuracy.h index 1d1732f3fe6..1175e23ceb9 100644 --- a/include/api/metrics/accuracy.h +++ b/include/api/metrics/accuracy.h @@ -23,12 +23,19 @@ namespace mindspore { constexpr int METRICS_CLASSIFICATION = 0; constexpr int METRICS_MULTILABEL = 1; +/// \brief AccuracyMetrics is the training accuracy class of MindSpore Lite. +/// AccuracyMetrics is a subclass of Metrics, which overrides the Clear method and Eval method of the Metrics class. class AccuracyMetrics : public Metrics { public: explicit AccuracyMetrics(int accuracy_metrics = METRICS_CLASSIFICATION, const std::vector &input_indexes = {1}, const std::vector &output_indexes = {0}); virtual ~AccuracyMetrics(); + + /// \brief The Clear method is used to reset the accuracy to zero. void Clear() override; + + /// \brief The Eval method is used for model validation. + /// \return Validation accuracy of the model with float type. float Eval() override; }; diff --git a/include/api/metrics/metrics.h b/include/api/metrics/metrics.h index 7154332f96d..02b57e300c4 100644 --- a/include/api/metrics/metrics.h +++ b/include/api/metrics/metrics.h @@ -13,6 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +/// Prevent multiple inclusion of header files #ifndef MINDSPORE_INCLUDE_API_METRICS_METRICS_H #define MINDSPORE_INCLUDE_API_METRICS_METRICS_H #include @@ -24,12 +25,27 @@ class MetricsImpl; class ModelImpl; class MSTensor; +/// Metrics is a training metric class in MindSpore Lite. +/// Metrics are indicators used to evaluate the performance of a model. class Metrics { public: - virtual ~Metrics() = default; + /// Virtual destructor + virtual ~Metrics() = default; + + /// The Clear method is used to reset the training metrics to zero. virtual void Clear() {} + + /// The Eval method is used for the model validation process and returns the model validation accuracy as a float value. + /// \return a floating-point number. + /// Default return 0.0 . virtual float Eval() { return 0.0; } + + /// The Update method implements the update of model input and output data. + /// \param[in] inputs is a vector of MSTensor, which is the model input. + /// \param[in] outputs is a vector of MSTensor, which are the outputs of the model. virtual void Update(std::vector inputs, std::vector outputs) {} + + /// Since Metrics is the training metric class of MindSpore Lite, the Model class and ModelImpl class need to call its private methods, which are achieved through the friend class mechanism. protected: friend class Model; friend class ModelImpl; diff --git a/include/api/types.h b/include/api/types.h index 816c4398ad0..eb4f2ab6857 100644 --- a/include/api/types.h +++ b/include/api/types.h @@ -34,6 +34,9 @@ #endif namespace mindspore { + +/// \brief The available model files have different model types. The ModelType enumeration variable represents the type of model file. +/// \brief When ModelType is 0, 1, 2, 3, 4, or 0xFFFFFFFF, the corresponding model types are kMindIR, kAIR, kOM, kONNX, kMindIR_Lite, and unknown type, respectively. enum ModelType : uint32_t { kMindIR = 0, kAIR = 1, @@ -369,7 +372,12 @@ struct MSCallBackParam { using MSKernelCallBack = std::function &inputs, const std::vector &outputs, const MSCallBackParam &opInfo)>; +/// \brief The CharVersion method calls the mindspore::lite::Version() method to obtain the version number of MindSpore Lite. +/// \return the version as a string. std::vector CharVersion(); + +/// \brief The Version method is used to retrieve the version number. +/// \return the version of MindSpore Lite as a string. inline std::string Version() { return CharToString(CharVersion()); } } // namespace mindspore diff --git a/mindspore/ccsrc/cxx_api/graph/graph.cc b/mindspore/ccsrc/cxx_api/graph/graph.cc index 7b6602d211e..1aff2071fd8 100644 --- a/mindspore/ccsrc/cxx_api/graph/graph.cc +++ b/mindspore/ccsrc/cxx_api/graph/graph.cc @@ -28,8 +28,14 @@ Graph::~Graph() {} Graph::Graph(std::nullptr_t) : graph_data_(nullptr) {} +/// \brief Overload the operator to determine if the graph model (the object pointed to by graph_data) fails to load. +/// If it fails to load, graph_data_==nullptr, and this method returns True. +/// Due to the possibility that the Graph object(this pointer) may be constant, and C++ rules dictate that only pointers to constants can be used to store the address of constant objects, the const keyword is used to modify the this pointer. bool Graph::operator==(std::nullptr_t) const { return graph_data_ == nullptr; } +/// \brief Overload the operator to determine whether the graph model (the object pointed to by graph_data) is loaded successfully. +/// If it is loaded successfully, graph_data_ != nullptr, and this method returns True. +/// Due to the possibility that the Graph object(this pointer) may be constant, and C++ rules dictate that only pointers to constants can be used to store the address of constant objects, the const keyword is used to modify the this pointer. bool Graph::operator!=(std::nullptr_t) const { return graph_data_ != nullptr; } ModelType Graph::ModelType() const { diff --git a/mindspore/ccsrc/cxx_api/graph/graph_data.h b/mindspore/ccsrc/cxx_api/graph/graph_data.h index 168eeb531ae..0b4b5f8091c 100644 --- a/mindspore/ccsrc/cxx_api/graph/graph_data.h +++ b/mindspore/ccsrc/cxx_api/graph/graph_data.h @@ -26,7 +26,7 @@ #include "ir/func_graph.h" namespace mindspore { -class Graph::GraphData { +class Graph::GraphData {/// Define the inner class GraphData of Graph. public: GraphData(); diff --git a/mindspore/ccsrc/pipeline/jit/parse/data_converter.cc b/mindspore/ccsrc/pipeline/jit/parse/data_converter.cc index 6da9e08559c..e62b1e1602b 100644 --- a/mindspore/ccsrc/pipeline/jit/parse/data_converter.cc +++ b/mindspore/ccsrc/pipeline/jit/parse/data_converter.cc @@ -41,7 +41,7 @@ struct PyDataToValueRegister { PyDataToValueRegister() { python_adapter::PyAdapterCallback::SetPyDataToValueHandler(data_converter::PyDataToValue); } } callback_register; } // namespace -using Tensor = mindspore::tensor::Tensor; +using Tensor = mindspore::tensor::Tensor; //导入变量 using TensorPtr = mindspore::tensor::TensorPtr; using MetaTensor = mindspore::tensor::MetaTensor; using MetaTensorPtr = mindspore::tensor::MetaTensorPtr; @@ -57,19 +57,19 @@ static constexpr int kBit16 = 16; static constexpr int kBit32 = 32; static constexpr int kBit64 = 64; -class DataConverter { +class DataConverter { // 数据转换器 public: - explicit DataConverter(InstanceConvertFunc convert_func) : convert_func_(std::move(convert_func)) {} + explicit DataConverter(InstanceConvertFunc convert_func) : convert_func_(std::move(convert_func)) {} //不允许隐式转换 - virtual ~DataConverter() = default; + virtual ~DataConverter() = default; // 虚析构函数 virtual bool Matched(const py::object &obj) = 0; - virtual ValuePtr ConvertPyObject(const py::object &obj, bool use_sig, const TypePtr &dtype) { - if (convert_func_ == nullptr) { + virtual ValuePtr ConvertPyObject(const py::object &obj, bool use_sig, const TypePtr &dtype) { //将python对象转换为valueptr + if (convert_func_ == nullptr) { //如果转换函数为空，使用 MS_LOG 宏记录一条异常日志，并抛出异常。 MS_LOG(EXCEPTION) << "convert func is null"; } - return convert_func_(obj, use_sig, dtype); + return convert_func_(obj, use_sig, dtype); //正常调用转换函数 } private: @@ -83,7 +83,7 @@ using ArgsObjSigConvertFunc = std::function; using ArgsOjbTypeConvertFunc = std::function; // Convert the data according instance type -template +template //模板类，通过不同的构造函数和类型检查函数，可以根据实例的类型匹配和执行不同的转换操作。 class ByTypeDataConverter : public DataConverter { public: explicit ByTypeDataConverter(const InstanceConvertFunc &convert_func) @@ -113,14 +113,14 @@ class ByTypeDataConverter : public DataConverter { ~ByTypeDataConverter() override = default; - bool Matched(const py::object &obj) override { return check_func_ != nullptr ? check_func_(obj) : false; } + bool Matched(const py::object &obj) override { return check_func_ != nullptr ? check_func_(obj) : false; } //用于检查传入的 Python 对象是否与该类型匹配 private: InstanceCheckFunc check_func_ = nullptr; }; // Convert the data according object attribute. -class ByAttrDataConverter : public DataConverter { +class ByAttrDataConverter : public DataConverter { //通过不同的构造函数和属性检查函数，可以根据对象的属性进行匹配和执行不同的转换操作。 public: ByAttrDataConverter(const std::string &attr_name, const ArgsObjConvertFunc &convert_func) : DataConverter( @@ -135,40 +135,40 @@ class ByAttrDataConverter : public DataConverter { ~ByAttrDataConverter() override = default; - bool Matched(const py::object &obj) override { return py::hasattr(obj, attr_name_.c_str()); } + bool Matched(const py::object &obj) override { return py::hasattr(obj, attr_name_.c_str()); } //用于检查传入的 Python 对象是否具有指定的属性。 private: std::string attr_name_; }; -FuncGraphPtr ConvertToBpropCut(const py::object &obj) { - std::vector results = data_converter::GetObjKey(obj); +FuncGraphPtr ConvertToBpropCut(const py::object &obj) { //将给定的 Python 对象转换为一个反向传播截断的 FuncGraphPtr 对象 + std::vector results = data_converter::GetObjKey(obj); //获取对象的关键字，将传入的 Python 对象转换为一个字符串数组 std::string obj_key = results[0]; - py::function bprop_func = py::getattr(obj, CUSTOM_BPROP_NAME); + py::function bprop_func = py::getattr(obj, CUSTOM_BPROP_NAME); //获取对象的反向传播函数 - auto bprop_graph = std::make_shared(); - std::vector outputs; + auto bprop_graph = std::make_shared(); //创建一个新的 FuncGraph 对象，用于构建反向传播图。 + std::vector outputs; //用于存储反向传播图的输出节点 - auto fake_bprop = std::make_shared("bprop_cut"); + auto fake_bprop = std::make_shared("bprop_cut"); //创建一个新的 PrimitivePy 对象，用于构建反向传播图的输出节点。 fake_bprop->AddBackwardHookFn(0, bprop_func); (void)fake_bprop->AddAttr(CUSTOM_BPROP_NAME, MakeValue(true)); outputs.push_back(NewValueNode(fake_bprop)); - py::object code_obj = py::getattr(bprop_func, "__code__"); + py::object code_obj = py::getattr(bprop_func, "__code__"); //获取反向传播函数的代码对象 // Three parameters self, out and dout need to be excluded - constexpr auto kBpropExcludeParamNum = 3; - size_t inputs_num = py::cast(py::getattr(code_obj, "co_argcount")) - kBpropExcludeParamNum; - for (size_t i = 0; i < inputs_num; ++i) { + constexpr auto kBpropExcludeParamNum = 3; //反向传播函数的参数个数 + size_t inputs_num = py::cast(py::getattr(code_obj, "co_argcount")) - kBpropExcludeParamNum; //计算反向传播函数的输入参数个数 + for (size_t i = 0; i < inputs_num; ++i) { //创建反向传播函数的输入参数节点 auto param = bprop_graph->add_parameter(); outputs.push_back(param); } - auto p1 = bprop_graph->add_parameter(); + auto p1 = bprop_graph->add_parameter(); //创建反向传播函数的输出参数节点 auto p2 = bprop_graph->add_parameter(); outputs.push_back(p1); outputs.push_back(p2); - bprop_graph->set_output(bprop_graph->NewCNode(std::move(outputs))); - data_converter::SetObjGraphValue(obj_key, bprop_graph); + bprop_graph->set_output(bprop_graph->NewCNode(std::move(outputs))); //设置反向传播函数的输出节点 + data_converter::SetObjGraphValue(obj_key, bprop_graph); //将反向传播函数的 FuncGraphPtr 对象存储到全局字典中 return bprop_graph; } diff --git a/mindspore/ccsrc/pipeline/pynative/pynative_execute.cc b/mindspore/ccsrc/pipeline/pynative/pynative_execute.cc index 0be7e3db32c..78b611a1484 100644 --- a/mindspore/ccsrc/pipeline/pynative/pynative_execute.cc +++ b/mindspore/ccsrc/pipeline/pynative/pynative_execute.cc @@ -84,6 +84,7 @@ std::map> kSessionBackends; std::map> kMindRtBackends; PyObjectIdCache g_pyobj_id_cache; +// A warpper to test function and handle the exception template void PynativeExecutorTry(const std::function &method, T *ret, const Args &... args) { const auto inst = PynativeExecutor::GetInstance(); @@ -2807,6 +2808,7 @@ void GradExecutor::GradNetInner(py::object *ret, const prim::GradOperationPtr &g auto size = args.size(); const auto &cell_id = GetGradCellId(grad->sens_param(), cell, args); MS_LOG(DEBUG) << "GradNet start " << size << " " << cell_id; + // return if cell has been compiled if (!top_cell()->need_compile_graph()) { MS_LOG(DEBUG) << "No need compile graph"; if (!cell_stack_.empty()) { @@ -3169,6 +3171,7 @@ void GradExecutor::RunGradGraph(py::object *ret, const py::object &cell, const p MS_EXCEPTION_IF_NULL(resource); MS_LOG(DEBUG) << "Run resource ptr " << resource.get(); + // Get args for run graph VectorRef arg_list; auto filter_args = FilterTensorArgs(args, has_sens); py::tuple converted_args = ConvertArgs(filter_args); @@ -3178,6 +3181,7 @@ void GradExecutor::RunGradGraph(py::object *ret, const py::object &cell, const p compile::VmEvalFuncPtr run = resource->GetResult(pipeline::kOutput).cast(); MS_EXCEPTION_IF_NULL(run); + // Run graph const auto &backend = MsContext::GetInstance()->backend_policy(); MS_LOG(DEBUG) << "Eval run " << backend; grad_is_running_ = true; @@ -3478,10 +3482,12 @@ void GradExecutor::ClearRes() { std::stack().swap(high_order_stack_); } +// return GradExecutor object Ptr grad_executor_ GradExecutorPtr PynativeExecutor::grad_executor() const { MS_EXCEPTION_IF_NULL(grad_executor_); return grad_executor_; } +// return ForwardExecutor object Ptr forward_executor_ ForwardExecutorPtr PynativeExecutor::forward_executor() const { MS_EXCEPTION_IF_NULL(forward_executor_); return forward_executor_; @@ -3533,6 +3539,7 @@ py::object PynativeExecutor::CheckAlreadyRun(const prim::GradOperationPtr &grad, return grad_executor()->CheckAlreadyRun(grad, cell, args); } +// PynativeExecutor_.__call__ py::object PynativeExecutor::Run(const py::object &cell, const py::tuple &args) { py::object ret; PynativeExecutorTry(grad_executor()->RunGraph, &ret, cell, args); @@ -3604,6 +3611,7 @@ py::object PynativeExecutor::GradMsFunction(const py::object &out, const py::arg return grad_executor()->GradMsFunction(out, args); } +// PynativeExecutor_.grad_net void PynativeExecutor::GradNet(const prim::GradOperationPtr &grad, const py::object &cell, const py::object &weights, const py::object &grad_position, const py::args &args) { py::object ret; diff --git a/mindspore/ccsrc/pipeline/pynative/pynative_execute.h b/mindspore/ccsrc/pipeline/pynative/pynative_execute.h index 4d6b1b4d0dd..63d9f98f1c8 100644 --- a/mindspore/ccsrc/pipeline/pynative/pynative_execute.h +++ b/mindspore/ccsrc/pipeline/pynative/pynative_execute.h @@ -398,6 +398,7 @@ class ForwardExecutor { std::string last_target_{"Unknown"}; }; +// The executor for pynative mode, std::enable_shared_from_this is for get shared_ptr of this. class PynativeExecutor : public std::enable_shared_from_this { public: static std::shared_ptr GetInstance() { @@ -449,6 +450,7 @@ class PynativeExecutor : public std::enable_shared_from_this { private: PynativeExecutor() = default; + // member variables static std::shared_ptr executor_; static std::mutex instance_lock_; static ForwardExecutorPtr forward_executor_; diff --git a/mindspore/python/mindspore/boost/adasum.py b/mindspore/python/mindspore/boost/adasum.py index 136bdd7e56d..ca1e76681b5 100644 --- a/mindspore/python/mindspore/boost/adasum.py +++ b/mindspore/python/mindspore/boost/adasum.py @@ -22,31 +22,32 @@ from mindspore.ops import composite as C from mindspore.ops import functional as F from mindspore.ops import operations as P from mindspore.ops.operations._inner_ops import Send, Receive - - +#from copy, hashlib, math and other other files export the required packages +#The content of this document defines a distributed machine learning gradient fusion algorithm-Adasum,Adaptive Summary can improve the accuracy of training for clusters of different scales and reduce the difficulty of parameter tuning for clusters of different scales. __all__ = ["AdaSum"] -MAX_NUM_HASH = 2 ** 31 +MAX_NUM_HASH = 2 ** 31 #Hash Table Parameters -_update_parameters = C.MultitypeFuncGraph("update_parameters") +_update_parameters = C.MultitypeFuncGraph("update_parameters") #Define update parameters and call the MultitypeFuncGraph class of composite in mindspore.ops. MultitypeFuncGraph is a class used to generate overloaded functions, using different types as input. + #Decorate register types with a register register register with input types. Enable this function to use different types as input calls. -@_update_parameters.register("Tensor", "Tensor", "Tensor", "Tensor") -def _update_parameters_after_broadcast(delta_weight, update_delta_weight, parameter, old_parameter): - shape = F.shape(delta_weight) - update_delta_weight = P.Reshape()(update_delta_weight, shape) - new_parameter = old_parameter - update_delta_weight - return P.Assign()(parameter, new_parameter) +@_update_parameters.register("Tensor", "Tensor", "Tensor", "Tensor") #The input types for register are four "Tensor". +def _update_parameters_after_broadcast(delta_weight, update_delta_weight, parameter, old_parameter): #after broadcasting, update the parameters. + shape = F.shape(delta_weight) #get the shape of functional + update_delta_weight = P.Reshape()(update_delta_weight, shape) #using reshape operation to get the new delta_weight. + new_parameter = old_parameter - update_delta_weight #get new parameters using sub subtraction. + return P.Assign()(parameter, new_parameter) #return the tensor using the Assign which has two Variable. -def _send_before_receive(send_part, send, recv): +def _send_before_receive(send_part, send, recv): #using send to finish this work. send_ok = send(send_part) return recv(send_ok) -def _receive_before_send(send_part, send, recv): +def _receive_before_send(send_part, send, recv): #receive the send result. receive_ok = recv(send_part) send_part = F.depend(send_part, receive_ok) return F.depend(receive_ok, send(send_part)) @@ -156,7 +157,7 @@ class AdaSum(Cell): Outputs: - **adasum_parameters** (Tuple(Tensor)) - Tuple of parameters after adasum process. """ - def __init__(self, rank, device_number, group_number, parameter_tuple): + def __init__(self, rank, device_number, group_number, parameter_tuple): #Initialize the Adasum class and declare the hypermap and _geberate_communication_op. super(AdaSum, self).__init__() self.rank = rank self.device_number = device_number @@ -165,7 +166,7 @@ class AdaSum(Cell): self._generate_communication_op() self.hyper_map = C.HyperMap() - def _generate_communication_op(self): + def _generate_communication_op(self): #Define this method specifically. """generate communication op.""" self.calc_times = int(math.log(self.group_number, 2)) self.send_node = [] @@ -178,28 +179,28 @@ class AdaSum(Cell): self.parameter_divisibility_list = [] self.allreduce_node_num_list = [] last_delta_weights = [] - group_start_rank = (self.rank // self.device_number) * self.device_number + group_start_rank = (self.rank // self.device_number) * self.device_number #declare some list and some variable. for step in range(self.calc_times): - current_group = self.device_number * (2 ** step) + current_group = self.device_number * (2 ** step) #current_group=device_number*pow(2,step) sr_target = self.rank - if (sr_target // current_group) % 2 == 0: - dest_target = sr_target + current_group + if (sr_target // current_group) % 2 == 0: #if sr_target divided by current_group is an even number. + dest_target = sr_target + current_group #dest_target is the result of adding the two variables. self.send_node.append(True) - else: - dest_target = sr_target - current_group + else: #if it is odd number. + dest_target = sr_target - current_group #subtraction self.send_node.append(False) neighbor_ids = [] group_name_last = 0 for index in range(2 ** (step + 1)): - node_rank = self.rank // self.device_number - double_d = 2 ** (step + 1) + node_rank = self.rank // self.device_number #get every node rank + double_d = 2 ** (step + 1) neighbor_id = (node_rank // double_d * double_d + index) * self.device_number + \ - self.rank % self.device_number + self.rank % self.device_number #get neighbor_id using the 2**(step+1) and self.devicenumber and self. neighbor_ids.append(neighbor_id) group_name_last += neighbor_id - group_name = "adasum_" + str(step) + "_" + str(group_name_last) + group_name = "adasum_" + str(step) + "_" + str(group_name_last) #Utilize newly acquired neighbors_update group information with ID. create_group(group_name, neighbor_ids) send_left = [] @@ -208,13 +209,13 @@ class AdaSum(Cell): recv_right = [] allreduce_node_num = () left_delta_weights, right_delta_weights, delta_weights_divisibility = \ - self._get_delta_weights_info(last_delta_weights) + self._get_delta_weights_info(last_delta_weights) #using the _get_delta_weights_info method to update the variabls. self.parameter_divisibility_list.append(delta_weights_divisibility) weights_index = 0 fusion_id = (step + 1) * 3 - for shape, dtype in left_delta_weights: + for shape, dtype in left_delta_weights: #using hash to get the tag send_tag = self._hash(step, sr_target, weights_index) - send = Send(sr_tag=send_tag, dest_rank=dest_target, group="hccl_world_group") + send = Send(sr_tag=send_tag, dest_rank=dest_target, group="hccl_world_group") #choose the type of group is "hccl_world_group" send.add_prim_attr("fusion", fusion_id) recv_tag = self._hash(step, dest_target, weights_index) recv = Receive(sr_tag=recv_tag, src_rank=dest_target, shape=shape, dtype=dtype, @@ -269,38 +270,42 @@ class AdaSum(Cell): def _get_delta_weights_info(self, last_delta_weights): """get delta weights info.""" half_delta_weights = [] - if last_delta_weights: + if last_delta_weights: #if the last one is exist, we should use the last one half_delta_weights = last_delta_weights else: - for parameter in self.parameter_tuple: + for parameter in self.parameter_tuple: # else we need to create a new half_delta_weights using the parameter new_shape = [int(x) for x in parameter.shape] half_delta_weights.append((new_shape, parameter.dtype)) left_delta_weights = [] right_delta_weights = [] delta_weights_divisibility = () - for shape, dtype in half_delta_weights: + for shape, dtype in half_delta_weights: #init left_shape = copy.deepcopy(shape) right_shape = copy.deepcopy(shape) divisibility_flag = False - for i in range(len(shape)): + for i in range(len(shape)): #left and right are half part of init shape if shape[i] > 1: left_shape[i] = int(shape[i] // 2) right_shape[i] = shape[i] - int(shape[i] // 2) - divisibility_flag = True + divisibility_flag = True #set the flag is true, which means exists the matrix which shape > 1 break left_delta_weights.append((left_shape, dtype)) right_delta_weights.append((right_shape, dtype)) - delta_weights_divisibility += (divisibility_flag,) + delta_weights_divisibility += (divisibility_flag,) #count the num of shape > 1 return left_delta_weights, right_delta_weights, delta_weights_divisibility def _hash(self, step, target, weights_index): - target = "tag" + str(step) + str(target) + str(weights_index) + target = "tag" + str(step) + str(target) + str(weights_index) #combinate the step, target,a nd weights index as the target target_hash = hashlib.sha1(target.encode()).hexdigest() - hash_res = int(int(target_hash, 16) % MAX_NUM_HASH) + hash_res = int(int(target_hash, 16) % MAX_NUM_HASH) #ensure the result is "int" type return hash_res def construct(self, delta_weights, parameters, old_parameters): - forward_weights = [delta_weights] + forward_weights = [delta_weights] #init the forward_weights + """HyperMap is a special class that requires passing in the mapping function f when constructing class objects, and passing in n parameter sequences of f when calling objects. + For more usage methods, see HyperMap. The mapping function f must be of type MultitypeFuncGraph, which can be referenced. + When using the for loop to batch process list elements, network compilation performance can be optimized through HyperMap equivalent semantic substitution. + """ for i in range(self.calc_times): process_weights = self.hyper_map(F.partial(_adasum_opt_forward, self.send_node[i], self.allreduce_list[i]), self.parameter_divisibility_list[i], self.allreduce_node_num_list[i], @@ -313,5 +318,5 @@ class AdaSum(Cell): self.send_list_rollback[j], self.recv_list_rollback[j]) forward_weights[j] = process_weights adasum_parameters = self.hyper_map(F.partial(_update_parameters), delta_weights, forward_weights[0], - parameters, old_parameters) + parameters, old_parameters) #after the hyper_map optimize will get the adasum_paramater return adasum_parameters diff --git a/mindspore/python/mindspore/boost/base.py b/mindspore/python/mindspore/boost/base.py index 61c246e3c6e..c15ac424975 100644 --- a/mindspore/python/mindspore/boost/base.py +++ b/mindspore/python/mindspore/boost/base.py @@ -21,14 +21,14 @@ import numpy as np from scipy import linalg as la from mindspore.context import ParallelMode import mindspore.nn as nn -from mindspore.nn.optim import LARS +from mindspore.nn.optim import LARS #import the Layer-wise Adaptive Rate Scaling. from mindspore import log as logger -from mindspore.common import Parameter +from mindspore.common import Parameter #`Parameter` is a `Tensor` subclass, when they are assigned as Cell attributes they are automatically added to the list of it`s parameters. from mindspore.communication.management import get_group_size from mindspore.train.serialization import load_checkpoint from mindspore.parallel._utils import _get_global_rank from mindspore.parallel._auto_parallel_context import auto_parallel_context -from .less_batch_normalization import CommonHeadLastFN +from .less_batch_normalization import CommonHeadLastFN #from other documents import something. __all__ = ["OptimizerProcess", "ParameterProcess"] @@ -67,7 +67,7 @@ class OptimizerProcess: >>> optimizer = optimizer_process.generate_new_optimizer() """ def __init__(self, opt): - if isinstance(opt, LARS): + if isinstance(opt, LARS): #Use isinstance to determine whether the optimizer is LARS,which is a Iterative algorithm for fast Feature selection and regression coefficient calculation for linear regression problems. self.is_lars = True self.single_opt = opt.opt self.opt_class = type(opt.opt) @@ -80,7 +80,7 @@ class OptimizerProcess: self.opt_class = type(opt) self.opt_init_args = opt.init_args self.learning_rate = opt.init_learning_rate - self.origin_params = opt.init_params["params"] + self.origin_params = opt.init_params["params"] #set the origin_params as opt.init_params def build_params_dict(self, network): r""" @@ -89,9 +89,9 @@ class OptimizerProcess: Args: network (Cell): The training network. """ - cells = network.cells_and_names() + cells = network.cells_and_names() #Initialize cells. params_dict = {} - for _, cell in cells: + for _, cell in cells: #Traverse each cell and utilize cell.get_The comparison of parameters to parameters_Assignment operation of dict. for par in cell.get_parameters(expand=False): params_dict[id(par)] = cell return params_dict @@ -105,31 +105,31 @@ class OptimizerProcess: parameters (list): The network's parameter list. """ group_params = [] - for group_param in parameters: - if 'order_params' in group_param.keys(): - group_params.append(group_param) + for group_param in parameters: #Fill in group_params. + if 'order_params' in group_param.keys(): + group_params.append(group_param) #need to determine whether thers is an order_params' in the keys of group_param, only append when existing continue params_gc_value = [] params_value = [] for param in group_param['params']: - if 'beta' not in param.name and 'gamma' not in param.name and 'bias' not in param.name: - param_cell = params_dict[id(param)] + if 'beta' not in param.name and 'gamma' not in param.name and 'bias' not in param.name: #param.name is not the beta' or the gamma' or the bias + param_cell = params_dict[id(param)] #only in this situation ,the param_cell will be assigned. if (isinstance(param_cell, nn.Conv2d) and param_cell.group > 1) or \ isinstance(param_cell, CommonHeadLastFN): - params_value.append(param) + params_value.append(param) #if the param_cell is nn.Conv2d and group > 1 or param_cell is CommonHeadLastFN. else: - params_gc_value.append(param) + params_gc_value.append(param) #or append into gc else: - params_value.append(param) - if params_gc_value: + params_value.append(param) #if not, append into the params_value. + if params_gc_value: #if list of gc is not null. new_group_param = copy.deepcopy(group_param) new_group_param['params'] = params_gc_value new_group_param['grad_centralization'] = True - group_params.append(new_group_param) + group_params.append(new_group_param) #append the params_gc_value to the group_params if params_value: new_group_param = copy.deepcopy(group_param) new_group_param['params'] = params_value - group_params.append(new_group_param) + group_params.append(new_group_param) #in the same way append the params_value to the group_params. return group_params def add_grad_centralization(self, network): @@ -141,27 +141,27 @@ class OptimizerProcess: """ params_dict = self.build_params_dict(network) - parameters = self.origin_params + parameters = self.origin_params #set the orign parameters if parameters is not None and not isinstance(parameters, list): - parameters = list(parameters) + parameters = list(parameters) #convert the parameters to the lists if not parameters: raise ValueError("Optimizer got an empty parameter list.") - if not isinstance(parameters[0], (dict, Parameter)): + if not isinstance(parameters[0], (dict, Parameter)): #Illegal situation judgment. raise TypeError("Only a list of Parameter or dict can be supported.") - if isinstance(parameters[0], Parameter): - logger.warning("Only group parameters support gradient centralization.") + if isinstance(parameters[0], Parameter): #if the parameter is just the list of Parameter, the network can not solve it. + logger.warning("Only group parameters support gradient centralization.") #so that means a warning need to be showed. return self.origin_params = self.build_gc_params_group(params_dict, parameters) def generate_new_optimizer(self): """Generate new optimizer.""" - if self.learning_rate is None: + if self.learning_rate is None: #check the learning_rate self.learning_rate = self.single_opt.learning_rate - if not self.is_lars: + if not self.is_lars: #check whether using the Least Angle Regression opt = self.opt_class(params=self.origin_params, learning_rate=self.learning_rate, **self.opt_init_args) else: opt = LARS(self.opt_class(params=self.origin_params, learning_rate=self.learning_rate, \ @@ -212,18 +212,18 @@ class ParameterProcess: parameters (list): The network's parameter list. split_point (list): The gradient split point of this network. default: None. """ - if not isinstance(parameters, (list, tuple)) or not parameters: + if not isinstance(parameters, (list, tuple)) or not parameters: #check whether the parameter's typt is list or tuple. return parameters parameter_len = len(parameters) - if split_point: + if split_point: #using the split_point to assign the index split_parameter_index = split_point else: - split_parameter_index = [parameter_len // 2] + split_parameter_index = [parameter_len // 2] #or using the len for i in range(parameter_len): if i in split_parameter_index: self._parameter_indices += 1 - parameters[i].comm_fusion = self._parameter_indices + parameters[i].comm_fusion = self._parameter_indices #assign the parameters[i].comm_fusion using the parametes_indices which is added by the index return parameters def generate_group_params(self, parameters, origin_params): @@ -234,10 +234,10 @@ class ParameterProcess: parameters (list): The network's parameter list. origin_params (list): The network's origin parameter list. """ - origin_params_copy = origin_params + origin_params_copy = origin_params #create a copy of the origin_params, which means we can revise the copy without changing the original one if origin_params_copy is not None: if not isinstance(origin_params_copy, list): - origin_params_copy = list(origin_params_copy) + origin_params_copy = list(origin_params_copy) #convert its type to list if not origin_params_copy: raise ValueError("Optimizer got an empty parameter list.") @@ -245,13 +245,13 @@ class ParameterProcess: if not isinstance(origin_params_copy[0], (dict, Parameter)): raise TypeError("Only a list of Parameter or dict can be supported.") - if isinstance(origin_params_copy[0], Parameter): + if isinstance(origin_params_copy[0], Parameter): #some check about its legality. group_params = [{"params": parameters}] return group_params - + # the operation below is similar to the operation in the build_gc_params_group,check the order_params' and fufill the group_params group_params = [] params_name = [param.name for param in parameters] - new_params_count = copy.deepcopy(params_name) + new_params_count = copy.deepcopy(params_name) #using the deepcopy to get the new_params_count so we will not change the value of the original params_name new_params_clone = {} max_key_number = 0 for group_param in origin_params_copy: @@ -298,34 +298,34 @@ def _get_local_pca_mat_path(weight_load_dir, pca_mat_path, n_component, device_n """ if pca_mat_path is not None and os.path.exists(pca_mat_path) and os.path.isfile(pca_mat_path) and \ pca_mat_path.endswith(".npy"): - full_pca_mat_path = pca_mat_path - pca_mat_exist = True + full_pca_mat_path = pca_mat_path # if the pc_mat_path is exists in the os.path and isfile is OK,set the full_pca_mat_path and pca_mat_exists + pca_mat_exist = True else: if weight_load_dir is None or not os.path.exists(weight_load_dir) or not os.path.isdir(weight_load_dir): - raise ValueError("The weight_load_dir: {} is None / not exists / not directory.".format(weight_load_dir)) + raise ValueError("The weight_load_dir: {} is None / not exists / not directory.".format(weight_load_dir)) #if not ,show the ValueError about "The weight_load_dir: {} is None / not exists / not directory". full_pca_mat_path = os.path.join(weight_load_dir, "pca_mat_temp.npy") pca_mat_exist = False save_pca_end_path = os.path.join(os.path.dirname(full_pca_mat_path), "save_pca_end.txt") - if os.path.exists(save_pca_end_path): + if os.path.exists(save_pca_end_path): #erase the old path. os.remove(save_pca_end_path) - rank = _get_global_rank() + rank = _get_global_rank() #assign the rank using the _get_global_rank, so the rank is global local_pca_mat_path = full_pca_mat_path[:-4] + "_rank_" + str(rank) + ".npy" - if os.path.exists(local_pca_mat_path): + if os.path.exists(local_pca_mat_path): #erase the old path os.remove(local_pca_mat_path) - if rank % device_number != 0: + if rank % device_number != 0: #need to ensure the rank is the multiple of device_number return local_pca_mat_path if pca_mat_exist: - pca_mat = np.load(full_pca_mat_path) + pca_mat = np.load(full_pca_mat_path) #using numpy to get the pac_mat else: - data = _load_weights(weight_load_dir, network) + data = _load_weights(weight_load_dir, network) #assign data using the _load_weights function which args is where to load the weighs and the network the weights ues to defined below. pca_mat = _compute_pca_mat(data, n_component) np.save(full_pca_mat_path, pca_mat) - _save_local_pca_mat(pca_mat, full_pca_mat_path, n_component) + _save_local_pca_mat(pca_mat, full_pca_mat_path, n_component) #the save function is completed by the function named _save_local_pca_mat return local_pca_mat_path @@ -338,7 +338,7 @@ def _load_weights(weight_load_dir, network): network (Cell): The network. """ param_requires_grad_list = [] - for param in network.trainable_params(): + for param in network.trainable_params(): #fufill the param_list using the param.name param_requires_grad_list.append(param.name) param_mat_tuple = () @@ -346,19 +346,19 @@ def _load_weights(weight_load_dir, network): for file in weight_file_list: if not file.endswith('.ckpt'): continue - file_path = os.path.join(weight_load_dir, file) - param_dict = load_checkpoint(file_path) + file_path = os.path.join(weight_load_dir, file) #get the file path using the dir and file name + param_dict = load_checkpoint(file_path) #we can use the file_path to set a checkpoint to save the weights param_tuple = () for key, value in param_dict.items(): - if key in param_requires_grad_list: + if key in param_requires_grad_list: #check whether the key is in the param_requires_grad_list to operate the param_tuple param_tuple += (value.asnumpy().reshape((1, -1)),) - param = np.concatenate(param_tuple, axis=1) + param = np.concatenate(param_tuple, axis=1) #concatenate all the tuple to the param param_mat_tuple += (param,) - param_mat = np.concatenate(param_mat_tuple, axis=0) + param_mat = np.concatenate(param_mat_tuple, axis=0) #concatenate all the mat tuple to the param_mat return param_mat -def _compute_pca_mat(data, n_component, randomized=True): +def _compute_pca_mat(data, n_component, randomized=True): #randomized=True means the components is the result of the _randomized_svd. """ compute pca mat. @@ -375,7 +375,7 @@ def _compute_pca_mat(data, n_component, randomized=True): if randomized: components = _randomized_svd(data, n_component) else: - components = _full_svd(data, n_component) + components = _full_svd(data, n_component) #or no need to randomized it return components @@ -392,24 +392,24 @@ def _randomized_svd(data, n_component, n_oversample=10, n_iter=1): n_oversample (int): oversample num n_iter (int): iteration count """ - mean = np.mean(data, axis=0) + mean = np.mean(data, axis=0) #get the average data -= mean - n_random = n_component + n_oversample + n_random = n_component + n_oversample #the n_random is the addition of the n_component and the n_oversample n_samples, n_features = data.shape - transpose = n_samples < n_features + transpose = n_samples < n_features # the value transpose recorsd the relation between the n_samples and the n_features if transpose: - data = data.T + data = data.T #Determine whether the data matrix needs to be transposed based on samples and features q_mat = _randomized_range_finder(data, n_random, n_iter) - b_mat = q_mat.T @ data - u_hat, _, vt_mat = la.svd(b_mat, full_matrices=False) + b_mat = q_mat.T @ data #b_mat is the randomed result and the multiply of data + u_hat, _, vt_mat = la.svd(b_mat, full_matrices=False) #using the la.svd del b_mat u_mat = np.dot(q_mat, u_hat) - u_mat, vt_mat = _svd_flip(u_mat, vt_mat, transpose) + u_mat, vt_mat = _svd_flip(u_mat, vt_mat, transpose) #using the _svd_flip if transpose: components = u_mat[:, :n_component].T else: components = vt_mat[:n_component, :] - return components + return components #components depends on the number of samples and features,if the ranspose is true, usinf the u_ma, or the vt_mat. def _full_svd(data, n_component): @@ -422,11 +422,11 @@ def _full_svd(data, n_component): and `n_features` is the number of features. n_component (int): pca component. """ - mean = np.mean(data, axis=0) + mean = np.mean(data, axis=0) #get the average data -= mean u, _, v = la.svd(data, full_matrices=False) _, v = _svd_flip(u, v) - components = v[:n_component] + components = v[:n_component] #from 0 to n_component return components @@ -444,7 +444,7 @@ def _randomized_range_finder(data, size, n_iter=1): q_mat = np.random.normal(size=(data.shape[1], size)) for _ in range(n_iter): - q_mat, _ = la.lu(data @ q_mat, permute_l=True) + q_mat, _ = la.lu(data @ q_mat, permute_l=True) q_mat, _ = la.lu(data.T @ q_mat, permute_l=True) q_mat, _ = la.qr(data @ q_mat, mode="economic") diff --git a/mindspore/python/mindspore/common/api.py b/mindspore/python/mindspore/common/api.py index f8c2542e423..7ed9d084f08 100644 --- a/mindspore/python/mindspore/common/api.py +++ b/mindspore/python/mindspore/common/api.py @@ -873,6 +873,7 @@ class _PynativeExecutor: Return: The return object after running grad graph. """ + # set the top cell args = args + tuple(kwargs.values()) return self._executor(obj, args) diff --git a/mindspore/python/mindspore/dataset/datapreprocess/preprocess_imagenet_validate_dataset.py b/mindspore/python/mindspore/dataset/datapreprocess/preprocess_imagenet_validate_dataset.py index 3790ace5aeb..b0053c52a2d 100644 --- a/mindspore/python/mindspore/dataset/datapreprocess/preprocess_imagenet_validate_dataset.py +++ b/mindspore/python/mindspore/dataset/datapreprocess/preprocess_imagenet_validate_dataset.py @@ -20,6 +20,7 @@ from mindspore import log as logger def preprocess_imagenet_validation_dataset(train_dataset_path, validation_dataset_path, image_label_mapping_file): + # this function is used to preprocess imagenet validation dataset """ Call this function before read imagenet validation dataset. @@ -30,19 +31,22 @@ def preprocess_imagenet_validation_dataset(train_dataset_path, validation_datase """ train_dataset_path = os.path.realpath(train_dataset_path) sub_dir = [dir_.name for dir_ in os.scandir(train_dataset_path) if dir_.is_dir()] + # create sub dir for sub_dir_name in sub_dir: validate_sub_dir = os.path.join(validation_dataset_path, sub_dir_name) validate_sub_dir = os.path.realpath(validate_sub_dir) if not os.path.exists(validate_sub_dir): os.makedirs(validate_sub_dir, mode=stat.S_IRWXU) - real_file_path = os.path.realpath(image_label_mapping_file) + real_file_path = os.path.realpath(image_label_mapping_file) # imagenet_validate_dataset_2012_image_dir_map.txt mappings = [mapping.strip() for mapping in open(real_file_path).readlines()] + # move image to sub dir for mapping in mappings: image_dir = mapping.split(':') old_image_path = os.path.join(validation_dataset_path, image_dir[0]) old_image_path = os.path.realpath(old_image_path) - if not os.path.exists(old_image_path): + if not os.path.exists(old_image_path): # imagenet_validate_dataset_2012_image_dir_map.txt logger.warning('Image is not existed %s', old_image_path) + # move image to sub dir new_image_sub_dir = os.path.join(validation_dataset_path, image_dir[1]) new_image_sub_dir = os.path.realpath(new_image_sub_dir) new_image_path = os.path.join(new_image_sub_dir, image_dir[0]) diff --git a/mindspore/python/mindspore/dataset/utils/browse_dataset.py b/mindspore/python/mindspore/dataset/utils/browse_dataset.py index ed0601537c9..f42237a310a 100644 --- a/mindspore/python/mindspore/dataset/utils/browse_dataset.py +++ b/mindspore/python/mindspore/dataset/utils/browse_dataset.py @@ -143,7 +143,7 @@ def imshow_det_bbox(image, bboxes, labels, segm=None, class_names=None, score_th image = image.transpose((1, 2, 0)) draw_image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR) - if bboxes is not None: + if bboxes is not None: # draw bboxes bbox_num = bboxes.shape[0] for i in range(bbox_num): draw_bbox = bboxes[i] @@ -165,7 +165,7 @@ def imshow_det_bbox(image, bboxes, labels, segm=None, class_names=None, score_th if segm is not None: mask = segm[i].astype(bool) draw_image[mask] = draw_image[mask] * 0.5 + np.array(mask_color) * 0.5 - else: + else: # draw segm if segm is not None: segm_num = segm.shape[0] for i in range(segm_num): diff --git a/mindspore/python/mindspore/nn/cell.py b/mindspore/python/mindspore/nn/cell.py index c102bb75f92..114216d2c20 100755 --- a/mindspore/python/mindspore/nn/cell.py +++ b/mindspore/python/mindspore/nn/cell.py @@ -966,6 +966,7 @@ class Cell(Cell_): self.compile(*inputs) new_inputs = [] + # init data for i in inputs: if isinstance(i, Tensor): if i.has_init: @@ -981,6 +982,7 @@ class Cell(Cell_): _check_all_tensor(i): new_inputs.append(i) + # call _cell_graph_executor to run graph and calculate gradients if self._auto_parallel_mode: if new_inputs and isinstance(new_inputs[0], Tensor) and inputs[0].virtual_flag: # get parallel inputs in sink mode, parallel inputs set in _cell_graph_executor.compile diff --git a/mindspore/python/mindspore/numpy/array_creations.py b/mindspore/python/mindspore/numpy/array_creations.py index 9da4d03ce1e..95c52d19307 100644 --- a/mindspore/python/mindspore/numpy/array_creations.py +++ b/mindspore/python/mindspore/numpy/array_creations.py @@ -43,13 +43,13 @@ from .dtypes import nan, pi # According to official numpy reference, the dimension of a numpy array must be less # than 32 -MAX_NUMPY_DIMS = 32 +MAX_NUMPY_DIMS = 32 #this is a restriction of the matrix division # All types that can be accepted as "array_like" parameters in graph mode. ARRAY_TYPES = (int, float, bool, list, tuple, Tensor) -_reduce_min_keepdims = P.ReduceMin(True) -_reduce_max_keepdims = P.ReduceMax(True) -_reduce_mean_keepdims = P.ReduceMean(True) +_reduce_min_keepdims = P.ReduceMin(True) #initialize the keepdims +_reduce_max_keepdims = P.ReduceMax(True) +_reduce_mean_keepdims = P.ReduceMean(True) def array(obj, dtype=None, copy=True, ndmin=0): @@ -89,7 +89,7 @@ def array(obj, dtype=None, copy=True, ndmin=0): dtype = _check_dtype(dtype) res = asarray(obj, dtype) - if ndmin > res.ndim: + if ndmin > res.ndim: #check whether the dim go beyond the limitation if res.size == 0: _raise_value_error("Empty tensor cannot be expanded beyond the current dimension.") res = _expand(res, ndmin) @@ -110,10 +110,10 @@ def asarray_const(a, dtype=None): if dtype is not None: dtype = _check_dtype(dtype) - if isinstance(a, (float, int, bool)) and dtype is None: + if isinstance(a, (float, int, bool)) and dtype is None: #float, int, bool meams the single constant. dtype = _get_dtype_from_scalar(a) - if isinstance(a, (list, tuple)): + if isinstance(a, (list, tuple)): #or if the type is list or tuple, which is an ITERATOR # Convert all tuple/nested tuples to lists a = _deep_list(a) # Convert all tensor sub-elements to numpy arrays @@ -167,17 +167,17 @@ def asarray(a, dtype=None): >>> print(np.asarray([1,2,3])) [1 2 3] """ - if dtype is not None: + if dtype is not None: #before the operation, need to check the dtype to ensure its legality. dtype = _check_dtype(dtype) if isinstance(a, Tensor): - if dtype is None or dtype == a.dtype: - return a + if dtype is None or dtype == a.dtype: #if a is a Tensor and its dtype is the dtype, the function will return a + return a #else return a.astype return a.astype(dtype) - return asarray_const(a, dtype) + return asarray_const(a, dtype) #if a is not a tensor, then a is a const, return the result of asarray_const @constexpr -def asfarray_const(a, dtype=mstype.float32): +def asfarray_const(a, dtype=mstype.float32): # the function is similar to the """Converts the input to tensor. Note here `a` cannot be tensor itself.""" _check_input_for_asarray(a) if isinstance(a, (list, tuple)): @@ -185,7 +185,7 @@ def asfarray_const(a, dtype=mstype.float32): a = _deep_list(a) # Convert all tensor sub-elements to numpy arrays a = _deep_tensor_to_nparray(a) - a = onp.asarray(a) + a = onp.asarray(a) if a.dtype is onp.dtype('object'): raise ValueError(f"For Tensor conversion, the input_data is {a} that contains unsupported element.") a = Tensor.from_numpy(a) @@ -225,8 +225,8 @@ def asfarray(a, dtype=mstype.float32): if dtype is None: return asarray(a) - dtype = _check_dtype(dtype) - if dtype not in (mstype.float16, mstype.float32, mstype.float64): + dtype = _check_dtype(dtype) + if dtype not in (mstype.float16, mstype.float32, mstype.float64): #three type of the dtype , which means the numpy have three type of float ,16,32 and 64. dtype = mstype.float32 if isinstance(a, Tensor): @@ -260,7 +260,7 @@ def copy_(a): [[1. 1.] [1. 1.]] """ - a = asarray(a) + a = asarray(a) #ensure the a is an array return a.copy() @@ -289,11 +289,11 @@ def ones(shape, dtype=mstype.float32): [[1. 1.] [1. 1.]] """ - shape = _check_shape(shape) - dtype = _check_dtype(dtype) - if _is_shape_empty(shape): + shape = _check_shape(shape) + dtype = _check_dtype(dtype) #check the shape and dtype + if _is_shape_empty(shape): #if the shape is null, fufill the array with the type of dtype and the shape of it, create the array full of "1.0" return full(shape, 1.0, dtype) - output = F.fill(dtype, shape, 1) + output = F.fill(dtype, shape, 1) #or using the functional->fill get a tensor full of "1" return output @@ -325,8 +325,8 @@ def zeros(shape, dtype=mstype.float32): shape = _check_shape(shape) dtype = _check_dtype(dtype) if _is_shape_empty(shape): - return full(shape, 0.0, dtype) - output = F.fill(dtype, shape, 0) + return full(shape, 0.0, dtype) #the type of the tensor is float 32. + output = F.fill(dtype, shape, 0) #but the output type is the integer return output @@ -360,7 +360,7 @@ def full(shape, fill_value, dtype=None): [True True]] """ shape = _check_shape(shape) - if not isinstance(fill_value, ARRAY_TYPES): + if not isinstance(fill_value, ARRAY_TYPES): #check the fill_value, need ensure the _raise_type_error("fill value should be int, float, bool, list, tuple, Tensor, but got", fill_value) if dtype is not None: dtype = _check_dtype(dtype) @@ -376,7 +376,7 @@ def full(shape, fill_value, dtype=None): if isinstance(fill_value, (list, tuple)): fill_value = asarray_const(fill_value) return broadcast_to(fill_value, shape) - # if shape contains zero, use c.Tensor() + # if shape contains zero, use c.Tensor() #using the _convert function to get the 32 type return _convert_64_to_32(empty_compile(dtype, shape)) diff --git a/mindspore/python/mindspore/numpy/math_ops.py b/mindspore/python/mindspore/numpy/math_ops.py index 93fb20776d8..f2cac3aed16 100644 --- a/mindspore/python/mindspore/numpy/math_ops.py +++ b/mindspore/python/mindspore/numpy/math_ops.py @@ -99,13 +99,13 @@ def absolute(x, dtype=None): """ original_dtype = x.dtype allowed_types = None - if _get_device() == "Ascend": - allowed_types = (mstype.float16, mstype.float32) + if _get_device() == "Ascend": #if the device is Ascend + allowed_types = (mstype.float16, mstype.float32) # it can support the service of float64 and float32, can not support int32 type else: allowed_types = (mstype.int32, mstype.float16, mstype.float32, mstype.float64) if original_dtype not in allowed_types and dtype is None: - x = x.astype(mstype.float32) - return _apply_tensor_op(F.absolute, x, dtype=dtype).astype(original_dtype) + x = x.astype(mstype.float32) #Default is float32 + return _apply_tensor_op(F.absolute, x, dtype=dtype).astype(original_dtype) #return the result of astype function, which is the result after absolute operation return _apply_tensor_op(F.absolute, x, dtype=dtype) @@ -140,7 +140,7 @@ def count_nonzero(x, axis=None, keepdims=False): >>> print(output) 6 """ - if _is_shape_empty(x.shape): + if _is_shape_empty(x.shape): #check the x.shape return ZERO_TENSOR if axis is None: axis = () @@ -184,9 +184,9 @@ def clip(x, xmin, xmax, dtype=None): >>> print(output) [1 2 2 0 0 2 2 0] """ - if xmin is None and xmax is None: + if xmin is None and xmax is None: #at least having one limitation _raise_value_error("One of max or min must be given.") - if xmin is not None: + if xmin is not None: #using the max or min function to get the lmitation x = maximum(x, xmin, dtype=dtype) if xmax is not None: x = minimum(x, xmax, dtype=dtype) @@ -222,7 +222,7 @@ def deg2rad(x, dtype=None): _check_input_tensor(x) def convert(a): - return a * pi / 180.0 + return a * pi / 180.0 #the type is float return _apply_tensor_op(convert, x, dtype=dtype) @@ -252,7 +252,7 @@ def rad2deg(x, dtype=None): _check_input_tensor(x) def convert(a): - return a * 180.0 / pi + return a * 180.0 / pi #the type is float return _apply_tensor_op(convert, x, dtype=dtype) @@ -289,7 +289,7 @@ def add(x1, x2, dtype=None): """ # broadcast is not fully supported in tensor_add on CPU, # so we use tensor_sub as a substitute solution - if _get_device() == 'CPU': + if _get_device() == 'CPU': #ADD function call the substract if the device is CPU return subtract(x1, F.neg_tensor(_to_tensor(x2)), dtype=dtype) return _apply_tensor_op(F.tensor_add, x1, x2, dtype=dtype) @@ -359,12 +359,12 @@ def multiply(x1, x2, dtype=None): [3 8] [3 8]] """ - if _get_device() == 'CPU': + if _get_device() == 'CPU': #if the device is CPU, need to check the tensor _check_input_tensor(x1, x2) # broadcast is not fully supported on CPU backend, # and explicit broadcasting is performed - shape_out = _infer_out_shape(F.shape(x1), F.shape(x2)) - x1 = _broadcast_to_shape(x1, shape_out) + shape_out = _infer_out_shape(F.shape(x1), F.shape(x2)) #show the parameter of the shape + x1 = _broadcast_to_shape(x1, shape_out) #broadcasting the x1,x2 x2 = _broadcast_to_shape(x2, shape_out) return _apply_tensor_op(F.tensor_mul, x1, x2, dtype=dtype) @@ -404,7 +404,7 @@ def divide(x1, x2, dtype=None): """ x1, x2 = _to_tensor(x1, x2) if not _check_is_float(F.dtype(x1)) and not _check_is_float(F.dtype(x2)): - x1 = F.cast(x1, mstype.float32) + x1 = F.cast(x1, mstype.float32) #need to cast it to the float32 x2 = F.cast(x2, mstype.float32) return _apply_tensor_op(F.tensor_div, x1, x2, dtype=dtype) @@ -442,7 +442,7 @@ def true_divide(x1, x2, dtype=None): [0.33333334 0.5 ] [0.33333334 0.5 ]] """ - return divide(x1, x2, dtype=dtype) + return divide(x1, x2, dtype=dtype) #noe need to do that because its integer division def power(x1, x2, dtype=None): @@ -522,7 +522,7 @@ def float_power(x1, x2, dtype=None): if not _check_same_type(F.dtype(x1), mstype.float32): x1 = F.cast(x1, mstype.float32) if not _check_same_type(F.dtype(x2), mstype.float32): - x2 = F.cast(x2, mstype.float32) + x2 = F.cast(x2, mstype.float32) #convert the value to the float32 return _apply_tensor_op(F.tensor_pow, x1, x2, dtype=dtype) @@ -562,7 +562,7 @@ def minimum(x1, x2, dtype=None): [[1 2] [1 2]] """ - if isinstance(x1, (int, float, bool, list, tuple)): + if isinstance(x1, (int, float, bool, list, tuple)): x1 = asarray_const(x1) elif not isinstance(x1, Tensor): _raise_type_error("Input x1 is expected to be array_like") @@ -676,15 +676,15 @@ def inner(a, b): [[3. 3. 3. 3. 3. 3. 3.] [3. 3. 3. 3. 3. 3. 3.]]] """ - if F.rank(a) == 0 or F.rank(b) == 0: + if F.rank(a) == 0 or F.rank(b) == 0: #if the rank of a or the rank of b is zero, then the inner of them is the result of multiply,For one-dimensional tensors, they are all the same return F.tensor_mul(a, b) _check_shape_aligned(F.shape(a), F.shape(b)) - aligned_shape_a = (F.shape_mul(F.shape(a)[:-1]), F.shape(a)[-1]) + aligned_shape_a = (F.shape_mul(F.shape(a)[:-1]), F.shape(a)[-1]) #using the shape of a[:-1] and the last rank of a to get the aligened_shape aligned_shape_b = (F.shape_mul(F.shape(b)[:-1]), F.shape(a)[-1]) - a_aligned = F.reshape(a, aligned_shape_a) + a_aligned = F.reshape(a, aligned_shape_a) #using the aligened a to reshape the a b_aligned = F.reshape(b, aligned_shape_b) - + #the qperation upon is to reshape the a and b to adjust to the _mat_mul_t function. res = _matmul_t(a_aligned, b_aligned) res = F.reshape(res, F.shape(a)[:-1] + F.shape(b)[:-1]) return res @@ -736,21 +736,21 @@ def dot(a, b): [[[105. 105. 105. 105.] [105. 105. 105. 105.]]] """ - ndim_a, ndim_b = F.rank(a), F.rank(b) + ndim_a, ndim_b = F.rank(a), F.rank(b) #get the rank of a and b if ndim_a == 0 or ndim_b == 0: - return F.tensor_mul(a, b) - if ndim_a > 0 and ndim_b >= 2: - perm = F.make_range(ndim_b) + return F.tensor_mul(a, b) # if one of them is zero, then the dot operation is multiple + if ndim_a > 0 and ndim_b >= 2: #both of them is not zero, and one of them have at least two dims. + perm = F.make_range(ndim_b) perm = perm[:-2] + (perm[-1],) + (perm[-2],) - b = F.transpose(b, perm) + b = F.transpose(b, perm) #using the result of make_range to transpose b. if F.shape(a)[-1] != F.shape(b)[-1]: _raise_value_error('shapes are not aligned') - a_aligned = F.reshape(a, (-1, F.shape(a)[-1])) + a_aligned = F.reshape(a, (-1, F.shape(a)[-1])) #get the aligened model to adjust to the matmul_t. b_aligned = F.reshape(b, (-1, F.shape(b)[-1])) - res = _matmul_t(a_aligned, b_aligned) - res = F.reshape(res, F.shape(a)[:-1] + F.shape(b)[:-1]) + res = _matmul_t(a_aligned, b_aligned) #get the res + res = F.reshape(res, F.shape(a)[:-1] + F.shape(b)[:-1]) #beed to reshape the res return res @@ -804,14 +804,14 @@ def outer(a, b): [6. 6. 6. 6.] [6. 6. 6. 6.]] """ - _check_input_tensor(a, b) + _check_input_tensor(a, b) #need to get the one dim array if F.rank(a) != 1: - a = ravel(a) + a = ravel(a) if F.rank(b) != 1: b = ravel(b) - a = F.reshape(a, (F.shape(a)[0], 1)) - b = _expand(b, 2) - return _matmul(a, b) + a = F.reshape(a, (F.shape(a)[0], 1)) #reshape a to one dim + b = _expand(b, 2) # the dim od b is 2 + return _matmul(a, b) #after these problem, return the _matmul result. def tensordot(a, b, axes=2): @@ -870,7 +870,7 @@ def tensordot(a, b, axes=2): >>> print(output.shape) (5, 2) """ - if F.rank(a)*F.rank(b) == 0 and axes == 0: + if F.rank(a)*F.rank(b) == 0 and axes == 0: #for the tensor if the name is a return F.tensor_mul(a, b) return C.tensor_dot(a, b, axes) @@ -1030,29 +1030,29 @@ def average(x, axis=None, weights=None, returned=False): """ _check_input_tensor(x) if axis is not None: - _check_axis_type(axis, True, True, False) - axis = _canonicalize_axis(axis, x.ndim) + _check_axis_type(axis, True, True, False) #ensure the type of anix is int, tuple and list + axis = _canonicalize_axis(axis, x.ndim) #using the x.ndim to canonical form - x_avg = full((), nan, F.dtype(x)) + x_avg = full((), nan, F.dtype(x)) #get a new tensor filled with nan. sum_of_weights = None if weights is None: - x_avg = mean(x, axis) + x_avg = mean(x, axis) sum_of_weights = compute_weights_for_mean(x, x_avg, axis) - else: + else: #if weights matrix is exist _check_input_tensor(weights) if x.shape == weights.shape: - x_avg, sum_of_weights = comput_avg(x, axis, weights) - elif F.rank(weights) == 1: + x_avg, sum_of_weights = comput_avg(x, axis, weights) #if the shape is right ,then we can use the comput_avg to get the + elif F.rank(weights) == 1: #if the rank of it is 1, then it must be int if not isinstance(axis, int): _raise_type_error("Axis must be specified when shapes of x and weights differ.") perm = _expanded_shape(x.ndim, weights.shape[0], axis) - weights = weights.reshape(perm) + weights = weights.reshape(perm) #get the perm to reshape the weights x_avg, sum_of_weights = comput_avg(x, axis, weights) else: _raise_type_error("Weights should be None, 1-D or the same shape as input x.") - if returned: + if returned: #only the weights which ensure the demand will be returned if x_avg.shape != sum_of_weights.shape: sum_of_weights = _broadcast_to(sum_of_weights, sum_of_weights.shape, x_avg.shape, x_avg.ndim) return (x_avg, sum_of_weights) @@ -1062,12 +1062,12 @@ def average(x, axis=None, weights=None, returned=False): def compute_weights_for_mean(x, x_avg, axis): """Computes weights for np.average.""" if axis is None: - sum_of_weights = full((), x.size, F.dtype(x)) + sum_of_weights = full((), x.size, F.dtype(x)) #no axis means weights is none so the sum_of_weights is a new tensor else: - fill_value = 1 + fill_value = 1 #else fill the tensor with 1 if isinstance(axis, int) or (isinstance(axis, tuple) and F.tuple_len(axis) == 1): fill_value = x.shape[axis] if isinstance(axis, int) else x.shape[axis[0]] - elif axis is None: + elif axis is None: #update the tensor to get the tensor for sh in x.shape: fill_value *= sh else: @@ -1127,7 +1127,7 @@ def matmul(x1, x2, dtype=None): [ 550. 620. 690. 760. 830.] [ 670. 756. 842. 928. 1014.]]] """ - return C.matmul(x1, x2, dtype=dtype) + return C.matmul(x1, x2, dtype=dtype) # this is from the composite def square(x, dtype=None): diff --git a/mindspore/python/mindspore/ops/composite/base.py b/mindspore/python/mindspore/ops/composite/base.py index 495323ede99..516046ae6d6 100644 --- a/mindspore/python/mindspore/ops/composite/base.py +++ b/mindspore/python/mindspore/ops/composite/base.py @@ -474,6 +474,7 @@ class _Grad(GradOperation_): @ms_function def after_grad(*args): return grad_(fn)(*args) + # If calling Grad in PYNATIVE_MODE, do grad in PYNATIVE_MODE elif self.pynative_: _pynative_executor.set_grad_position(grad_, grad_position) @@ -482,10 +483,12 @@ class _Grad(GradOperation_): if _pynative_executor.check_graph(fn, *args, **kwargs): print("Another grad step is running") self._pynative_forward_run(grad_, args, kwargs, fn) + # call _pynative_executor to run graph and calculate gradients _pynative_executor.grad(grad_, fn, weights, grad_position, *args, **kwargs) out = _pynative_executor(fn, *args, **kwargs) _pynative_executor.clear_grad(fn, *args, **kwargs) return out + # If calling Grad in pure PYNATIVE_MODE, do grad in PYNATIVE_MODE else: grad_.pynative_ = True # after_grad of this branch can't use @ms_function, just directly call grad_ @@ -514,16 +517,18 @@ class _Grad(GradOperation_): new_kwargs.pop('sens') else: args = args[:-1] + # fn is a function if isinstance(fn, FunctionType): if not _pynative_executor.check_run(grad, fn, *args, **new_kwargs): _pynative_executor.set_grad_flag(True) _pynative_executor.new_graph(fn, *args, **new_kwargs) outputs = fn(*args, **new_kwargs) _pynative_executor.end_graph(fn, outputs, *args, **new_kwargs) + # fn is a Cell else: # Check if fn has run already. if not _pynative_executor.check_run(grad, fn, *args, **new_kwargs): - fn.set_grad() + fn.set_grad() # set grad flag fn(*args, **new_kwargs) fn.set_grad(False) diff --git a/mindspore/python/mindspore/ops/functional.py b/mindspore/python/mindspore/ops/functional.py index d5fa7513dbe..57a66c05c62 100644 --- a/mindspore/python/mindspore/ops/functional.py +++ b/mindspore/python/mindspore/ops/functional.py @@ -289,6 +289,8 @@ def grad(fn, grad_position=0, sens_param=False): [[-2.00000000e+00, 6.00000000e+00], [-3.00000000e+00, 8.00000000e+00]])) """ + + # Depending on the parameters, call the module callable class in different ways grad_position = _convert_grad_position_type(grad_position) if sens_param: return grad_by_position_with_sens(fn, None, grad_position)