From 59efbb88ec48bcf79d59dbb443fee705a5a99763 Mon Sep 17 00:00:00 2001 From: shenwei41 Date: Mon, 9 Aug 2021 10:07:04 +0800 Subject: [PATCH] fix waring --- .../dataset/callback/py_ds_callback.cc | 12 +- .../dataset/callback/py_ds_callback.h | 12 +- .../ccsrc/minddata/dataset/core/tensor.cc | 4 +- .../ccsrc/minddata/dataset/core/tensor.h | 2 +- .../dataset/engine/cache/cache_grpc_client.cc | 2 +- .../minddata/dataset/engine/data_schema.cc | 2 +- .../engine/datasetops/source/album_op.cc | 18 +-- .../engine/datasetops/source/album_op.h | 18 +-- .../engine/datasetops/source/cifar_op.cc | 2 +- .../engine/datasetops/source/mindrecord_op.cc | 2 +- .../engine/ir/cache/dataset_cache_impl.cc | 16 +- .../engine/ir/datasetops/epoch_ctrl_node.cc | 4 +- .../engine/ir/datasetops/epoch_ctrl_node.h | 4 +- .../engine/ir/datasetops/source/album_node.cc | 2 +- .../engine/ir/datasetops/source/album_node.h | 2 +- .../engine/ir/datasetops/source/mnist_node.cc | 2 +- .../engine/ir/datasetops/source/mnist_node.h | 2 +- .../ir/datasetops/source/random_node.cc | 2 +- .../engine/ir/datasetops/source/random_node.h | 2 +- .../ir/datasetops/source/tf_record_node.cc | 4 +- .../ir/datasetops/source/tf_record_node.h | 4 +- .../dataset/engine/opt/post/repeat_pass.cc | 2 +- .../dataset/engine/opt/post/repeat_pass.h | 2 +- .../minddata/dataset/kernels/image/crop_op.cc | 12 +- .../dataset/kernels/image/hwc_to_chw_op.cc | 8 +- .../kernels/image/lite_cv/image_process.cc | 6 +- .../kernels/image/lite_cv/warp_affine.cc | 4 +- .../image/random_crop_and_resize_op.cc | 12 +- .../dataset/kernels/image/random_crop_op.cc | 12 +- .../dataset/kernels/image/resize_op.cc | 12 +- .../dataset/kernels/image/uniform_aug_op.h | 2 +- .../mindrecord/io/shard_index_generator.cc | 2 +- .../minddata/mindrecord/io/shard_reader.cc | 4 +- mindspore/dataset/core/validator_helpers.py | 2 - mindspore/dataset/engine/validators.py | 1 - mindspore/lite/minddata/wrapper/MDToDApi.cc | 4 - .../lite/minddata/wrapper/album_op_android.cc | 18 +-- .../lite/minddata/wrapper/album_op_android.h | 18 +-- tests/ut/python/dataset/test_schema.py | 2 +- third_party/patch/sqlite/sqlite.patch001 | 138 +++++++++++++----- 40 files changed, 239 insertions(+), 140 deletions(-) diff --git a/mindspore/ccsrc/minddata/dataset/callback/py_ds_callback.cc b/mindspore/ccsrc/minddata/dataset/callback/py_ds_callback.cc index 85e1177906a..8838c06efd2 100644 --- a/mindspore/ccsrc/minddata/dataset/callback/py_ds_callback.cc +++ b/mindspore/ccsrc/minddata/dataset/callback/py_ds_callback.cc @@ -61,27 +61,27 @@ Status PyDSCallback::ExecutePyfunc(py::function f, const CallbackParam &cb_param } return Status::OK(); } -void PyDSCallback::setBegin(py::function f) { +void PyDSCallback::setBegin(const py::function &f) { begin_func_ = f; begin_needed_ = true; } -void PyDSCallback::setEnd(py::function f) { +void PyDSCallback::setEnd(const py::function &f) { end_func_ = f; end_needed_ = true; } -void PyDSCallback::setEpochBegin(py::function f) { +void PyDSCallback::setEpochBegin(const py::function &f) { epoch_begin_func_ = f; epoch_begin_needed_ = true; } -void PyDSCallback::setEpochEnd(py::function f) { +void PyDSCallback::setEpochEnd(const py::function &f) { epoch_end_func_ = f; epoch_end_needed_ = true; } -void PyDSCallback::setStepBegin(py::function f) { +void PyDSCallback::setStepBegin(const py::function &f) { step_begin_func_ = f; step_begin_needed_ = true; } -void PyDSCallback::setStepEnd(py::function f) { +void PyDSCallback::setStepEnd(const py::function &f) { step_end_func_ = f; step_end_needed_ = true; } diff --git a/mindspore/ccsrc/minddata/dataset/callback/py_ds_callback.h b/mindspore/ccsrc/minddata/dataset/callback/py_ds_callback.h index dcc57415014..d3782d51542 100644 --- a/mindspore/ccsrc/minddata/dataset/callback/py_ds_callback.h +++ b/mindspore/ccsrc/minddata/dataset/callback/py_ds_callback.h @@ -44,12 +44,12 @@ class PyDSCallback : public DSCallback { ~PyDSCallback() = default; - void setBegin(py::function f); - void setEnd(py::function f); - void setEpochBegin(py::function f); - void setEpochEnd(py::function f); - void setStepBegin(py::function f); - void setStepEnd(py::function f); + void setBegin(const py::function &f); + void setEnd(const py::function &f); + void setEpochBegin(const py::function &f); + void setEpochEnd(const py::function &f); + void setStepBegin(const py::function &f); + void setStepEnd(const py::function &f); /// \brief actual callback function for begin, needs to be overridden in the derived class /// \param cb_param, callback parameter passed in from DatasetOp when calling the callback diff --git a/mindspore/ccsrc/minddata/dataset/core/tensor.cc b/mindspore/ccsrc/minddata/dataset/core/tensor.cc index b7dda10f75d..315ce87ed84 100644 --- a/mindspore/ccsrc/minddata/dataset/core/tensor.cc +++ b/mindspore/ccsrc/minddata/dataset/core/tensor.cc @@ -508,7 +508,9 @@ Status Tensor::GetItemPtr(uchar **ptr, const std::vector &index, offset RETURN_IF_NOT_OK(shape_.ToFlatIndex(index, &flat_idx)); offset_t length_temp = 0; RETURN_IF_NOT_OK(GetStringAt(flat_idx, ptr, &length_temp)); - if (length != nullptr) *length = length_temp; + if (length != nullptr) { + *length = length_temp; + } return Status::OK(); } else { std::string err = "data type not compatible"; diff --git a/mindspore/ccsrc/minddata/dataset/core/tensor.h b/mindspore/ccsrc/minddata/dataset/core/tensor.h index ddcad66cc93..41b1be8f1a9 100644 --- a/mindspore/ccsrc/minddata/dataset/core/tensor.h +++ b/mindspore/ccsrc/minddata/dataset/core/tensor.h @@ -68,7 +68,7 @@ class Tensor { Tensor(const Tensor &other) = delete; Tensor &operator=(const Tensor &other) = delete; - /// Create a tensor using shape and type. This constructor should not be used directly, use CreateFromTensor instead + /// Create a tensor using shape and type. This constructor should not be used directly, use CreateFromTensor instead. /// \note The shape and type information should be known and valid /// \note The constructor does not allocate data /// \param shape TensorShape diff --git a/mindspore/ccsrc/minddata/dataset/engine/cache/cache_grpc_client.cc b/mindspore/ccsrc/minddata/dataset/engine/cache/cache_grpc_client.cc index e1ce544f08c..428192bd785 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/cache/cache_grpc_client.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/cache/cache_grpc_client.cc @@ -73,7 +73,7 @@ Status CacheClientGreeter::DoServiceStop() { void *tag; while (cq_.Next(&tag, &success)) { auto r = reinterpret_cast(tag); - req_.erase(r->seqNo_); + (void)req_.erase(r->seqNo_); } } return Status::OK(); diff --git a/mindspore/ccsrc/minddata/dataset/engine/data_schema.cc b/mindspore/ccsrc/minddata/dataset/engine/data_schema.cc index 136c331db31..849b903cdb8 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/data_schema.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/data_schema.cc @@ -408,7 +408,7 @@ void DataSchema::Print(std::ostream &out) const { // Adds a column descriptor to the schema Status DataSchema::AddColumn(const ColDescriptor &cd) { // Sanity check there's not a duplicate name before adding the column - for (int32_t i = 0; i < col_descs_.size(); ++i) { + for (auto i = 0; i < col_descs_.size(); ++i) { if (col_descs_[i].name() == cd.name()) { std::ostringstream ss; ss << "column name '" << cd.name() << "' already exists in schema."; diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/album_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/album_op.cc index d11a5a7eb8f..3c8af4dd067 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/album_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/album_op.cc @@ -118,7 +118,7 @@ bool AlbumOp::CheckImageType(const std::string &file_name, bool *valid) { return true; } -Status AlbumOp::LoadImageTensor(const std::string &image_file_path, uint32_t col_num, TensorRow *row) { +Status AlbumOp::LoadImageTensor(const std::string &image_file_path, int32_t col_num, TensorRow *row) { TensorPtr image; std::ifstream fs; fs.open(image_file_path, std::ios::binary | std::ios::in); @@ -168,7 +168,7 @@ Status AlbumOp::LoadImageTensor(const std::string &image_file_path, uint32_t col return Status::OK(); } -Status AlbumOp::LoadStringArrayTensor(const nlohmann::json &json_obj, uint32_t col_num, TensorRow *row) { +Status AlbumOp::LoadStringArrayTensor(const nlohmann::json &json_obj, int32_t col_num, TensorRow *row) { std::vector data = json_obj; MS_LOG(INFO) << "String array label found: " << data << "."; @@ -178,7 +178,7 @@ Status AlbumOp::LoadStringArrayTensor(const nlohmann::json &json_obj, uint32_t c return Status::OK(); } -Status AlbumOp::LoadStringTensor(const nlohmann::json &json_obj, uint32_t col_num, TensorRow *row) { +Status AlbumOp::LoadStringTensor(const nlohmann::json &json_obj, int32_t col_num, TensorRow *row) { std::string data = json_obj; // now we iterate over the elements in json @@ -189,7 +189,7 @@ Status AlbumOp::LoadStringTensor(const nlohmann::json &json_obj, uint32_t col_nu return Status::OK(); } -Status AlbumOp::LoadIntArrayTensor(const nlohmann::json &json_obj, uint32_t col_num, TensorRow *row) { +Status AlbumOp::LoadIntArrayTensor(const nlohmann::json &json_obj, int32_t col_num, TensorRow *row) { TensorPtr label; // consider templating this function to handle all ints if (data_schema_->column(col_num).type() == DataType::DE_INT64) { @@ -218,7 +218,7 @@ Status AlbumOp::LoadIntArrayTensor(const nlohmann::json &json_obj, uint32_t col_ return Status::OK(); } -Status AlbumOp::LoadFloatArrayTensor(const nlohmann::json &json_obj, uint32_t col_num, TensorRow *row) { +Status AlbumOp::LoadFloatArrayTensor(const nlohmann::json &json_obj, int32_t col_num, TensorRow *row) { TensorPtr float_array; // consider templating this function to handle all ints if (data_schema_->column(col_num).type() == DataType::DE_FLOAT64) { @@ -247,7 +247,7 @@ Status AlbumOp::LoadFloatArrayTensor(const nlohmann::json &json_obj, uint32_t co return Status::OK(); } -Status AlbumOp::LoadIDTensor(const std::string &file, uint32_t col_num, TensorRow *row) { +Status AlbumOp::LoadIDTensor(const std::string &file, int32_t col_num, TensorRow *row) { if (data_schema_->column(col_num).type() == DataType::DE_STRING) { TensorPtr id; RETURN_IF_NOT_OK(Tensor::CreateScalar(file, &id)); @@ -263,7 +263,7 @@ Status AlbumOp::LoadIDTensor(const std::string &file, uint32_t col_num, TensorRo return Status::OK(); } -Status AlbumOp::LoadEmptyTensor(uint32_t col_num, TensorRow *row) { +Status AlbumOp::LoadEmptyTensor(int32_t col_num, TensorRow *row) { // hack to get the file name without extension, the 1 is to get rid of the backslash character TensorPtr empty_tensor; RETURN_IF_NOT_OK(Tensor::CreateEmpty(TensorShape({0}), data_schema_->column(col_num).type(), &empty_tensor)); @@ -275,7 +275,7 @@ Status AlbumOp::LoadEmptyTensor(uint32_t col_num, TensorRow *row) { // So we actually have to check what type we want to fill the tensor with. // Float64 doesn't work with reinterpret cast here. Otherwise we limit the float in the schema to // only be float32, seems like a weird limitation to impose -Status AlbumOp::LoadFloatTensor(const nlohmann::json &json_obj, uint32_t col_num, TensorRow *row) { +Status AlbumOp::LoadFloatTensor(const nlohmann::json &json_obj, int32_t col_num, TensorRow *row) { TensorPtr float_tensor; if (data_schema_->column(col_num).type() == DataType::DE_FLOAT64) { double data = json_obj; @@ -291,7 +291,7 @@ Status AlbumOp::LoadFloatTensor(const nlohmann::json &json_obj, uint32_t col_num } // Loads a tensor with int value, we have to cast the value to type specified in the schema. -Status AlbumOp::LoadIntTensor(const nlohmann::json &json_obj, uint32_t col_num, TensorRow *row) { +Status AlbumOp::LoadIntTensor(const nlohmann::json &json_obj, int32_t col_num, TensorRow *row) { TensorPtr int_tensor; if (data_schema_->column(col_num).type() == DataType::DE_INT64) { int64_t data = json_obj; diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/album_op.h b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/album_op.h index 8c8b3e9fd72..f069c7bdbcf 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/album_op.h +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/album_op.h @@ -88,62 +88,62 @@ class AlbumOp : public MappableLeafOp { /// \param[in] col_num Column num in schema /// \param[in, out] row Tensor row to push to /// \return Status The status code returned - Status LoadImageTensor(const std::string &image_file, uint32_t col_num, TensorRow *row); + Status LoadImageTensor(const std::string &image_file, int32_t col_num, TensorRow *row); /// \brief Load vector of ints to tensor, append tensor to tensor row /// \param[in] json_obj Json object containing multi-dimensional label /// \param[in] col_num Column num in schema /// \param[in, out] row Tensor row to push to /// \return Status The status code returned - Status LoadIntArrayTensor(const nlohmann::json &json_obj, uint32_t col_num, TensorRow *row); + Status LoadIntArrayTensor(const nlohmann::json &json_obj, int32_t col_num, TensorRow *row); /// \brief Load vector of floatss to tensor, append tensor to tensor row /// \param[in] json_obj Json object containing array data /// \param[in] col_num Column num in schema /// \param[in, out] row Tensor row to push to /// \return Status The status code returned - Status LoadFloatArrayTensor(const nlohmann::json &json_obj, uint32_t col_num, TensorRow *row); + Status LoadFloatArrayTensor(const nlohmann::json &json_obj, int32_t col_num, TensorRow *row); /// \brief Load string array into a tensor, append tensor to tensor row /// \param[in] json_obj Json object containing string tensor /// \param[in] col_num Column num in schema /// \param[in, out] row Tensor row to push to /// \return Status The status code returned - Status LoadStringArrayTensor(const nlohmann::json &json_obj, uint32_t col_num, TensorRow *row); + Status LoadStringArrayTensor(const nlohmann::json &json_obj, int32_t col_num, TensorRow *row); /// \brief Load string into a tensor, append tensor to tensor row /// \param[in] json_obj Json object containing string tensor /// \param[in] col_num Column num in schema /// \param[in, out] row Tensor row to push to /// \return Status The status code returned - Status LoadStringTensor(const nlohmann::json &json_obj, uint32_t col_num, TensorRow *row); + Status LoadStringTensor(const nlohmann::json &json_obj, int32_t col_num, TensorRow *row); /// \brief Load float value to tensor row /// \param[in] json_obj Json object containing float /// \param[in] col_num Column num in schema /// \param[in, out] row Tensor row to push to /// \return Status The status code returned - Status LoadFloatTensor(const nlohmann::json &json_obj, uint32_t col_num, TensorRow *row); + Status LoadFloatTensor(const nlohmann::json &json_obj, int32_t col_num, TensorRow *row); /// \brief Load int value to tensor row /// \param[in] json_obj Json object containing int /// \param[in] col_num Column num in schema /// \param[in, out] row Tensor row to push to /// \return Status The status code returned - Status LoadIntTensor(const nlohmann::json &json_obj, uint32_t col_num, TensorRow *row); + Status LoadIntTensor(const nlohmann::json &json_obj, int32_t col_num, TensorRow *row); /// \brief Load empty tensor to tensor row /// \param[in] col_num Column num in schema /// \param[in, out] row Tensor row to push to /// \return Status The status code returned - Status LoadEmptyTensor(uint32_t col_num, TensorRow *row); + Status LoadEmptyTensor(int32_t col_num, TensorRow *row); /// \brief Load id from file name to tensor row /// \param[in] file The file name to get ID from /// \param[in] col_num Column num in schema /// \param[in, out] row Tensor row to push to /// \return Status The status code returned - Status LoadIDTensor(const std::string &file, uint32_t col_num, TensorRow *row); + Status LoadIDTensor(const std::string &file, int32_t col_num, TensorRow *row); /// \brief Load a tensor row according to a json file /// \param[in] row_id_type row_id - id for this tensor row diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/cifar_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/cifar_op.cc index 38dd454328e..6b865917ed4 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/cifar_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/cifar_op.cc @@ -368,7 +368,7 @@ Status CifarOp::CountTotalRows(const std::string &dir, const std::string &usage, Status CifarOp::ComputeColMap() { // set the column name map (base class field) if (column_name_id_map_.empty()) { - for (uint32_t i = 0; i < data_schema_->NumColumns(); ++i) { + for (int32_t i = 0; i < data_schema_->NumColumns(); ++i) { column_name_id_map_[data_schema_->column(i).name()] = i; } } else { diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/mindrecord_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/mindrecord_op.cc index 48b8597be9b..91d7c14566c 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/mindrecord_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/mindrecord_op.cc @@ -223,7 +223,7 @@ Status MindRecordOp::GetRowFromReader(TensorRow *fetched_row, uint64_t row_id, i Status MindRecordOp::LoadTensorRow(TensorRow *tensor_row, const std::vector &columns_blob, const mindrecord::json &columns_json, const mindrecord::TaskType task_type) { - for (uint32_t i_col = 0; i_col < columns_to_load_.size(); i_col++) { + for (int32_t i_col = 0; i_col < columns_to_load_.size(); i_col++) { auto column_name = columns_to_load_[i_col]; // Initialize column parameters diff --git a/mindspore/ccsrc/minddata/dataset/engine/ir/cache/dataset_cache_impl.cc b/mindspore/ccsrc/minddata/dataset/engine/ir/cache/dataset_cache_impl.cc index e818089636d..55e13659c67 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/ir/cache/dataset_cache_impl.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/ir/cache/dataset_cache_impl.cc @@ -31,10 +31,18 @@ Status DatasetCacheImpl::Build() { CacheClient::Builder builder; builder.SetSessionId(session_id_).SetCacheMemSz(cache_mem_sz_).SetSpill(spill_); - if (hostname_) builder.SetHostname(hostname_.value()); - if (port_) builder.SetPort(port_.value()); - if (num_connections_) builder.SetNumConnections(num_connections_.value()); - if (prefetch_sz_) builder.SetPrefetchSize(prefetch_sz_.value()); + if (hostname_) { + (void)builder.SetHostname(hostname_.value()); + } + if (port_) { + (void)builder.SetPort(port_.value()); + } + if (num_connections_) { + (void)builder.SetNumConnections(num_connections_.value()); + } + if (prefetch_sz_) { + (void)builder.SetPrefetchSize(prefetch_sz_.value()); + } return builder.Build(&cache_client_); } diff --git a/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/epoch_ctrl_node.cc b/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/epoch_ctrl_node.cc index 883f1673ac5..e41b475c694 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/epoch_ctrl_node.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/epoch_ctrl_node.cc @@ -71,13 +71,13 @@ Status EpochCtrlNode::ValidateParams() { } // Visitor accepting method for IRNodePass -Status EpochCtrlNode::Accept(IRNodePass *p, bool *const modified) { +Status EpochCtrlNode::Accept(IRNodePass *const p, bool *const modified) { // Downcast shared pointer then call visitor return p->Visit(shared_from_base(), modified); } // Visitor accepting method for IRNodePass -Status EpochCtrlNode::AcceptAfter(IRNodePass *p, bool *const modified) { +Status EpochCtrlNode::AcceptAfter(IRNodePass *const p, bool *const modified) { // Downcast shared pointer then call visitor return p->VisitAfter(shared_from_base(), modified); } diff --git a/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/epoch_ctrl_node.h b/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/epoch_ctrl_node.h index 709f92afa43..867a3010674 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/epoch_ctrl_node.h +++ b/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/epoch_ctrl_node.h @@ -67,13 +67,13 @@ class EpochCtrlNode : public RepeatNode { /// \param[in] p The node to visit /// \param[out] modified Indicator if the node was modified /// \return Status of the node visit - Status Accept(IRNodePass *p, bool *const modified) override; + Status Accept(IRNodePass *const p, bool *const modified) override; /// \brief Base-class override for accepting IRNodePass visitor /// \param[in] p The node to visit /// \param[out] modified Indicator if the node was modified /// \return Status of the node visit - Status AcceptAfter(IRNodePass *p, bool *const modified) override; + Status AcceptAfter(IRNodePass *const p, bool *const modified) override; }; } // namespace dataset diff --git a/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/album_node.cc b/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/album_node.cc index 2617c11fa03..54d191be18a 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/album_node.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/album_node.cc @@ -83,7 +83,7 @@ Status AlbumNode::Build(std::vector> *const node_ops) } // Get the shard id of node -Status AlbumNode::GetShardId(int32_t *shard_id) { +Status AlbumNode::GetShardId(int32_t *const shard_id) { *shard_id = sampler_->ShardId(); return Status::OK(); diff --git a/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/album_node.h b/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/album_node.h index dc19c2c1ca5..23cd4519995 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/album_node.h +++ b/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/album_node.h @@ -59,7 +59,7 @@ class AlbumNode : public MappableSourceNode { /// \brief Get the shard id of node /// \return Status Status::OK() if get shard id successfully - Status GetShardId(int32_t *shard_id) override; + Status GetShardId(int32_t *const shard_id) override; /// \brief Base-class override for GetDatasetSize /// \param[in] size_getter Shared pointer to DatasetSizeGetter diff --git a/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/mnist_node.cc b/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/mnist_node.cc index e3fa2eca3aa..b14a803ae12 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/mnist_node.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/mnist_node.cc @@ -70,7 +70,7 @@ Status MnistNode::Build(std::vector> *const node_ops) } // Get the shard id of node -Status MnistNode::GetShardId(int32_t *shard_id) { +Status MnistNode::GetShardId(int32_t *const shard_id) { *shard_id = sampler_->ShardId(); return Status::OK(); diff --git a/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/mnist_node.h b/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/mnist_node.h index 6c1c37a91d1..183ef75cea5 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/mnist_node.h +++ b/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/mnist_node.h @@ -58,7 +58,7 @@ class MnistNode : public MappableSourceNode { /// \brief Get the shard id of node /// \return Status Status::OK() if get shard id successfully - Status GetShardId(int32_t *shard_id) override; + Status GetShardId(int32_t *const shard_id) override; /// \brief Base-class override for GetDatasetSize /// \param[in] size_getter Shared pointer to DatasetSizeGetter diff --git a/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/random_node.cc b/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/random_node.cc index d92b9f5bd1a..e1183c49389 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/random_node.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/random_node.cc @@ -118,7 +118,7 @@ Status RandomNode::Build(std::vector> *const node_ops } // Get the shard id of node -Status RandomNode::GetShardId(int32_t *shard_id) { +Status RandomNode::GetShardId(int32_t *const shard_id) { // RandomDataset doesn't support multiple shards *shard_id = 0; return Status::OK(); diff --git a/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/random_node.h b/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/random_node.h index 0758fd2bd91..f099910e677 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/random_node.h +++ b/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/random_node.h @@ -80,7 +80,7 @@ class RandomNode : public NonMappableSourceNode { /// \brief Get the shard id of node /// \return Status Status::OK() if get shard id successfully - Status GetShardId(int32_t *shard_id) override; + Status GetShardId(int32_t *const shard_id) override; /// \brief Base-class override for GetDatasetSize /// \param[in] size_getter Shared pointer to DatasetSizeGetter diff --git a/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/tf_record_node.cc b/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/tf_record_node.cc index 3a5e3e97e9f..b9bf8fec4d9 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/tf_record_node.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/tf_record_node.cc @@ -156,7 +156,7 @@ Status TFRecordNode::Build(std::vector> *const node_o } // Get the shard id of node -Status TFRecordNode::GetShardId(int32_t *shard_id) { +Status TFRecordNode::GetShardId(int32_t *const shard_id) { *shard_id = shard_id_; return Status::OK(); @@ -259,7 +259,7 @@ Status TFRecordNode::Accept(IRNodePass *p, bool *const modified) { } // Visitor accepting method for IRNodePass -Status TFRecordNode::AcceptAfter(IRNodePass *p, bool *const modified) { +Status TFRecordNode::AcceptAfter(IRNodePass *const p, bool *const modified) { // Downcast shared pointer then call visitor return p->VisitAfter(shared_from_base(), modified); } diff --git a/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/tf_record_node.h b/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/tf_record_node.h index c56f205b580..9a9ccfc0266 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/tf_record_node.h +++ b/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/tf_record_node.h @@ -95,7 +95,7 @@ class TFRecordNode : public NonMappableSourceNode { /// \brief Get the shard id of node /// \return Status Status::OK() if get shard id successfully - Status GetShardId(int32_t *shard_id) override; + Status GetShardId(int32_t *const shard_id) override; /// \brief Base-class override for GetDatasetSize /// \param[in] size_getter Shared pointer to DatasetSizeGetter @@ -152,7 +152,7 @@ class TFRecordNode : public NonMappableSourceNode { /// \param[in] p The node to visit /// \param[out] modified Indicator if the node was modified /// \return Status of the node visit - Status AcceptAfter(IRNodePass *p, bool *const modified) override; + Status AcceptAfter(IRNodePass *const p, bool *const modified) override; private: std::vector dataset_files_; diff --git a/mindspore/ccsrc/minddata/dataset/engine/opt/post/repeat_pass.cc b/mindspore/ccsrc/minddata/dataset/engine/opt/post/repeat_pass.cc index 8eb9b5599fa..778c1262b5d 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/opt/post/repeat_pass.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/opt/post/repeat_pass.cc @@ -192,7 +192,7 @@ Status RepeatPass::VisitAfter(std::shared_ptr node, bool *const mo } // Adds an operator to the cached operator stack save area -void RepeatPass::AddToCachedNodeStack(std::shared_ptr node) { cached_node_stacks_.push(node); } +void RepeatPass::AddToCachedNodeStack(const std::shared_ptr &node) { cached_node_stacks_.push(node); } // Pops an operator from the cached operator stack save area std::shared_ptr RepeatPass::PopFromCachedNodeStack() { diff --git a/mindspore/ccsrc/minddata/dataset/engine/opt/post/repeat_pass.h b/mindspore/ccsrc/minddata/dataset/engine/opt/post/repeat_pass.h index 6c9f257bd02..708b04ba9c5 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/opt/post/repeat_pass.h +++ b/mindspore/ccsrc/minddata/dataset/engine/opt/post/repeat_pass.h @@ -112,7 +112,7 @@ class RepeatPass : public IRNodePass { /// \brief Adds an operator to the cached stack save area /// \param node - The dataset node to add to cached stack /// \return Status The status code returned - void AddToCachedNodeStack(std::shared_ptr node); + void AddToCachedNodeStack(const std::shared_ptr &node); /// \brief Pops an operator from the cached stack save area /// \return shared_ptr to the popped dataset node diff --git a/mindspore/ccsrc/minddata/dataset/kernels/image/crop_op.cc b/mindspore/ccsrc/minddata/dataset/kernels/image/crop_op.cc index 389452da4e3..46939c4aa32 100644 --- a/mindspore/ccsrc/minddata/dataset/kernels/image/crop_op.cc +++ b/mindspore/ccsrc/minddata/dataset/kernels/image/crop_op.cc @@ -41,9 +41,15 @@ Status CropOp::OutputShape(const std::vector &inputs, std::vector &inputs, std::vect outputs.clear(); TensorShape in = inputs[0]; TensorShape out = TensorShape{in[2], in[0], in[1]}; - if (inputs[0].Rank() == 3) outputs.emplace_back(out); - if (!outputs.empty()) return Status::OK(); + if (inputs[0].Rank() == 3) { + (void)outputs.emplace_back(out); + } + if (!outputs.empty()) { + return Status::OK(); + } return Status( StatusCode::kMDUnexpectedError, "HWC2CHW: invalid input shape, expected 3D input, but got input dimension is:" + std::to_string(inputs[0].Rank())); diff --git a/mindspore/ccsrc/minddata/dataset/kernels/image/lite_cv/image_process.cc b/mindspore/ccsrc/minddata/dataset/kernels/image/lite_cv/image_process.cc index d10828c579c..3e1c6f6fe49 100644 --- a/mindspore/ccsrc/minddata/dataset/kernels/image/lite_cv/image_process.cc +++ b/mindspore/ccsrc/minddata/dataset/kernels/image/lite_cv/image_process.cc @@ -1015,7 +1015,7 @@ std::vector> GetDefaultBoxes(BoxesConfig config) { } scales.push_back(1.0f); std::vector> default_boxes; - for (int i = 0; i < config.feature_size.size(); i++) { + for (auto i = 0; i < config.feature_size.size(); i++) { float sk1 = scales[i]; float sk2 = scales[i + 1]; float sk3 = sqrt(sk1 * sk2); @@ -1069,10 +1069,10 @@ void ConvertBoxes(std::vector> &boxes, const std::vector ApplyNms(const std::vector> &all_boxes, std::vector &all_scores, float thres, int max_boxes) { - int boxes_num = all_boxes.size(); + size_t boxes_num = all_boxes.size(); std::vector areas(boxes_num); std::vector order(boxes_num); - for (int i = 0; i < boxes_num; i++) { + for (auto i = 0; i < boxes_num; i++) { if (all_boxes[i].size() < 4) { return {}; } diff --git a/mindspore/ccsrc/minddata/dataset/kernels/image/lite_cv/warp_affine.cc b/mindspore/ccsrc/minddata/dataset/kernels/image/lite_cv/warp_affine.cc index 1099941bffb..2ec3fb0fed3 100644 --- a/mindspore/ccsrc/minddata/dataset/kernels/image/lite_cv/warp_affine.cc +++ b/mindspore/ccsrc/minddata/dataset/kernels/image/lite_cv/warp_affine.cc @@ -410,7 +410,7 @@ bool WarpAffineBilinear(const LiteMat &src, LiteMat &dst, const LiteMat &M, int int *a = &_a[0], *b = a + dst.width_; const int SCALE = 1 << 10; const int B_SIZE = 64; - int16_t WH[B_SIZE * B_SIZE * 2]; + int16_t *WH = new int16_t[B_SIZE * B_SIZE * 2]; int16_t A_Ptr[B_SIZE * B_SIZE]; int r_delta = SCALE / kTabSz / 2; int x, y, x1, y1; @@ -449,7 +449,7 @@ bool WarpAffineBilinear(const LiteMat &src, LiteMat &dst, const LiteMat &M, int Remap(src, lite_part, _HW, _matA, borderType, borderValue); } } - + delete[] WH; delete[] _a; return true; } diff --git a/mindspore/ccsrc/minddata/dataset/kernels/image/random_crop_and_resize_op.cc b/mindspore/ccsrc/minddata/dataset/kernels/image/random_crop_and_resize_op.cc index b8fc8ef866d..33d209f37a5 100644 --- a/mindspore/ccsrc/minddata/dataset/kernels/image/random_crop_and_resize_op.cc +++ b/mindspore/ccsrc/minddata/dataset/kernels/image/random_crop_and_resize_op.cc @@ -61,9 +61,15 @@ Status RandomCropAndResizeOp::OutputShape(const std::vector &inputs RETURN_IF_NOT_OK(TensorOp::OutputShape(inputs, outputs)); outputs.clear(); TensorShape out = TensorShape{target_height_, target_width_}; - if (inputs[0].Rank() == 2) outputs.emplace_back(out); - if (inputs[0].Rank() == 3) outputs.emplace_back(out.AppendDim(inputs[0][2])); - if (!outputs.empty()) return Status::OK(); + if (inputs[0].Rank() == 2) { + (void)outputs.emplace_back(out); + } + if (inputs[0].Rank() == 3) { + (void)outputs.emplace_back(out.AppendDim(inputs[0][2])); + } + if (!outputs.empty()) { + return Status::OK(); + } return Status(StatusCode::kMDUnexpectedError, "RandomCropAndResize: invalid input shape"); } Status RandomCropAndResizeOp::GetCropBox(int h_in, int w_in, int *x, int *y, int *crop_height, int *crop_width) { diff --git a/mindspore/ccsrc/minddata/dataset/kernels/image/random_crop_op.cc b/mindspore/ccsrc/minddata/dataset/kernels/image/random_crop_op.cc index e69fc2ab8b7..561e28b0262 100644 --- a/mindspore/ccsrc/minddata/dataset/kernels/image/random_crop_op.cc +++ b/mindspore/ccsrc/minddata/dataset/kernels/image/random_crop_op.cc @@ -143,9 +143,15 @@ Status RandomCropOp::OutputShape(const std::vector &inputs, std::ve RETURN_IF_NOT_OK(TensorOp::OutputShape(inputs, outputs)); outputs.clear(); TensorShape out = TensorShape{crop_height_, crop_width_}; - if (inputs[0].Rank() == 2) outputs.emplace_back(out); - if (inputs[0].Rank() == 3) outputs.emplace_back(out.AppendDim(inputs[0][2])); - if (!outputs.empty()) return Status::OK(); + if (inputs[0].Rank() == 2) { + (void)outputs.emplace_back(out); + } + if (inputs[0].Rank() == 3) { + (void)outputs.emplace_back(out.AppendDim(inputs[0][2])); + } + if (!outputs.empty()) { + return Status::OK(); + } return Status(StatusCode::kMDUnexpectedError, "RandomCrop: invalid input shape, expected 2D or 3D input, but got input dimension is:" + std::to_string(inputs[0].Rank())); diff --git a/mindspore/ccsrc/minddata/dataset/kernels/image/resize_op.cc b/mindspore/ccsrc/minddata/dataset/kernels/image/resize_op.cc index 62614b89c10..9e06072fc23 100644 --- a/mindspore/ccsrc/minddata/dataset/kernels/image/resize_op.cc +++ b/mindspore/ccsrc/minddata/dataset/kernels/image/resize_op.cc @@ -61,9 +61,15 @@ Status ResizeOp::OutputShape(const std::vector &inputs, std::vector outputW = size2_; } TensorShape out = TensorShape{outputH, outputW}; - if (inputs[0].Rank() == 2) outputs.emplace_back(out); - if (inputs[0].Rank() == 3) outputs.emplace_back(out.AppendDim(inputs[0][2])); - if (!outputs.empty()) return Status::OK(); + if (inputs[0].Rank() == 2) { + (void)outputs.emplace_back(out); + } + if (inputs[0].Rank() == 3) { + (void)outputs.emplace_back(out.AppendDim(inputs[0][2])); + } + if (!outputs.empty()) { + return Status::OK(); + } return Status(StatusCode::kMDUnexpectedError, "Resize: invalid input wrong shape."); } } // namespace dataset diff --git a/mindspore/ccsrc/minddata/dataset/kernels/image/uniform_aug_op.h b/mindspore/ccsrc/minddata/dataset/kernels/image/uniform_aug_op.h index 435876ad947..8fa83efa91c 100644 --- a/mindspore/ccsrc/minddata/dataset/kernels/image/uniform_aug_op.h +++ b/mindspore/ccsrc/minddata/dataset/kernels/image/uniform_aug_op.h @@ -49,8 +49,8 @@ class UniformAugOp : public TensorOp { std::string Name() const override { return kUniformAugOp; } private: - int32_t num_ops_; std::vector> tensor_op_list_; + int32_t num_ops_; std::mt19937 rnd_; }; } // namespace dataset diff --git a/mindspore/ccsrc/minddata/mindrecord/io/shard_index_generator.cc b/mindspore/ccsrc/minddata/mindrecord/io/shard_index_generator.cc index 21e223be24e..59a68116912 100644 --- a/mindspore/ccsrc/minddata/mindrecord/io/shard_index_generator.cc +++ b/mindspore/ccsrc/minddata/mindrecord/io/shard_index_generator.cc @@ -223,7 +223,7 @@ MSRStatus ShardIndexGenerator::CreateShardNameTable(sqlite3 *db, const std::stri sql = "INSERT INTO SHARD_NAME (NAME) VALUES (:SHARD_NAME);"; sqlite3_stmt *stmt = nullptr; if (sqlite3_prepare_v2(db, common::SafeCStr(sql), -1, &stmt, 0) != SQLITE_OK) { - if (stmt) { + if (stmt != nullptr) { (void)sqlite3_finalize(stmt); } MS_LOG(ERROR) << "SQL error: could not prepare statement, sql: " << sql; diff --git a/mindspore/ccsrc/minddata/mindrecord/io/shard_reader.cc b/mindspore/ccsrc/minddata/mindrecord/io/shard_reader.cc index aff17e3efc5..f182d503b1e 100644 --- a/mindspore/ccsrc/minddata/mindrecord/io/shard_reader.cc +++ b/mindspore/ccsrc/minddata/mindrecord/io/shard_reader.cc @@ -877,7 +877,9 @@ std::pair> ShardReader::GetLabels(int page_id, int sqlite3_free(errmsg); } std::vector ret; - for (unsigned int i = 0; i < labels_ptr->size(); ++i) ret.emplace_back(json{}); + for (unsigned int i = 0; i < labels_ptr->size(); ++i) { + (void)ret.emplace_back(json{}); + } for (unsigned int i = 0; i < labels_ptr->size(); ++i) { json construct_json; for (unsigned int j = 0; j < columns.size(); ++j) { diff --git a/mindspore/dataset/core/validator_helpers.py b/mindspore/dataset/core/validator_helpers.py index 4ba42a401b6..7ca1696e857 100644 --- a/mindspore/dataset/core/validator_helpers.py +++ b/mindspore/dataset/core/validator_helpers.py @@ -482,8 +482,6 @@ def check_filename(path): if filename.startswith(' ') or filename.endswith(' '): raise ValueError("filename should not start/end with space.") - return True - def check_dir(dataset_dir): """ diff --git a/mindspore/dataset/engine/validators.py b/mindspore/dataset/engine/validators.py index aae27db0ba5..b11a008b576 100644 --- a/mindspore/dataset/engine/validators.py +++ b/mindspore/dataset/engine/validators.py @@ -838,7 +838,6 @@ def check_schema(method): [schema_file], _ = parse_user_args(method, *args, **kwargs) if schema_file is not None: - type_check(schema_file, (str,), "schema_file") check_file(schema_file) return method(self, *args, **kwargs) diff --git a/mindspore/lite/minddata/wrapper/MDToDApi.cc b/mindspore/lite/minddata/wrapper/MDToDApi.cc index b05007dfd50..cca50a99a64 100644 --- a/mindspore/lite/minddata/wrapper/MDToDApi.cc +++ b/mindspore/lite/minddata/wrapper/MDToDApi.cc @@ -269,10 +269,6 @@ extern "C" int MDToDApi_GetNext(MDToDApi *pMDToDApi, MDToDResult_t *results) { MS_LOG(INFO) << "Start GetNext [1]" << pMDToDApi; // get next row for dataset std::unordered_map> row; - if (pMDToDApi->_iter == nullptr) { - MS_LOG(ERROR) << "GetNext called with no iteratoe. abort"; - return -1; - } // create Execute functions, this replaces Map in Pipeline bool ret = pMDToDApi->_iter->GetNextRow(&row); diff --git a/mindspore/lite/minddata/wrapper/album_op_android.cc b/mindspore/lite/minddata/wrapper/album_op_android.cc index 48d040a96b5..472ce0a1305 100644 --- a/mindspore/lite/minddata/wrapper/album_op_android.cc +++ b/mindspore/lite/minddata/wrapper/album_op_android.cc @@ -177,7 +177,7 @@ bool AlbumOp::IsReadColumn(const std::string &column_name) { return false; } -Status AlbumOp::LoadImageTensor(const std::string &image_file_path, uint32_t col_num, TensorPtr *tensor) { +Status AlbumOp::LoadImageTensor(const std::string &image_file_path, int32_t col_num, TensorPtr *tensor) { TensorPtr image; TensorPtr rotate_tensor; std::ifstream fs; @@ -257,7 +257,7 @@ int AlbumOp::GetOrientation(const std::string &folder_path) { return code; } -Status AlbumOp::LoadStringArrayTensor(const nlohmann::json &json_obj, uint32_t col_num, TensorPtr *tensor) { +Status AlbumOp::LoadStringArrayTensor(const nlohmann::json &json_obj, int32_t col_num, TensorPtr *tensor) { std::vector data = json_obj.get>(); MS_LOG(INFO) << "String array label found: " << data << "."; @@ -265,7 +265,7 @@ Status AlbumOp::LoadStringArrayTensor(const nlohmann::json &json_obj, uint32_t c return Status::OK(); } -Status AlbumOp::LoadStringTensor(const nlohmann::json &json_obj, uint32_t col_num, TensorPtr *tensor) { +Status AlbumOp::LoadStringTensor(const nlohmann::json &json_obj, int32_t col_num, TensorPtr *tensor) { std::string data = json_obj; // now we iterate over the elements in json @@ -275,7 +275,7 @@ Status AlbumOp::LoadStringTensor(const nlohmann::json &json_obj, uint32_t col_nu return Status::OK(); } -Status AlbumOp::LoadIntArrayTensor(const nlohmann::json &json_obj, uint32_t col_num, TensorPtr *tensor) { +Status AlbumOp::LoadIntArrayTensor(const nlohmann::json &json_obj, int32_t col_num, TensorPtr *tensor) { // consider templating this function to handle all ints if (data_schema_->column(col_num).type() == DataType::DE_INT64) { std::vector data; @@ -302,7 +302,7 @@ Status AlbumOp::LoadIntArrayTensor(const nlohmann::json &json_obj, uint32_t col_ return Status::OK(); } -Status AlbumOp::LoadFloatArrayTensor(const nlohmann::json &json_obj, uint32_t col_num, TensorPtr *tensor) { +Status AlbumOp::LoadFloatArrayTensor(const nlohmann::json &json_obj, int32_t col_num, TensorPtr *tensor) { // consider templating this function to handle all ints if (data_schema_->column(col_num).type() == DataType::DE_FLOAT64) { std::vector data; @@ -329,7 +329,7 @@ Status AlbumOp::LoadFloatArrayTensor(const nlohmann::json &json_obj, uint32_t co return Status::OK(); } -Status AlbumOp::LoadIDTensor(const std::string &file, uint32_t col_num, TensorPtr *tensor) { +Status AlbumOp::LoadIDTensor(const std::string &file, int32_t col_num, TensorPtr *tensor) { if (data_schema_->column(col_num).type() == DataType::DE_STRING) { RETURN_IF_NOT_OK(Tensor::CreateScalar(file, tensor)); return Status::OK(); @@ -341,7 +341,7 @@ Status AlbumOp::LoadIDTensor(const std::string &file, uint32_t col_num, TensorPt return Status::OK(); } -Status AlbumOp::LoadEmptyTensor(uint32_t col_num, TensorPtr *tensor) { +Status AlbumOp::LoadEmptyTensor(int32_t col_num, TensorPtr *tensor) { // hack to get the file name without extension, the 1 is to get rid of the backslash character RETURN_IF_NOT_OK(Tensor::CreateEmpty(TensorShape({0}), data_schema_->column(col_num).type(), tensor)); return Status::OK(); @@ -351,7 +351,7 @@ Status AlbumOp::LoadEmptyTensor(uint32_t col_num, TensorPtr *tensor) { // So we actually have to check what type we want to fill the tensor with. // Float64 doesn't work with reinterpret cast here. Otherwise we limit the float in the schema to // only be float32, seems like a weird limitation to impose -Status AlbumOp::LoadFloatTensor(const nlohmann::json &json_obj, uint32_t col_num, TensorPtr *tensor) { +Status AlbumOp::LoadFloatTensor(const nlohmann::json &json_obj, int32_t col_num, TensorPtr *tensor) { if (data_schema_->column(col_num).type() == DataType::DE_FLOAT64) { double data = json_obj; MS_LOG(INFO) << "double found: " << json_obj << "."; @@ -365,7 +365,7 @@ Status AlbumOp::LoadFloatTensor(const nlohmann::json &json_obj, uint32_t col_num } // Loads a tensor with int value, we have to cast the value to type specified in the schema. -Status AlbumOp::LoadIntTensor(const nlohmann::json &json_obj, uint32_t col_num, TensorPtr *tensor) { +Status AlbumOp::LoadIntTensor(const nlohmann::json &json_obj, int32_t col_num, TensorPtr *tensor) { if (data_schema_->column(col_num).type() == DataType::DE_INT64) { int64_t data = json_obj; MS_LOG(INFO) << "int64 found: " << json_obj << "."; diff --git a/mindspore/lite/minddata/wrapper/album_op_android.h b/mindspore/lite/minddata/wrapper/album_op_android.h index 10d74d073ca..226ba66c9a4 100644 --- a/mindspore/lite/minddata/wrapper/album_op_android.h +++ b/mindspore/lite/minddata/wrapper/album_op_android.h @@ -93,62 +93,62 @@ class AlbumOp { /// \param[in] col_num Column num in schema /// \param[in,out] Tensor to push to /// \return Status The error code returned - Status LoadImageTensor(const std::string &image_file, uint32_t col_num, TensorPtr *tensor); + Status LoadImageTensor(const std::string &image_file, int32_t col_num, TensorPtr *tensor); /// \brief Load vector of ints to tensor, append tensor to tensor /// \param[in] json_obj Json object containing multi-dimensional label /// \param[in] col_num Column num in schema /// \param[in,out] Tensor to push to /// \return Status The error code returned - Status LoadIntArrayTensor(const nlohmann::json &json_obj, uint32_t col_num, TensorPtr *tensor); + Status LoadIntArrayTensor(const nlohmann::json &json_obj, int32_t col_num, TensorPtr *tensor); /// \brief Load vector of floatss to tensor, append tensor to tensor /// \param[in] json_obj Json object containing array data /// \param[in] col_num Column num in schema /// \param[in,out] Tensor to push to /// \return Status The error code returned - Status LoadFloatArrayTensor(const nlohmann::json &json_obj, uint32_t col_num, TensorPtr *tensor); + Status LoadFloatArrayTensor(const nlohmann::json &json_obj, int32_t col_num, TensorPtr *tensor); /// \brief Load string array into a tensor, append tensor to tensor /// \param[in] json_obj Json object containing string tensor /// \param[in] col_num Column num in schema /// \param[in,out] Tensor to push to /// \return Status The error code returned - Status LoadStringArrayTensor(const nlohmann::json &json_obj, uint32_t col_num, TensorPtr *tensor); + Status LoadStringArrayTensor(const nlohmann::json &json_obj, int32_t col_num, TensorPtr *tensor); /// \brief Load string into a tensor, append tensor to tensor /// \param[in] json_obj Json object containing string tensor /// \param[in] col_num Column num in schema /// \param[in,out] Tensor to push to /// \return Status The error code returned - Status LoadStringTensor(const nlohmann::json &json_obj, uint32_t col_num, TensorPtr *tensor); + Status LoadStringTensor(const nlohmann::json &json_obj, int32_t col_num, TensorPtr *tensor); /// \brief Load float value to tensor /// \param[in] json_obj Json object containing float /// \param[in] col_num Column num in schema /// \param[in,out] Tensor to push to /// \return Status The error code returned - Status LoadFloatTensor(const nlohmann::json &json_obj, uint32_t col_num, TensorPtr *tensor); + Status LoadFloatTensor(const nlohmann::json &json_obj, int32_t col_num, TensorPtr *tensor); /// \brief Load int value to tensor /// \param[in] json_obj Json object containing int /// \param[in] col_num Column num in schema /// \param[in,out] Tensor to push to /// \return Status The error code returned - Status LoadIntTensor(const nlohmann::json &json_obj, uint32_t col_num, TensorPtr *tensor); + Status LoadIntTensor(const nlohmann::json &json_obj, int32_t col_num, TensorPtr *tensor); /// \brief Load empty tensor to tensor /// \param[in] col_num Column num in schema /// \param[in,out] Tensor to push to /// \return Status The error code returned - Status LoadEmptyTensor(uint32_t col_num, TensorPtr *tensor); + Status LoadEmptyTensor(int32_t col_num, TensorPtr *tensor); /// \brief Load id from file name to tensor /// \param[in] file The file name to get ID from /// \param[in] col_num Column num in schema /// \param[in,out] Tensor to push to /// \return Status The error code returned - Status LoadIDTensor(const std::string &file, uint32_t col_num, TensorPtr *tensor); + Status LoadIDTensor(const std::string &file, int32_t col_num, TensorPtr *tensor); /// \brief Load a tensor according to a json file /// \param[in] row_id_type row_id - id for this tensor row diff --git a/tests/ut/python/dataset/test_schema.py b/tests/ut/python/dataset/test_schema.py index f31400dffe5..84ff09f498e 100644 --- a/tests/ut/python/dataset/test_schema.py +++ b/tests/ut/python/dataset/test_schema.py @@ -48,7 +48,7 @@ def test_schema_exception(): with pytest.raises(TypeError) as info: ds.Schema(1) - assert "Argument schema_file with value 1 is not of type []" in str(info.value) + assert "path: 1 is not string" in str(info.value) with pytest.raises(RuntimeError) as info: schema = ds.Schema(SCHEMA_FILE) diff --git a/third_party/patch/sqlite/sqlite.patch001 b/third_party/patch/sqlite/sqlite.patch001 index d40825a1488..bd3210dbaf7 100644 --- a/third_party/patch/sqlite/sqlite.patch001 +++ b/third_party/patch/sqlite/sqlite.patch001 @@ -1,6 +1,6 @@ -diff -Npur sqlite-version-3.32.2/src/expr.c sqlite-version-3.32.2-patched/src/expr.c ---- sqlite-version-3.32.2/src/expr.c 2020-06-04 08:58:43.000000000 -0400 -+++ sqlite-version-3.32.2-patched/src/expr.c 2021-04-29 04:06:04.544208700 -0400 +diff -Npur sqlite-version-3.32.2-new/src/expr.c sqlite-version-3.32.2/src/expr.c +--- sqlite-version-3.32.2-new/src/expr.c 2020-06-04 08:58:43.000000000 -0400 ++++ sqlite-version-3.32.2/src/expr.c 2021-08-04 11:57:45.029230992 -0400 @@ -3813,6 +3813,7 @@ expr_code_doover: AggInfo *pAggInfo = pExpr->pAggInfo; struct AggInfo_col *pCol; @@ -32,9 +32,9 @@ diff -Npur sqlite-version-3.32.2/src/expr.c sqlite-version-3.32.2-patched/src/ex int i; struct SrcCount *p = pWalker->u.pSrcCount; SrcList *pSrc = p->pSrc; -diff -Npur sqlite-version-3.32.2/src/global.c sqlite-version-3.32.2-patched/src/global.c ---- sqlite-version-3.32.2/src/global.c 2020-06-04 08:58:43.000000000 -0400 -+++ sqlite-version-3.32.2-patched/src/global.c 2021-04-29 04:06:04.544208700 -0400 +diff -Npur sqlite-version-3.32.2-new/src/global.c sqlite-version-3.32.2/src/global.c +--- sqlite-version-3.32.2-new/src/global.c 2020-06-04 08:58:43.000000000 -0400 ++++ sqlite-version-3.32.2/src/global.c 2021-08-04 11:57:45.033230992 -0400 @@ -300,6 +300,11 @@ sqlite3_uint64 sqlite3NProfileCnt = 0; int sqlite3PendingByte = 0x40000000; #endif @@ -47,9 +47,9 @@ diff -Npur sqlite-version-3.32.2/src/global.c sqlite-version-3.32.2-patched/src/ #include "opcodes.h" /* ** Properties of opcodes. The OPFLG_INITIALIZER macro is -diff -Npur sqlite-version-3.32.2/src/resolve.c sqlite-version-3.32.2-patched/src/resolve.c ---- sqlite-version-3.32.2/src/resolve.c 2020-06-04 08:58:43.000000000 -0400 -+++ sqlite-version-3.32.2-patched/src/resolve.c 2021-04-29 04:06:04.545208700 -0400 +diff -Npur sqlite-version-3.32.2-new/src/resolve.c sqlite-version-3.32.2/src/resolve.c +--- sqlite-version-3.32.2-new/src/resolve.c 2020-06-04 08:58:43.000000000 -0400 ++++ sqlite-version-3.32.2/src/resolve.c 2021-08-04 11:57:45.033230992 -0400 @@ -1715,6 +1715,14 @@ static int resolveSelectStep(Walker *pWa return WRC_Abort; } @@ -65,9 +65,9 @@ diff -Npur sqlite-version-3.32.2/src/resolve.c sqlite-version-3.32.2-patched/src } #endif -diff -Npur sqlite-version-3.32.2/src/select.c sqlite-version-3.32.2-patched/src/select.c ---- sqlite-version-3.32.2/src/select.c 2020-06-04 08:58:43.000000000 -0400 -+++ sqlite-version-3.32.2-patched/src/select.c 2021-04-29 04:07:21.458212191 -0400 +diff -Npur sqlite-version-3.32.2-new/src/select.c sqlite-version-3.32.2/src/select.c +--- sqlite-version-3.32.2-new/src/select.c 2020-06-04 08:58:43.000000000 -0400 ++++ sqlite-version-3.32.2/src/select.c 2021-08-04 12:27:34.737267443 -0400 @@ -15,20 +15,6 @@ #include "sqliteInt.h" @@ -89,7 +89,27 @@ diff -Npur sqlite-version-3.32.2/src/select.c sqlite-version-3.32.2-patched/src/ ** An instance of the following object is used to record information about ** how to process the DISTINCT keyword, to simplify passing that information ** into the selectInnerLoop() routine. -@@ -4426,11 +4412,14 @@ static int pushDownWhereTerms( +@@ -2717,9 +2703,7 @@ static int multiSelect( + selectOpName(p->op))); + rc = sqlite3Select(pParse, p, &uniondest); + testcase( rc!=SQLITE_OK ); +- /* Query flattening in sqlite3Select() might refill p->pOrderBy. +- ** Be sure to delete p->pOrderBy, therefore, to avoid a memory leak. */ +- sqlite3ExprListDelete(db, p->pOrderBy); ++ assert( p->pOrderBy==0 ); + pDelete = p->pPrior; + p->pPrior = pPrior; + p->pOrderBy = 0; +@@ -4105,7 +4089,7 @@ static int flattenSubquery( + ** We look at every expression in the outer query and every place we see + ** "a" we substitute "x*3" and every place we see "b" we substitute "y+10". + */ +- if( pSub->pOrderBy ){ ++ if( pSub->pOrderBy && (pParent->selFlags & SF_NoopOrderBy)==0 ){ + /* At this point, any non-zero iOrderByCol values indicate that the + ** ORDER BY column expression is identical to the iOrderByCol'th + ** expression returned by SELECT statement pSub. Since these values +@@ -4426,11 +4410,14 @@ static int pushDownWhereTerms( ){ Expr *pNew; int nChng = 0; @@ -105,7 +125,7 @@ diff -Npur sqlite-version-3.32.2/src/select.c sqlite-version-3.32.2-patched/src/ #endif #ifdef SQLITE_DEBUG -@@ -5553,7 +5542,9 @@ static void explainSimpleCount( +@@ -5553,7 +5540,9 @@ static void explainSimpleCount( static int havingToWhereExprCb(Walker *pWalker, Expr *pExpr){ if( pExpr->op!=TK_AND ){ Select *pS = pWalker->u.pSelect; @@ -116,7 +136,7 @@ diff -Npur sqlite-version-3.32.2/src/select.c sqlite-version-3.32.2-patched/src/ sqlite3 *db = pWalker->pParse->db; Expr *pNew = sqlite3Expr(db, TK_INTEGER, "1"); if( pNew ){ -@@ -5766,6 +5757,9 @@ int sqlite3Select( +@@ -5766,6 +5755,9 @@ int sqlite3Select( } if( sqlite3AuthCheck(pParse, SQLITE_SELECT, 0, 0, 0) ) return 1; memset(&sAggInfo, 0, sizeof(sAggInfo)); @@ -126,7 +146,15 @@ diff -Npur sqlite-version-3.32.2/src/select.c sqlite-version-3.32.2-patched/src/ #if SELECTTRACE_ENABLED SELECTTRACE(1,pParse,p, ("begin processing:\n", pParse->addrExplain)); if( sqlite3SelectTrace & 0x100 ){ -@@ -5804,19 +5798,6 @@ int sqlite3Select( +@@ -5787,6 +5779,7 @@ int sqlite3Select( + sqlite3ExprListDelete(db, p->pOrderBy); + p->pOrderBy = 0; + p->selFlags &= ~SF_Distinct; ++ p->selFlags |= SF_NoopOrderBy; + } + sqlite3SelectPrep(pParse, p, 0); + if( pParse->nErr || db->mallocFailed ){ +@@ -5804,19 +5797,6 @@ int sqlite3Select( generateColumnNames(pParse, p); } @@ -146,7 +174,7 @@ diff -Npur sqlite-version-3.32.2/src/select.c sqlite-version-3.32.2-patched/src/ pTabList = p->pSrc; isAgg = (p->selFlags & SF_Aggregate)!=0; memset(&sSort, 0, sizeof(sSort)); -@@ -6144,7 +6125,7 @@ int sqlite3Select( +@@ -6144,7 +6124,7 @@ int sqlite3Select( if( (p->selFlags & (SF_Distinct|SF_Aggregate))==SF_Distinct && sqlite3ExprListCompare(sSort.pOrderBy, pEList, -1)==0 #ifndef SQLITE_OMIT_WINDOWFUNC @@ -155,7 +183,7 @@ diff -Npur sqlite-version-3.32.2/src/select.c sqlite-version-3.32.2-patched/src/ #endif ){ p->selFlags &= ~SF_Distinct; -@@ -6791,6 +6772,14 @@ int sqlite3Select( +@@ -6791,6 +6771,14 @@ int sqlite3Select( select_end: sqlite3ExprListDelete(db, pMinMaxOrderBy); sqlite3DbFree(db, sAggInfo.aCol); @@ -170,9 +198,9 @@ diff -Npur sqlite-version-3.32.2/src/select.c sqlite-version-3.32.2-patched/src/ sqlite3DbFree(db, sAggInfo.aFunc); #if SELECTTRACE_ENABLED SELECTTRACE(0x1,pParse,p,("end processing\n")); -diff -Npur sqlite-version-3.32.2/src/sqliteInt.h sqlite-version-3.32.2-patched/src/sqliteInt.h ---- sqlite-version-3.32.2/src/sqliteInt.h 2020-06-04 08:58:43.000000000 -0400 -+++ sqlite-version-3.32.2-patched/src/sqliteInt.h 2021-04-29 04:06:04.547208700 -0400 +diff -Npur sqlite-version-3.32.2-new/src/sqliteInt.h sqlite-version-3.32.2/src/sqliteInt.h +--- sqlite-version-3.32.2-new/src/sqliteInt.h 2020-06-04 08:58:43.000000000 -0400 ++++ sqlite-version-3.32.2/src/sqliteInt.h 2021-08-04 12:28:22.825268422 -0400 @@ -976,7 +976,12 @@ typedef INT16_TYPE LogEst; */ #if defined(SQLITE_ENABLE_SELECTTRACE) @@ -211,7 +239,15 @@ diff -Npur sqlite-version-3.32.2/src/sqliteInt.h sqlite-version-3.32.2-patched/s ** The datatype ynVar is a signed integer, either 16-bit or 32-bit. ** Usually it is 16-bits. But if SQLITE_MAX_VARIABLE_NUMBER is greater ** than 32767 we have to make it 32-bit. 16-bit is preferred because -@@ -4546,10 +4566,11 @@ extern const unsigned char sqlite3UpperT +@@ -3105,6 +3125,7 @@ struct Select { + #define SF_WhereBegin 0x0080000 /* Really a WhereBegin() call. Debug Only */ + #define SF_WinRewrite 0x0100000 /* Window function rewrite accomplished */ + #define SF_View 0x0200000 /* SELECT statement is a view */ ++#define SF_NoopOrderBy 0x0400000 /* ORDER BY is ignored for this query */ + + /* + ** The results of a SELECT can be distributed in several ways, as defined +@@ -4546,10 +4567,11 @@ extern const unsigned char sqlite3UpperT extern const unsigned char sqlite3CtypeMap[]; extern SQLITE_WSD struct Sqlite3Config sqlite3Config; extern FuncDefHash sqlite3BuiltinFunctions; @@ -224,9 +260,9 @@ diff -Npur sqlite-version-3.32.2/src/sqliteInt.h sqlite-version-3.32.2-patched/s #ifdef VDBE_PROFILE extern sqlite3_uint64 sqlite3NProfileCnt; #endif -diff -Npur sqlite-version-3.32.2/src/test1.c sqlite-version-3.32.2-patched/src/test1.c ---- sqlite-version-3.32.2/src/test1.c 2020-06-04 08:58:43.000000000 -0400 -+++ sqlite-version-3.32.2-patched/src/test1.c 2021-04-29 04:06:04.548208700 -0400 +diff -Npur sqlite-version-3.32.2-new/src/test1.c sqlite-version-3.32.2/src/test1.c +--- sqlite-version-3.32.2-new/src/test1.c 2020-06-04 08:58:43.000000000 -0400 ++++ sqlite-version-3.32.2/src/test1.c 2021-08-04 11:57:45.037230992 -0400 @@ -8164,7 +8164,7 @@ int Sqlitetest1_Init(Tcl_Interp *interp) #endif #endif @@ -236,9 +272,9 @@ diff -Npur sqlite-version-3.32.2/src/test1.c sqlite-version-3.32.2-patched/src/t #endif for(i=0; idb; Select *pSub = 0; /* The subquery */ -diff -Npur sqlite-version-3.32.2/test/having.test sqlite-version-3.32.2-patched/test/having.test ---- sqlite-version-3.32.2/test/having.test 2020-06-04 08:58:43.000000000 -0400 -+++ sqlite-version-3.32.2-patched/test/having.test 2021-04-29 04:08:11.785214475 -0400 +diff -Npur sqlite-version-3.32.2-new/test/having.test sqlite-version-3.32.2/test/having.test +--- sqlite-version-3.32.2-new/test/having.test 2020-06-04 08:58:43.000000000 -0400 ++++ sqlite-version-3.32.2/test/having.test 2021-08-04 11:57:45.041230992 -0400 @@ -154,5 +154,24 @@ do_execsql_test 4.3 { SELECT a, sum(b) FROM t3 WHERE nondeter(a) GROUP BY a } {1 4 2 2} - + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 5.0 { @@ -274,11 +310,41 @@ diff -Npur sqlite-version-3.32.2/test/having.test sqlite-version-3.32.2-patched/ + SELECT x FROM t2 WHERE a=2 GROUP BY y HAVING 0 + ) FROM t1; +} {b {}} - + finish_test -diff -Npur sqlite-version-3.32.2/test/window1.test sqlite-version-3.32.2-patched/test/window1.test ---- sqlite-version-3.32.2/test/window1.test 2020-06-04 08:58:43.000000000 -0400 -+++ sqlite-version-3.32.2-patched/test/window1.test 2021-04-29 04:06:04.549208700 -0400 +diff -Npur sqlite-version-3.32.2-new/test/selectA.test sqlite-version-3.32.2/test/selectA.test +--- sqlite-version-3.32.2-new/test/selectA.test 2020-06-04 08:58:43.000000000 -0400 ++++ sqlite-version-3.32.2/test/selectA.test 2021-08-04 12:29:43.021270055 -0400 +@@ -1446,5 +1446,26 @@ do_execsql_test 6.1 { + SELECT * FROM (SELECT a FROM t1 UNION SELECT b FROM t2) WHERE a=a; + } {12345} + ++# 2020-06-15 ticket 8f157e8010b22af0 ++# ++reset_db ++do_execsql_test 7.1 { ++ CREATE TABLE t1(c1); INSERT INTO t1 VALUES(12),(123),(1234),(NULL),('abc'); ++ CREATE TABLE t2(c2); INSERT INTO t2 VALUES(44),(55),(123); ++ CREATE TABLE t3(c3,c4); INSERT INTO t3 VALUES(66,1),(123,2),(77,3); ++ CREATE VIEW t4 AS SELECT c3 FROM t3; ++ CREATE VIEW t5 AS SELECT c3 FROM t3 ORDER BY c4; ++} ++do_execsql_test 7.2 { ++ SELECT * FROM t1, t2 WHERE c1=(SELECT 123 INTERSECT SELECT c2 FROM t4) AND c1=123; ++} {123 123} ++do_execsql_test 7.3 { ++ SELECT * FROM t1, t2 WHERE c1=(SELECT 123 INTERSECT SELECT c2 FROM t5) AND c1=123; ++} {123 123} ++do_execsql_test 7.4 { ++ CREATE TABLE a(b); ++ CREATE VIEW c(d) AS SELECT b FROM a ORDER BY b; ++ SELECT sum(d) OVER( PARTITION BY(SELECT 0 FROM c JOIN a WHERE b =(SELECT b INTERSECT SELECT d FROM c) AND b = 123)) FROM c; ++} {} + + finish_test +diff -Npur sqlite-version-3.32.2-new/test/window1.test sqlite-version-3.32.2/test/window1.test +--- sqlite-version-3.32.2-new/test/window1.test 2020-06-04 08:58:43.000000000 -0400 ++++ sqlite-version-3.32.2/test/window1.test 2021-08-04 11:57:45.041230992 -0400 @@ -1743,5 +1743,47 @@ do_execsql_test 53.0 { WHERE a.c); } {4 4 4 4}