From 7ebf5b6011e2c00fd5d37e88c2d875129c985341 Mon Sep 17 00:00:00 2001 From: jonyguo Date: Fri, 11 Jun 2021 11:20:48 +0800 Subject: [PATCH] fix: regular path --- .../ccsrc/minddata/dataset/api/execute.cc | 4 +- mindspore/ccsrc/minddata/dataset/api/text.cc | 9 ++++- .../dataset/engine/cache/CMakeLists.txt | 2 +- .../minddata/dataset/engine/cache/cache_hw.cc | 21 +++++++++- .../engine/datasetops/source/celeba_op.cc | 10 ++++- .../engine/datasetops/source/clue_op.cc | 17 ++++++++- .../engine/datasetops/source/coco_op.cc | 9 ++++- .../engine/datasetops/source/csv_op.cc | 35 +++++++++++++++-- .../engine/datasetops/source/manifest_op.cc | 17 ++++++++- .../engine/datasetops/source/text_file_op.cc | 17 ++++++++- .../engine/datasetops/source/tf_reader_op.cc | 33 ++++++++++++++-- .../engine/datasetops/source/voc_op.cc | 10 ++++- .../ir/datasetops/source/celeba_node.cc | 18 ++++++++- .../dataset/engine/perf/cpu_sampling.cc | 8 ++++ .../ccsrc/minddata/dataset/engine/serdes.cc | 14 ++++++- .../dataset/include/dataset/data_helper.h | 1 + .../dataset/text/sentence_piece_vocab.cc | 12 +++++- .../ccsrc/minddata/dataset/text/vocab.cc | 23 ++++++++--- .../mindrecord/io/shard_index_generator.cc | 18 ++++++++- .../minddata/mindrecord/io/shard_reader.cc | 38 ++++++++++++++++--- .../minddata/mindrecord/io/shard_writer.cc | 17 ++++++++- .../minddata/mindrecord/meta/shard_header.cc | 26 +++++++++++-- .../preprocess_imagenet_validate_dataset.py | 3 +- .../dataset/engine/serializer_deserializer.py | 3 +- mindspore/mindrecord/tools/cifar10.py | 6 ++- mindspore/mindrecord/tools/cifar100.py | 6 ++- mindspore/mindrecord/tools/imagenet_to_mr.py | 8 ++-- mindspore/mindrecord/tools/mnist_to_mr.py | 6 ++- 28 files changed, 336 insertions(+), 55 deletions(-) diff --git a/mindspore/ccsrc/minddata/dataset/api/execute.cc b/mindspore/ccsrc/minddata/dataset/api/execute.cc index d5a64aa9709..68e62d566d2 100644 --- a/mindspore/ccsrc/minddata/dataset/api/execute.cc +++ b/mindspore/ccsrc/minddata/dataset/api/execute.cc @@ -14,9 +14,11 @@ * limitations under the License. */ +#include "minddata/dataset/include/dataset/execute.h" + #include #include -#include "minddata/dataset/include/dataset/execute.h" + #include "minddata/dataset/core/de_tensor.h" #include "minddata/dataset/core/tensor_row.h" #include "minddata/dataset/core/tensor.h" diff --git a/mindspore/ccsrc/minddata/dataset/api/text.cc b/mindspore/ccsrc/minddata/dataset/api/text.cc index 52ec4b333f4..bee04fe44be 100644 --- a/mindspore/ccsrc/minddata/dataset/api/text.cc +++ b/mindspore/ccsrc/minddata/dataset/api/text.cc @@ -18,6 +18,7 @@ #include #include +#include "debug/common.h" #include "minddata/dataset/include/dataset/text.h" #include "minddata/dataset/core/type_id.h" #include "minddata/dataset/text/ir/kernels/text_ir.h" @@ -165,7 +166,13 @@ Status JiebaTokenizer::AddDictChar(const std::vector &file_path) { Status JiebaTokenizer::ParserFile(const std::string &file_path, std::vector> *const user_dict) { - std::ifstream ifs(file_path); + auto realpath = Common::GetRealPath(file_path); + if (!realpath.has_value()) { + MS_LOG(ERROR) << "Get real path failed, path=" << file_path; + RETURN_STATUS_SYNTAX_ERROR("Get real path failed, path=" + file_path); + } + + std::ifstream ifs(realpath.value()); if (!ifs) { std::string err_msg = "JiebaTokenizer : Fail to load dictionary from the input file, check the file path."; MS_LOG(ERROR) << err_msg; diff --git a/mindspore/ccsrc/minddata/dataset/engine/cache/CMakeLists.txt b/mindspore/ccsrc/minddata/dataset/engine/cache/CMakeLists.txt index d566ac44ee2..e291c7228f4 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/cache/CMakeLists.txt +++ b/mindspore/ccsrc/minddata/dataset/engine/cache/CMakeLists.txt @@ -108,7 +108,7 @@ if(ENABLE_CACHE) add_executable(cache_admin cache_admin.cc cache_admin_arg.cc) target_link_libraries(cache_admin _c_dataengine _c_mindrecord mindspore::protobuf ${PYTHON_LIBRARIES} pthread) - target_link_libraries(cache_admin mindspore mindspore_shared_lib) + target_link_libraries(cache_admin mindspore mindspore_core mindspore_shared_lib) if(USE_GLOG) target_link_libraries(cache_admin mindspore::glog) diff --git a/mindspore/ccsrc/minddata/dataset/engine/cache/cache_hw.cc b/mindspore/ccsrc/minddata/dataset/engine/cache/cache_hw.cc index d1d8c485f08..877ad27205c 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/cache/cache_hw.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/cache/cache_hw.cc @@ -14,6 +14,7 @@ * limitations under the License. */ #include "minddata/dataset/engine/cache/cache_hw.h" + #ifdef NUMA_ENABLED #include #endif @@ -24,7 +25,10 @@ #include #include #include + +#include "debug/common.h" #include "utils/log_adapter.h" + namespace mindspore { namespace dataset { CacheServerHW::CacheServerHW() { @@ -118,7 +122,14 @@ Status CacheServerHW::GetNumaNodeInfo() { auto node_dir = p.Basename(); numa_id_t numa_node = static_cast(strtol(node_dir.data() + strlen(kNodeName), nullptr, kDecimal)); Path f = p / kCpuList; - std::ifstream fs(f.toString()); + + auto realpath = Common::GetRealPath(f.toString()); + if (!realpath.has_value()) { + MS_LOG(ERROR) << "Get real path failed, path=" << f.toString(); + RETURN_STATUS_UNEXPECTED("Get real path failed, path=" + f.toString()); + } + + std::ifstream fs(realpath.value()); CHECK_FAIL_RETURN_UNEXPECTED(!fs.fail(), "Fail to open file: " + f.toString()); std::string cpu_string; cpu_set_t cpuset; @@ -235,7 +246,13 @@ bool CacheServerHW::numa_enabled() { } uint64_t CacheServerHW::GetAvailableMemory() { - std::ifstream mem_file(kMemInfoFileName); + auto realpath = Common::GetRealPath(kMemInfoFileName); + if (!realpath.has_value()) { + MS_LOG(ERROR) << "Get real path failed, path=" << kMemInfoFileName; + return 0; + } + + std::ifstream mem_file(realpath.value()); if (mem_file.fail()) { MS_LOG(WARNING) << "Fail to open file: " << kMemInfoFileName; return 0; diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/celeba_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/celeba_op.cc index 16cbddb4cda..f9c8be4423c 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/celeba_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/celeba_op.cc @@ -18,6 +18,7 @@ #include #include #include +#include "debug/common.h" #include "minddata/dataset/core/config_manager.h" #include "minddata/dataset/util/path.h" #include "minddata/dataset/engine/datasetops/source/sampler/sequential_sampler.h" @@ -69,7 +70,14 @@ Status CelebAOp::LaunchThreadsAndInitOp() { Status CelebAOp::ParseAttrFile() { TaskManager::FindMe()->Post(); Path folder_path(folder_path_); - std::ifstream attr_file((folder_path / "list_attr_celeba.txt").toString()); + + auto realpath = Common::GetRealPath((folder_path / "list_attr_celeba.txt").toString()); + if (!realpath.has_value()) { + MS_LOG(ERROR) << "Get real path failed, path=" << (folder_path / "list_attr_celeba.txt").toString(); + RETURN_STATUS_UNEXPECTED("Get real path failed, path=" + (folder_path / "list_attr_celeba.txt").toString()); + } + + std::ifstream attr_file(realpath.value()); if (!attr_file.is_open()) { std::string attr_file_name = (folder_path / "list_attr_celeba.txt").toString(); return Status(StatusCode::kMDFileNotExist, __LINE__, __FILE__, diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/clue_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/clue_op.cc index 77188d75c83..519fa655ed5 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/clue_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/clue_op.cc @@ -22,6 +22,7 @@ #include #include +#include "debug/common.h" #include "minddata/dataset/core/config_manager.h" #include "minddata/dataset/engine/jagged_connector.h" #include "minddata/dataset/engine/execution_tree.h" @@ -83,7 +84,13 @@ Status ClueOp::GetValue(const nlohmann::json &js, std::vector key_c } Status ClueOp::LoadFile(const std::string &file, int64_t start_offset, int64_t end_offset, int32_t worker_id) { - std::ifstream handle(file); + auto realpath = Common::GetRealPath(file); + if (!realpath.has_value()) { + MS_LOG(ERROR) << "Get real path failed, path=" << file; + RETURN_STATUS_UNEXPECTED("Get real path failed, path=" + file); + } + + std::ifstream handle(realpath.value()); if (!handle.is_open()) { RETURN_STATUS_UNEXPECTED("Invalid file, failed to open file: " + file); } @@ -223,7 +230,13 @@ Status ClueOp::CalculateNumRowsPerShard() { } int64_t CountTotalRowsPerFile(const std::string &file) { - std::ifstream handle(file); + auto realpath = Common::GetRealPath(file); + if (!realpath.has_value()) { + MS_LOG(ERROR) << "Get real path failed, path=" << file; + return 0; + } + + std::ifstream handle(realpath.value()); if (!handle.is_open()) { MS_LOG(ERROR) << "Invalid file, failed to open file: " << file; return 0; diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/coco_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/coco_op.cc index fc2d8cd708f..c7fc8de7408 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/coco_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/coco_op.cc @@ -17,6 +17,7 @@ #include #include +#include "debug/common.h" #include "utils/ms_utils.h" #include "minddata/dataset/core/config_manager.h" #include "minddata/dataset/core/tensor_shape.h" @@ -335,7 +336,13 @@ Status CocoOp::SearchNodeInJson(const nlohmann::json &input_tree, std::string no Status CocoOp::ParseAnnotationIds() { nlohmann::json js; try { - std::ifstream in(annotation_path_); + auto realpath = Common::GetRealPath(annotation_path_); + if (!realpath.has_value()) { + MS_LOG(ERROR) << "Get real path failed, path=" << annotation_path_; + RETURN_STATUS_UNEXPECTED("Get real path failed, path=" + annotation_path_); + } + + std::ifstream in(realpath.value()); in >> js; } catch (const std::exception &err) { RETURN_STATUS_UNEXPECTED("Invalid file, failed to open JSON file: " + annotation_path_ + "."); diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/csv_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/csv_op.cc index 7a657873100..c63ecc71b39 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/csv_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/csv_op.cc @@ -20,6 +20,7 @@ #include #include +#include "debug/common.h" #include "minddata/dataset/core/config_manager.h" #include "minddata/dataset/engine/jagged_connector.h" #include "minddata/dataset/engine/execution_tree.h" @@ -487,8 +488,15 @@ Status CsvOp::LoadFile(const std::string &file, int64_t start_offset, int64_t en RETURN_IF_NOT_OK(csv_parser.InitCsvParser()); csv_parser.SetStartOffset(start_offset); csv_parser.SetEndOffset(end_offset); + + auto realpath = Common::GetRealPath(file); + if (!realpath.has_value()) { + MS_LOG(ERROR) << "Get real path failed, path=" << file; + RETURN_STATUS_UNEXPECTED("Get real path failed, path=" + file); + } + std::ifstream ifs; - ifs.open(file, std::ifstream::in); + ifs.open(realpath.value(), std::ifstream::in); if (!ifs.is_open()) { RETURN_STATUS_UNEXPECTED("Invalid file, failed to open file: " + file); } @@ -618,8 +626,15 @@ int64_t CsvOp::CountTotalRows(const std::string &file) { MS_LOG(ERROR) << "Failed to initialize CSV Parser. Error:" << rc; return 0; } + + auto realpath = Common::GetRealPath(file); + if (!realpath.has_value()) { + MS_LOG(ERROR) << "Get real path failed, path=" << file; + return 0; + } + std::ifstream ifs; - ifs.open(file, std::ifstream::in); + ifs.open(realpath.value(), std::ifstream::in); if (!ifs.is_open()) { return 0; } @@ -703,8 +718,14 @@ Status CsvOp::ColMapAnalyse(const std::string &csv_file_name) { if (column_name_list_.empty()) { // Actually we only deal with the first file, because the column name set in other files must remain the same if (!check_flag_) { + auto realpath = Common::GetRealPath(csv_file_name); + if (!realpath.has_value()) { + MS_LOG(ERROR) << "Get real path failed, path=" << csv_file_name; + RETURN_STATUS_UNEXPECTED("Get real path failed, path=" + csv_file_name); + } + std::string line; - std::ifstream handle(csv_file_name); + std::ifstream handle(realpath.value()); getline(handle, line); std::vector col_names = split(line, field_delim_); @@ -757,8 +778,14 @@ bool CsvOp::ColumnNameValidate() { std::string match_file; for (auto &csv_file : csv_files_list_) { + auto realpath = Common::GetRealPath(csv_file); + if (!realpath.has_value()) { + MS_LOG(ERROR) << "Get real path failed, path=" << csv_file; + return false; + } + std::string line; - std::ifstream handle(csv_file); + std::ifstream handle(realpath.value()); // Parse the csv_file into column name set getline(handle, line); diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/manifest_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/manifest_op.cc index a38f816ba42..0e80fdb4ac7 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/manifest_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/manifest_op.cc @@ -21,6 +21,7 @@ #include #include +#include "debug/common.h" #include "utils/ms_utils.h" #include "minddata/dataset/core/config_manager.h" #include "minddata/dataset/core/tensor_shape.h" @@ -168,7 +169,13 @@ Status ManifestOp::GetClassIds(std::map> *cls_ids) // {"source": "/path/to/image1.jpg", "usage":"train", annotation": ...} // {"source": "/path/to/image2.jpg", "usage":"eval", "annotation": ...} Status ManifestOp::ParseManifestFile() { - std::ifstream file_handle(file_); + auto realpath = Common::GetRealPath(file_); + if (!realpath.has_value()) { + MS_LOG(ERROR) << "Get real path failed, path=" << file_; + RETURN_STATUS_UNEXPECTED("Get real path failed, path=" + file_); + } + + std::ifstream file_handle(realpath.value()); if (!file_handle.is_open()) { RETURN_STATUS_UNEXPECTED("Invalid file, failed to open Manifest file: " + file_); } @@ -235,10 +242,16 @@ Status ManifestOp::ParseManifestFile() { // Only support JPEG/PNG/GIF/BMP Status ManifestOp::CheckImageType(const std::string &file_name, bool *valid) { + auto realpath = Common::GetRealPath(file_name); + if (!realpath.has_value()) { + MS_LOG(ERROR) << "Get real path failed, path=" << file_name; + RETURN_STATUS_UNEXPECTED("Get real path failed, path=" + file_name); + } + std::ifstream file_handle; constexpr int read_num = 3; *valid = false; - file_handle.open(file_name, std::ios::binary | std::ios::in); + file_handle.open(realpath.value(), std::ios::binary | std::ios::in); if (!file_handle.is_open()) { RETURN_STATUS_UNEXPECTED("Invalid file, failed to open image file: " + file_name); } diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/text_file_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/text_file_op.cc index 033cbeae2fd..db6d1b4dd43 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/text_file_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/text_file_op.cc @@ -20,6 +20,7 @@ #include #include +#include "debug/common.h" #include "minddata/dataset/engine/datasetops/source/text_file_op.h" #include "minddata/dataset/core/config_manager.h" #include "minddata/dataset/util/wait_post.h" @@ -78,7 +79,13 @@ Status TextFileOp::LoadTensor(const std::string &line, TensorRow *out_row) { } Status TextFileOp::LoadFile(const std::string &file, int64_t start_offset, int64_t end_offset, int32_t worker_id) { - std::ifstream handle(file); + auto realpath = Common::GetRealPath(file); + if (!realpath.has_value()) { + MS_LOG(ERROR) << "Get real path failed, path=" << file; + RETURN_STATUS_UNEXPECTED("Get real path failed, path=" + file); + } + + std::ifstream handle(realpath.value()); if (!handle.is_open()) { RETURN_STATUS_UNEXPECTED("Invalid file, failed to open file: " + file); } @@ -162,7 +169,13 @@ Status TextFileOp::FillIOBlockQueue(const std::vector &i_keys) { // Internal helper function to calculate rows int64_t CountTotalRows(const std::string &file) { - std::ifstream handle(file); + auto realpath = Common::GetRealPath(file); + if (!realpath.has_value()) { + MS_LOG(ERROR) << "Get real path failed, path=" << file; + return 0; + } + + std::ifstream handle(realpath.value()); if (!handle.is_open()) { MS_LOG(ERROR) << "Invalid file, failed to open file: " << file; return 0; diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/tf_reader_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/tf_reader_op.cc index 0b3ae8ab0b8..fda009a0d75 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/tf_reader_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/tf_reader_op.cc @@ -24,6 +24,7 @@ #include #include +#include "debug/common.h" #include "proto/example.pb.h" #include "minddata/dataset/core/config_manager.h" #include "minddata/dataset/core/global_context.h" @@ -42,8 +43,14 @@ namespace dataset { const int64_t kTFRecordFileLimit = 0x140000000; bool TFReaderOp::ValidateFirstRowCrc(const std::string &filename) { + auto realpath = Common::GetRealPath(filename); + if (!realpath.has_value()) { + MS_LOG(ERROR) << "Get real path failed, path=" << filename; + return false; + } + std::ifstream reader; - reader.open(filename); + reader.open(realpath.value()); if (!reader) { return false; } @@ -262,8 +269,14 @@ Status TFReaderOp::FillIOBlockNoShuffle() { // Reads a tf_file file and loads the data into multiple TensorRows. Status TFReaderOp::LoadFile(const std::string &filename, int64_t start_offset, int64_t end_offset, int32_t worker_id) { + auto realpath = Common::GetRealPath(filename); + if (!realpath.has_value()) { + MS_LOG(ERROR) << "Get real path failed, path=" << filename; + RETURN_STATUS_UNEXPECTED("Get real path failed, path=" + filename); + } + std::ifstream reader; - reader.open(filename); + reader.open(realpath.value()); if (!reader) { RETURN_STATUS_UNEXPECTED("Invalid file, failed to open file: " + filename); } @@ -540,8 +553,14 @@ Status TFReaderOp::LoadIntList(const ColDescriptor ¤t_col, const dataengin } Status TFReaderOp::CreateSchema(const std::string tf_file, std::vector columns_to_load) { + auto realpath = Common::GetRealPath(tf_file); + if (!realpath.has_value()) { + MS_LOG(ERROR) << "Get real path failed, path=" << tf_file; + RETURN_STATUS_UNEXPECTED("Get real path failed, path=" + tf_file); + } + std::ifstream reader; - reader.open(tf_file); + reader.open(realpath.value()); // read length int64_t record_length = 0; @@ -663,8 +682,14 @@ Status TFReaderOp::CountTotalRows(int64_t *out_total_rows, const std::vector &filenames, int64_t begin, int64_t end) { int64_t rows_read = 0; for (int i = begin; i < end; i++) { + auto realpath = Common::GetRealPath(filenames[i]); + if (!realpath.has_value()) { + MS_LOG(ERROR) << "Get real path failed, path=" << filenames[i]; + continue; + } + std::ifstream reader; - reader.open(filenames[i]); + reader.open(realpath.value()); if (!reader) { MS_LOG(DEBUG) << "TFReader operator failed to open file " << filenames[i] << "."; } diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/voc_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/voc_op.cc index 82cb5998043..3c176224b50 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/voc_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/voc_op.cc @@ -19,6 +19,7 @@ #include #include +#include "debug/common.h" #include "minddata/dataset/core/config_manager.h" #include "minddata/dataset/core/tensor_shape.h" #include "minddata/dataset/engine/datasetops/source/sampler/sequential_sampler.h" @@ -166,8 +167,15 @@ Status VOCOp::ParseImageIds() { } else if (task_type_ == TaskType::Detection) { image_sets_file = folder_path_ + std::string(kImageSetsMain) + usage_ + std::string(kImageSetsExtension); } + + auto realpath = Common::GetRealPath(image_sets_file); + if (!realpath.has_value()) { + MS_LOG(ERROR) << "Get real path failed, path=" << image_sets_file; + RETURN_STATUS_UNEXPECTED("Get real path failed, path=" + image_sets_file); + } + std::ifstream in_file; - in_file.open(image_sets_file); + in_file.open(realpath.value()); if (in_file.fail()) { RETURN_STATUS_UNEXPECTED("Invalid file, failed to open file: " + image_sets_file); } diff --git a/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/celeba_node.cc b/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/celeba_node.cc index 941f3d5a202..0df0670db24 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/celeba_node.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/celeba_node.cc @@ -23,6 +23,7 @@ #include #include +#include "debug/common.h" #include "minddata/dataset/engine/datasetops/source/celeba_op.h" #include "minddata/dataset/util/status.h" namespace mindspore { @@ -92,7 +93,14 @@ Status CelebANode::GetDatasetSize(const std::shared_ptr &size std::ifstream partition_file; std::string line; Path folder_path(dataset_dir_); - std::ifstream attr_file((folder_path / "list_attr_celeba.txt").toString()); + + auto realpath = Common::GetRealPath((folder_path / "list_attr_celeba.txt").toString()); + if (!realpath.has_value()) { + MS_LOG(ERROR) << "Get real path failed, path=" << (folder_path / "list_attr_celeba.txt").toString(); + RETURN_STATUS_UNEXPECTED("Get real path failed, path=" + (folder_path / "list_attr_celeba.txt").toString()); + } + + std::ifstream attr_file(realpath.value()); if (!attr_file.is_open()) { std::string attr_file_name = (folder_path / "list_attr_celeba.txt").toString(); RETURN_STATUS_UNEXPECTED("Invalid file, failed to open Celeba attr file: " + attr_file_name); @@ -125,7 +133,13 @@ Status CelebANode::GetDatasetSize(const std::shared_ptr &size } } if (!partition_file.is_open()) { - partition_file.open((folder_path / "list_eval_partition.txt").toString()); + auto realpath_eval = Common::GetRealPath((folder_path / "list_eval_partition.txt").toString()); + if (!realpath_eval.has_value()) { + MS_LOG(ERROR) << "Get real path failed, path=" << (folder_path / "list_eval_partition.txt").toString(); + RETURN_STATUS_UNEXPECTED("Get real path failed, path=" + (folder_path / "list_eval_partition.txt").toString()); + } + + partition_file.open(realpath_eval.value()); } if (partition_file.is_open()) { while (getline(partition_file, line)) { diff --git a/mindspore/ccsrc/minddata/dataset/engine/perf/cpu_sampling.cc b/mindspore/ccsrc/minddata/dataset/engine/perf/cpu_sampling.cc index c78156914f7..5e5c14d11a1 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/perf/cpu_sampling.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/perf/cpu_sampling.cc @@ -14,6 +14,7 @@ * limitations under the License. */ #include "minddata/dataset/engine/perf/cpu_sampling.h" + #if !defined(_WIN32) && !defined(_WIN64) && !defined(__ANDROID__) && !defined(ANDROID) && !defined(__APPLE__) #include #endif @@ -23,6 +24,7 @@ #include #include #include + #include "minddata/dataset/api/python/pybind_conversion.h" #include "minddata/dataset/core/config_manager.h" #include "minddata/dataset/engine/execution_tree.h" @@ -211,6 +213,7 @@ Status DeviceCpu::SaveToFile(const std::string &file_path) { // Discard the content of the file when opening. std::ofstream os(file_path, std::ios::trunc); os << output; + os.close(); MS_LOG(INFO) << "Save device CPU success."; return Status::OK(); @@ -415,6 +418,7 @@ Status OperatorCpu::SaveToFile(const std::string &file_path) { // Discard the content of the file when opening. std::ofstream os(file_path, std::ios::trunc); os << output; + os.close(); MS_LOG(INFO) << "Save device CPU success."; return Status::OK(); @@ -532,9 +536,11 @@ Status ProcessCpu::SaveToFile(const std::string &file_path) { output["process_info"] = {{"user_utilization", user_util}, {"sys_utilization", sys_util}}; output["cpu_processor_num"] = cpu_processor_num_; + // Discard the content of the file when opening. std::ofstream os(file_path, std::ios::trunc); os << output; + os.close(); MS_LOG(INFO) << "Save process CPU success."; return Status::OK(); @@ -569,6 +575,7 @@ Status CpuSampling::SaveTimeStampToFile() { output["time_stamp"] = time_stamp_; std::ofstream os(file_path_, std::ios::trunc); os << output; + os.close(); return Status::OK(); } @@ -584,6 +591,7 @@ Status CpuSampling::SaveSamplingItervalToFile() { output["sampling_interval"] = GlobalContext::config_manager()->monitor_sampling_interval(); std::ofstream os(file_path_, std::ios::trunc); os << output; + os.close(); return Status::OK(); } diff --git a/mindspore/ccsrc/minddata/dataset/engine/serdes.cc b/mindspore/ccsrc/minddata/dataset/engine/serdes.cc index 7851d3f12f2..335f6620f39 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/serdes.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/serdes.cc @@ -15,6 +15,9 @@ */ #include "minddata/dataset/engine/serdes.h" +#include "debug/common.h" +#include "utils/utils.h" + namespace mindspore { namespace dataset { @@ -46,8 +49,17 @@ Status Serdes::SaveToJSON(std::shared_ptr node, const std::string & Status Serdes::SaveJSONToFile(nlohmann::json json_string, const std::string &file_name) { try { - std::ofstream file(file_name); + auto realpath = Common::GetRealPath(file_name); + if (!realpath.has_value()) { + MS_LOG(ERROR) << "Get real path failed, path=" << file_name; + RETURN_STATUS_UNEXPECTED("Get real path failed, path=" + file_name); + } + + std::ofstream file(realpath.value()); file << json_string; + file.close(); + + ChangeFileMode(realpath.value(), S_IRUSR | S_IWUSR); } catch (const std::exception &err) { RETURN_STATUS_UNEXPECTED("Save json string into " + file_name + " failed!"); } diff --git a/mindspore/ccsrc/minddata/dataset/include/dataset/data_helper.h b/mindspore/ccsrc/minddata/dataset/include/dataset/data_helper.h index 0643fb17ebe..e51195ba3b9 100644 --- a/mindspore/ccsrc/minddata/dataset/include/dataset/data_helper.h +++ b/mindspore/ccsrc/minddata/dataset/include/dataset/data_helper.h @@ -25,6 +25,7 @@ #include #include #include + #include "include/api/dual_abi_helper.h" #include "include/api/status.h" diff --git a/mindspore/ccsrc/minddata/dataset/text/sentence_piece_vocab.cc b/mindspore/ccsrc/minddata/dataset/text/sentence_piece_vocab.cc index 7b4406665bf..81d6f4c6b77 100644 --- a/mindspore/ccsrc/minddata/dataset/text/sentence_piece_vocab.cc +++ b/mindspore/ccsrc/minddata/dataset/text/sentence_piece_vocab.cc @@ -20,7 +20,9 @@ #include #include +#include "debug/common.h" #include "utils/ms_utils.h" +#include "utils/utils.h" #include "minddata/dataset/util/path.h" namespace mindspore { @@ -104,9 +106,17 @@ Status SentencePieceVocab::SaveModel(const std::shared_ptr * #endif std::string abs_real_path = (Path(real_path) / Path(filename)).toString(); - std::ofstream os_file(abs_real_path, std::ios::out); + auto realpath = Common::GetRealPath(abs_real_path); + if (!realpath.has_value()) { + RETURN_STATUS_UNEXPECTED("Get real path failed, path=" + abs_real_path); + } + + std::ofstream os_file(realpath.value(), std::ios::out); (void)os_file.write(vocab->get()->model_proto().data(), vocab->get()->model_proto().size()); os_file.close(); + + ChangeFileMode(realpath.value(), S_IRUSR | S_IWUSR); + return Status::OK(); } diff --git a/mindspore/ccsrc/minddata/dataset/text/vocab.cc b/mindspore/ccsrc/minddata/dataset/text/vocab.cc index d8209948349..497069d4d49 100644 --- a/mindspore/ccsrc/minddata/dataset/text/vocab.cc +++ b/mindspore/ccsrc/minddata/dataset/text/vocab.cc @@ -13,13 +13,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + +#include "minddata/dataset/text/vocab.h" + #include #include #include #include #include -#include "minddata/dataset/text/vocab.h" +#include "debug/common.h" #ifndef ENABLE_ANDROID #include "utils/log_adapter.h" #else @@ -121,8 +124,9 @@ Status Vocab::BuildFromFileCpp(const std::string &path, const std::string &delim const std::vector &special_tokens, bool prepend_special, std::shared_ptr *vocab) { // Validate parameters - if (path.empty()) { - RETURN_STATUS_UNEXPECTED("from_file: vocab file path is not set!"); + auto realpath = Common::GetRealPath(path); + if (!realpath.has_value()) { + RETURN_STATUS_UNEXPECTED("Get real path failed, path=" + path); } if (vocab_size < 0 && vocab_size != -1) { @@ -154,9 +158,10 @@ Status Vocab::BuildFromFileCpp(const std::string &path, const std::string &delim } WordIdType word_id = prepend_special ? static_cast(special_tokens.size()) : 0; std::unordered_map word2id; - std::fstream handle(path, std::ios::in); + + std::fstream handle(realpath.value(), std::ios::in); if (!handle.good() || !handle.is_open()) { - RETURN_STATUS_UNEXPECTED("from_file: fail to open: " + path); + RETURN_STATUS_UNEXPECTED("from_file: fail to open: " + realpath.value()); } std::string word; while (std::getline(handle, word)) { @@ -198,7 +203,13 @@ Status Vocab::BuildFromFile(const std::string &path, const std::string &delimite } WordIdType word_id = prepend_special ? static_cast(special_tokens.size()) : 0; std::unordered_map word2id; - std::fstream handle(path, std::ios::in); + + auto realpath = Common::GetRealPath(path); + if (!realpath.has_value()) { + RETURN_STATUS_UNEXPECTED("Get real path failed, path=" + path); + } + + std::fstream handle(realpath.value(), std::ios::in); CHECK_FAIL_RETURN_UNEXPECTED(handle.good() && handle.is_open(), "from_file: fail to open:" + path); std::string word; while (std::getline(handle, word)) { diff --git a/mindspore/ccsrc/minddata/mindrecord/io/shard_index_generator.cc b/mindspore/ccsrc/minddata/mindrecord/io/shard_index_generator.cc index d7e608be959..27f22dd4708 100644 --- a/mindspore/ccsrc/minddata/mindrecord/io/shard_index_generator.cc +++ b/mindspore/ccsrc/minddata/mindrecord/io/shard_index_generator.cc @@ -16,6 +16,8 @@ #include #include "minddata/mindrecord/include/shard_index_generator.h" + +#include "debug/common.h" #include "utils/ms_utils.h" using mindspore::LogStream; @@ -177,8 +179,14 @@ std::pair ShardIndexGenerator::GenerateFieldName( } std::pair ShardIndexGenerator::CheckDatabase(const std::string &shard_address) { + auto realpath = Common::GetRealPath(shard_address); + if (!realpath.has_value()) { + MS_LOG(ERROR) << "Get real path failed, path=" << shard_address; + return {FAILED, nullptr}; + } + sqlite3 *db = nullptr; - std::ifstream fin(common::SafeCStr(shard_address)); + std::ifstream fin(realpath.value()); if (!append_ && fin.good()) { MS_LOG(ERROR) << "Invalid file, DB file already exist: " << shard_address; fin.close(); @@ -522,8 +530,14 @@ MSRStatus ShardIndexGenerator::ExecuteTransaction(const int &shard_no, std::pair return FAILED; } + auto realpath = Common::GetRealPath(shard_address); + if (!realpath.has_value()) { + MS_LOG(ERROR) << "Get real path failed, path=" << shard_address; + return FAILED; + } + std::fstream in; - in.open(common::SafeCStr(shard_address), std::ios::in | std::ios::binary); + in.open(realpath.value(), std::ios::in | std::ios::binary); if (!in.good()) { MS_LOG(ERROR) << "Invalid file, failed to open file: " << shard_address; return FAILED; diff --git a/mindspore/ccsrc/minddata/mindrecord/io/shard_reader.cc b/mindspore/ccsrc/minddata/mindrecord/io/shard_reader.cc index 8a91c04819e..0413258308b 100644 --- a/mindspore/ccsrc/minddata/mindrecord/io/shard_reader.cc +++ b/mindspore/ccsrc/minddata/mindrecord/io/shard_reader.cc @@ -14,11 +14,13 @@ * limitations under the License. */ +#include "minddata/mindrecord/include/shard_reader.h" + #include #include +#include "debug/common.h" #include "minddata/mindrecord/include/shard_distributed_sample.h" -#include "minddata/mindrecord/include/shard_reader.h" #include "utils/ms_utils.h" using mindspore::LogStream; @@ -199,8 +201,14 @@ MSRStatus ShardReader::Open() { file_streams_.clear(); for (const auto &file : file_paths_) { + auto realpath = Common::GetRealPath(file); + if (!realpath.has_value()) { + MS_LOG(ERROR) << "Get real path failed, path=" << file; + return FAILED; + } + std::shared_ptr fs = std::make_shared(); - fs->open(common::SafeCStr(file), std::ios::in | std::ios::binary); + fs->open(realpath.value(), std::ios::in | std::ios::binary); if (!fs->good()) { MS_LOG(ERROR) << "Invalid file, failed to open file: " << file; return FAILED; @@ -217,8 +225,14 @@ MSRStatus ShardReader::Open(int n_consumer) { std::vector>>(n_consumer, std::vector>()); for (const auto &file : file_paths_) { for (int j = 0; j < n_consumer; ++j) { + auto realpath = Common::GetRealPath(file); + if (!realpath.has_value()) { + MS_LOG(ERROR) << "Get real path failed, path=" << file; + return FAILED; + } + std::shared_ptr fs = std::make_shared(); - fs->open(common::SafeCStr(file), std::ios::in | std::ios::binary); + fs->open(realpath.value(), std::ios::in | std::ios::binary); if (!fs->good()) { MS_LOG(ERROR) << "Invalid file, failed to open file: " << file; return FAILED; @@ -405,9 +419,16 @@ MSRStatus ShardReader::ReadAllRowsInShard(int shard_id, const std::string &sql, MS_LOG(INFO) << "Get " << static_cast(labels.size()) << " records from shard " << shard_id << " index."; std::string file_name = file_paths_[shard_id]; + + auto realpath = Common::GetRealPath(file_name); + if (!realpath.has_value()) { + MS_LOG(ERROR) << "Get real path failed, path=" << file_name; + return FAILED; + } + std::shared_ptr fs = std::make_shared(); if (!all_in_index_) { - fs->open(common::SafeCStr(file_name), std::ios::in | std::ios::binary); + fs->open(realpath.value(), std::ios::in | std::ios::binary); if (!fs->good()) { MS_LOG(ERROR) << "Invalid file, failed to open file: " << file_name; return FAILED; @@ -722,9 +743,16 @@ MSRStatus ShardReader::QueryWithCriteria(sqlite3 *db, const string &sql, const s std::pair> ShardReader::GetLabelsFromBinaryFile( int shard_id, const std::vector &columns, const std::vector> &label_offsets) { std::string file_name = file_paths_[shard_id]; + + auto realpath = Common::GetRealPath(file_name); + if (!realpath.has_value()) { + MS_LOG(ERROR) << "Get real path failed, path=" << file_name; + return {FAILED, {}}; + } + std::vector res; std::shared_ptr fs = std::make_shared(); - fs->open(common::SafeCStr(file_name), std::ios::in | std::ios::binary); + fs->open(realpath.value(), std::ios::in | std::ios::binary); if (!fs->good()) { MS_LOG(ERROR) << "Invalid file, failed to open file: " << file_name; return {FAILED, {}}; diff --git a/mindspore/ccsrc/minddata/mindrecord/io/shard_writer.cc b/mindspore/ccsrc/minddata/mindrecord/io/shard_writer.cc index 69b3504eb2b..db6315d9dc1 100644 --- a/mindspore/ccsrc/minddata/mindrecord/io/shard_writer.cc +++ b/mindspore/ccsrc/minddata/mindrecord/io/shard_writer.cc @@ -16,6 +16,7 @@ #include "minddata/dataset/util/random.h" #include "minddata/mindrecord/include/shard_writer.h" +#include "debug/common.h" #include "utils/ms_utils.h" #include "minddata/mindrecord/include/common/shard_utils.h" #include "./securec.h" @@ -77,10 +78,16 @@ MSRStatus ShardWriter::GetFullPathFromFileName(const std::vector &p MSRStatus ShardWriter::OpenDataFiles(bool append) { // Open files for (const auto &file : file_paths_) { + auto realpath = Common::GetRealPath(file); + if (!realpath.has_value()) { + MS_LOG(ERROR) << "Get real path failed, path=" << file; + return FAILED; + } + std::shared_ptr fs = std::make_shared(); if (!append) { // if not append and mindrecord file exist, return FAILED - fs->open(common::SafeCStr(file), std::ios::in | std::ios::binary); + fs->open(realpath.value(), std::ios::in | std::ios::binary); if (fs->good()) { MS_LOG(ERROR) << "MindRecord file already existed, please delete file: " << common::SafeCStr(file); fs->close(); @@ -558,8 +565,14 @@ int ShardWriter::LockWriter(bool parallel_writer) { // Open files file_streams_.clear(); for (const auto &file : file_paths_) { + auto realpath = Common::GetRealPath(file); + if (!realpath.has_value()) { + MS_LOG(ERROR) << "Get real path failed, path=" << file; + return -1; + } + std::shared_ptr fs = std::make_shared(); - fs->open(common::SafeCStr(file), std::ios::in | std::ios::out | std::ios::binary); + fs->open(realpath.value(), std::ios::in | std::ios::out | std::ios::binary); if (fs->fail()) { MS_LOG(ERROR) << "Invalid file, failed to open file: " << file; return -1; diff --git a/mindspore/ccsrc/minddata/mindrecord/meta/shard_header.cc b/mindspore/ccsrc/minddata/mindrecord/meta/shard_header.cc index 631e91e1731..040aa115e3e 100644 --- a/mindspore/ccsrc/minddata/mindrecord/meta/shard_header.cc +++ b/mindspore/ccsrc/minddata/mindrecord/meta/shard_header.cc @@ -22,6 +22,7 @@ #include #include +#include "debug/common.h" #include "utils/ms_utils.h" #include "minddata/mindrecord/include/shard_error.h" #include "minddata/mindrecord/include/shard_page.h" @@ -67,7 +68,13 @@ MSRStatus ShardHeader::InitializeHeader(const std::vector &headers, bool l } MSRStatus ShardHeader::CheckFileStatus(const std::string &path) { - std::ifstream fin(common::SafeCStr(path), std::ios::in | std::ios::binary); + auto realpath = Common::GetRealPath(path); + if (!realpath.has_value()) { + MS_LOG(ERROR) << "Get real path failed, path=" << path; + return FAILED; + } + + std::ifstream fin(realpath.value(), std::ios::in | std::ios::binary); if (!fin) { MS_LOG(ERROR) << "File does not exist or permission denied. path: " << path; return FAILED; @@ -700,8 +707,14 @@ std::pair, MSRStatus> ShardHeader::GetStatisticByID( } MSRStatus ShardHeader::PagesToFile(const std::string dump_file_name) { + auto realpath = Common::GetRealPath(dump_file_name); + if (!realpath.has_value()) { + MS_LOG(ERROR) << "Get real path failed, path=" << dump_file_name; + return FAILED; + } + // write header content to file, dump whatever is in the file before - std::ofstream page_out_handle(dump_file_name.c_str(), std::ios_base::trunc | std::ios_base::out); + std::ofstream page_out_handle(realpath.value(), std::ios_base::trunc | std::ios_base::out); if (page_out_handle.fail()) { MS_LOG(ERROR) << "Failed in opening page file"; return FAILED; @@ -720,8 +733,15 @@ MSRStatus ShardHeader::FileToPages(const std::string dump_file_name) { for (auto &v : pages_) { // clean pages v.clear(); } + + auto realpath = Common::GetRealPath(dump_file_name); + if (!realpath.has_value()) { + MS_LOG(ERROR) << "Get real path failed, path=" << dump_file_name; + return FAILED; + } + // attempt to open the file contains the page in json - std::ifstream page_in_handle(dump_file_name.c_str()); + std::ifstream page_in_handle(realpath.value()); if (!page_in_handle.good()) { MS_LOG(INFO) << "No page file exists."; diff --git a/mindspore/dataset/datapreprocess/preprocess_imagenet_validate_dataset.py b/mindspore/dataset/datapreprocess/preprocess_imagenet_validate_dataset.py index 160e2b8bf44..5d15318c85c 100644 --- a/mindspore/dataset/datapreprocess/preprocess_imagenet_validate_dataset.py +++ b/mindspore/dataset/datapreprocess/preprocess_imagenet_validate_dataset.py @@ -36,7 +36,8 @@ def preprocess_imagenet_validation_dataset(train_dataset_path, validation_datase if not os.path.exists(validate_sub_dir): os.makedirs(validate_sub_dir) - mappings = [mapping.strip() for mapping in open(image_label_mapping_file).readlines()] + real_file_path = os.path.realpath(image_label_mapping_file) + mappings = [mapping.strip() for mapping in open(real_file_path).readlines()] for mapping in mappings: image_dir = mapping.split(':') old_image_path = os.path.join(validation_dataset_path, image_dir[0]) diff --git a/mindspore/dataset/engine/serializer_deserializer.py b/mindspore/dataset/engine/serializer_deserializer.py index b6a6d80abf5..71b73811899 100644 --- a/mindspore/dataset/engine/serializer_deserializer.py +++ b/mindspore/dataset/engine/serializer_deserializer.py @@ -89,7 +89,8 @@ def deserialize(input_dict=None, json_filepath=None): if json_filepath: dict_pipeline = dict() - with open(json_filepath, 'r') as json_file: + real_file_path = os.path.realpath(json_filepath) + with open(real_file_path, 'r') as json_file: dict_pipeline = json.load(json_file) data = construct_pipeline(dict_pipeline) diff --git a/mindspore/mindrecord/tools/cifar10.py b/mindspore/mindrecord/tools/cifar10.py index 15907d4e550..ccea3affeea 100644 --- a/mindspore/mindrecord/tools/cifar10.py +++ b/mindspore/mindrecord/tools/cifar10.py @@ -101,12 +101,14 @@ class Cifar10: files = os.listdir(self.path) for file in files: if re.match("data_batch_*", file): - with open(os.path.join(self.path, file), 'rb') as f: # load train data + real_file_path = os.path.realpath(self.path) + with open(os.path.join(real_file_path, file), 'rb') as f: # load train data dic = restricted_loads(f.read()) images = np.r_[images, dic[b"data"].reshape([-1, 3, 32, 32])] labels.append(dic[b"labels"]) elif re.match("test_batch", file): # load test data - with open(os.path.join(self.path, file), 'rb') as f: + real_file_path = os.path.realpath(self.path) + with open(os.path.join(real_file_path, file), 'rb') as f: dic = restricted_loads(f.read()) test_images = np.array(dic[b"data"].reshape([-1, 3, 32, 32])) test_labels = np.array(dic[b"labels"]) diff --git a/mindspore/mindrecord/tools/cifar100.py b/mindspore/mindrecord/tools/cifar100.py index eceb547a414..cc6e0869a4e 100644 --- a/mindspore/mindrecord/tools/cifar100.py +++ b/mindspore/mindrecord/tools/cifar100.py @@ -105,13 +105,15 @@ class Cifar100: files = os.listdir(self.path) for file in files: if file == "train": - with open(os.path.join(self.path, file), 'rb') as f: # load train data + real_file_path = os.path.realpath(self.path) + with open(os.path.join(real_file_path, file), 'rb') as f: # load train data dic = restricted_loads(f.read()) images = np.array(dic[b"data"].reshape([-1, 3, 32, 32])) fine_labels.append(dic[b"fine_labels"]) coarse_labels.append(dic[b"coarse_labels"]) elif file == "test": # load test data - with open(os.path.join(self.path, file), 'rb') as f: + real_file_path = os.path.realpath(self.path) + with open(os.path.join(real_file_path, file), 'rb') as f: dic = restricted_loads(f.read()) test_images = np.array(dic[b"data"].reshape([-1, 3, 32, 32])) test_fine_labels = np.array(dic[b"fine_labels"]) diff --git a/mindspore/mindrecord/tools/imagenet_to_mr.py b/mindspore/mindrecord/tools/imagenet_to_mr.py index 6f772e15a71..d6fcb245dea 100644 --- a/mindspore/mindrecord/tools/imagenet_to_mr.py +++ b/mindspore/mindrecord/tools/imagenet_to_mr.py @@ -74,11 +74,12 @@ class ImageNetToMR: Yields: data (dict of list): imagenet data list which contains dict. """ - if not os.path.exists(self.map_file): + real_file_path = os.path.realpath(self.map_file) + if not os.path.exists(real_file_path): raise IOError("map file {} not exists".format(self.map_file)) label_dict = {} - with open(self.map_file) as fp: + with open(real_file_path) as fp: line = fp.readline() while line: labels = line.split(" ") @@ -109,7 +110,8 @@ class ImageNetToMR: data["label"] = int(label) # get the image data - image_file = open(file_name, "rb") + real_file_path = os.path.realpath(file_name) + image_file = open(real_file_path, "rb") image_bytes = image_file.read() image_file.close() if not image_bytes: diff --git a/mindspore/mindrecord/tools/mnist_to_mr.py b/mindspore/mindrecord/tools/mnist_to_mr.py index b4b2560b009..3a341c9ae29 100644 --- a/mindspore/mindrecord/tools/mnist_to_mr.py +++ b/mindspore/mindrecord/tools/mnist_to_mr.py @@ -80,7 +80,8 @@ class MnistToMR: def _extract_images(self, filename): """Extract the images into a 4D tensor [image index, y, x, channels].""" - with gzip.open(filename) as bytestream: + real_file_path = os.path.realpath(filename) + with gzip.open(real_file_path) as bytestream: bytestream.read(16) buf = bytestream.read() data = np.frombuffer(buf, dtype=np.uint8) @@ -89,7 +90,8 @@ class MnistToMR: def _extract_labels(self, filename): """Extract the labels into a vector of int64 label IDs.""" - with gzip.open(filename) as bytestream: + real_file_path = os.path.realpath(filename) + with gzip.open(real_file_path) as bytestream: bytestream.read(8) buf = bytestream.read() labels = np.frombuffer(buf, dtype=np.uint8).astype(np.int64)