JHV000
/
mindspore2022
forked from huawei/mindspore2022
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix: regular path

This commit is contained in:
jonyguo 2021-06-11 11:20:48 +08:00
parent 74be005bc6
commit 7ebf5b6011
28 changed files with 336 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
mindspore/ccsrc/minddata/dataset/api/execute.cc
View File

@ -14,9 +14,11 @@
* limitations under the License.
*/
#include "minddata/dataset/include/dataset/execute.h"
#include <algorithm>
#include <fstream>
#include "minddata/dataset/include/dataset/execute.h"
#include "minddata/dataset/core/de_tensor.h"
#include "minddata/dataset/core/tensor_row.h"
#include "minddata/dataset/core/tensor.h"

9
mindspore/ccsrc/minddata/dataset/api/text.cc
View File

@ -18,6 +18,7 @@
#include <fstream>
#include <regex>
#include "debug/common.h"
#include "minddata/dataset/include/dataset/text.h"
#include "minddata/dataset/core/type_id.h"
#include "minddata/dataset/text/ir/kernels/text_ir.h"
@ -165,7 +166,13 @@ Status JiebaTokenizer::AddDictChar(const std::vector<char> &file_path) {
Status JiebaTokenizer::ParserFile(const std::string &file_path,
std::vector<std::pair<std::string, int64_t>> *const user_dict) {
std::ifstream ifs(file_path);
auto realpath = Common::GetRealPath(file_path);
if (!realpath.has_value()) {
MS_LOG(ERROR) << "Get real path failed, path=" << file_path;
RETURN_STATUS_SYNTAX_ERROR("Get real path failed, path=" + file_path);
}
std::ifstream ifs(realpath.value());
if (!ifs) {
std::string err_msg = "JiebaTokenizer : Fail to load dictionary from the input file, check the file path.";
MS_LOG(ERROR) << err_msg;

2
mindspore/ccsrc/minddata/dataset/engine/cache/CMakeLists.txt vendored
View File

@ -108,7 +108,7 @@ if(ENABLE_CACHE)
add_executable(cache_admin cache_admin.cc cache_admin_arg.cc)
target_link_libraries(cache_admin _c_dataengine _c_mindrecord mindspore::protobuf ${PYTHON_LIBRARIES} pthread)
target_link_libraries(cache_admin mindspore mindspore_shared_lib)
target_link_libraries(cache_admin mindspore mindspore_core mindspore_shared_lib)
if(USE_GLOG)
target_link_libraries(cache_admin mindspore::glog)

21
mindspore/ccsrc/minddata/dataset/engine/cache/cache_hw.cc vendored
View File

@ -14,6 +14,7 @@
* limitations under the License.
*/
#include "minddata/dataset/engine/cache/cache_hw.h"
#ifdef NUMA_ENABLED
#include <numa.h>
#endif
@ -24,7 +25,10 @@
#include <fstream>
#include <regex>
#include <thread>
#include "debug/common.h"
#include "utils/log_adapter.h"
namespace mindspore {
namespace dataset {
CacheServerHW::CacheServerHW() {
@ -118,7 +122,14 @@ Status CacheServerHW::GetNumaNodeInfo() {
auto node_dir = p.Basename();
numa_id_t numa_node = static_cast<numa_id_t>(strtol(node_dir.data() + strlen(kNodeName), nullptr, kDecimal));
Path f = p / kCpuList;
std::ifstream fs(f.toString());
auto realpath = Common::GetRealPath(f.toString());
if (!realpath.has_value()) {
MS_LOG(ERROR) << "Get real path failed, path=" << f.toString();
RETURN_STATUS_UNEXPECTED("Get real path failed, path=" + f.toString());
}
std::ifstream fs(realpath.value());
CHECK_FAIL_RETURN_UNEXPECTED(!fs.fail(), "Fail to open file: " + f.toString());
std::string cpu_string;
cpu_set_t cpuset;
@ -235,7 +246,13 @@ bool CacheServerHW::numa_enabled() {
}
uint64_t CacheServerHW::GetAvailableMemory() {
std::ifstream mem_file(kMemInfoFileName);
auto realpath = Common::GetRealPath(kMemInfoFileName);
if (!realpath.has_value()) {
MS_LOG(ERROR) << "Get real path failed, path=" << kMemInfoFileName;
return 0;
}
std::ifstream mem_file(realpath.value());
if (mem_file.fail()) {
MS_LOG(WARNING) << "Fail to open file: " << kMemInfoFileName;
return 0;

10
mindspore/ccsrc/minddata/dataset/engine/datasetops/source/celeba_op.cc
View File

@ -18,6 +18,7 @@
#include <algorithm>
#include <fstream>
#include <iomanip>
#include "debug/common.h"
#include "minddata/dataset/core/config_manager.h"
#include "minddata/dataset/util/path.h"
#include "minddata/dataset/engine/datasetops/source/sampler/sequential_sampler.h"
@ -69,7 +70,14 @@ Status CelebAOp::LaunchThreadsAndInitOp() {
Status CelebAOp::ParseAttrFile() {
TaskManager::FindMe()->Post();
Path folder_path(folder_path_);
std::ifstream attr_file((folder_path / "list_attr_celeba.txt").toString());
auto realpath = Common::GetRealPath((folder_path / "list_attr_celeba.txt").toString());
if (!realpath.has_value()) {
MS_LOG(ERROR) << "Get real path failed, path=" << (folder_path / "list_attr_celeba.txt").toString();
RETURN_STATUS_UNEXPECTED("Get real path failed, path=" + (folder_path / "list_attr_celeba.txt").toString());
}
std::ifstream attr_file(realpath.value());
if (!attr_file.is_open()) {
std::string attr_file_name = (folder_path / "list_attr_celeba.txt").toString();
return Status(StatusCode::kMDFileNotExist, __LINE__, __FILE__,

17
mindspore/ccsrc/minddata/dataset/engine/datasetops/source/clue_op.cc
View File

@ -22,6 +22,7 @@
#include <iomanip>
#include <utility>
#include "debug/common.h"
#include "minddata/dataset/core/config_manager.h"
#include "minddata/dataset/engine/jagged_connector.h"
#include "minddata/dataset/engine/execution_tree.h"
@ -83,7 +84,13 @@ Status ClueOp::GetValue(const nlohmann::json &js, std::vector<std::string> key_c
}
Status ClueOp::LoadFile(const std::string &file, int64_t start_offset, int64_t end_offset, int32_t worker_id) {
std::ifstream handle(file);
auto realpath = Common::GetRealPath(file);
if (!realpath.has_value()) {
MS_LOG(ERROR) << "Get real path failed, path=" << file;
RETURN_STATUS_UNEXPECTED("Get real path failed, path=" + file);
}
std::ifstream handle(realpath.value());
if (!handle.is_open()) {
RETURN_STATUS_UNEXPECTED("Invalid file, failed to open file: " + file);
}
@ -223,7 +230,13 @@ Status ClueOp::CalculateNumRowsPerShard() {
}
int64_t CountTotalRowsPerFile(const std::string &file) {
std::ifstream handle(file);
auto realpath = Common::GetRealPath(file);
if (!realpath.has_value()) {
MS_LOG(ERROR) << "Get real path failed, path=" << file;
return 0;
}
std::ifstream handle(realpath.value());
if (!handle.is_open()) {
MS_LOG(ERROR) << "Invalid file, failed to open file: " << file;
return 0;

9
mindspore/ccsrc/minddata/dataset/engine/datasetops/source/coco_op.cc
View File

@ -17,6 +17,7 @@
#include <algorithm>
#include <fstream>
#include "debug/common.h"
#include "utils/ms_utils.h"
#include "minddata/dataset/core/config_manager.h"
#include "minddata/dataset/core/tensor_shape.h"
@ -335,7 +336,13 @@ Status CocoOp::SearchNodeInJson(const nlohmann::json &input_tree, std::string no
Status CocoOp::ParseAnnotationIds() {
nlohmann::json js;
try {
std::ifstream in(annotation_path_);
auto realpath = Common::GetRealPath(annotation_path_);
if (!realpath.has_value()) {
MS_LOG(ERROR) << "Get real path failed, path=" << annotation_path_;
RETURN_STATUS_UNEXPECTED("Get real path failed, path=" + annotation_path_);
}
std::ifstream in(realpath.value());
in >> js;
} catch (const std::exception &err) {
RETURN_STATUS_UNEXPECTED("Invalid file, failed to open JSON file: " + annotation_path_ + ".");

35
mindspore/ccsrc/minddata/dataset/engine/datasetops/source/csv_op.cc
View File

@ -20,6 +20,7 @@
#include <iomanip>
#include <stdexcept>
#include "debug/common.h"
#include "minddata/dataset/core/config_manager.h"
#include "minddata/dataset/engine/jagged_connector.h"
#include "minddata/dataset/engine/execution_tree.h"
@ -487,8 +488,15 @@ Status CsvOp::LoadFile(const std::string &file, int64_t start_offset, int64_t en
RETURN_IF_NOT_OK(csv_parser.InitCsvParser());
csv_parser.SetStartOffset(start_offset);
csv_parser.SetEndOffset(end_offset);
auto realpath = Common::GetRealPath(file);
if (!realpath.has_value()) {
MS_LOG(ERROR) << "Get real path failed, path=" << file;
RETURN_STATUS_UNEXPECTED("Get real path failed, path=" + file);
}
std::ifstream ifs;
ifs.open(file, std::ifstream::in);
ifs.open(realpath.value(), std::ifstream::in);
if (!ifs.is_open()) {
RETURN_STATUS_UNEXPECTED("Invalid file, failed to open file: " + file);
}
@ -618,8 +626,15 @@ int64_t CsvOp::CountTotalRows(const std::string &file) {
MS_LOG(ERROR) << "Failed to initialize CSV Parser. Error:" << rc;
return 0;
}
auto realpath = Common::GetRealPath(file);
if (!realpath.has_value()) {
MS_LOG(ERROR) << "Get real path failed, path=" << file;
return 0;
}
std::ifstream ifs;
ifs.open(file, std::ifstream::in);
ifs.open(realpath.value(), std::ifstream::in);
if (!ifs.is_open()) {
return 0;
}
@ -703,8 +718,14 @@ Status CsvOp::ColMapAnalyse(const std::string &csv_file_name) {
if (column_name_list_.empty()) {
// Actually we only deal with the first file, because the column name set in other files must remain the same
if (!check_flag_) {
auto realpath = Common::GetRealPath(csv_file_name);
if (!realpath.has_value()) {
MS_LOG(ERROR) << "Get real path failed, path=" << csv_file_name;
RETURN_STATUS_UNEXPECTED("Get real path failed, path=" + csv_file_name);
}
std::string line;
std::ifstream handle(csv_file_name);
std::ifstream handle(realpath.value());
getline(handle, line);
std::vector<std::string> col_names = split(line, field_delim_);
@ -757,8 +778,14 @@ bool CsvOp::ColumnNameValidate() {
std::string match_file;
for (auto &csv_file : csv_files_list_) {
auto realpath = Common::GetRealPath(csv_file);
if (!realpath.has_value()) {
MS_LOG(ERROR) << "Get real path failed, path=" << csv_file;
return false;
}
std::string line;
std::ifstream handle(csv_file);
std::ifstream handle(realpath.value());
// Parse the csv_file into column name set
getline(handle, line);

17
mindspore/ccsrc/minddata/dataset/engine/datasetops/source/manifest_op.cc
View File

@ -21,6 +21,7 @@
#include <set>
#include <nlohmann/json.hpp>
#include "debug/common.h"
#include "utils/ms_utils.h"
#include "minddata/dataset/core/config_manager.h"
#include "minddata/dataset/core/tensor_shape.h"
@ -168,7 +169,13 @@ Status ManifestOp::GetClassIds(std::map<int32_t, std::vector<int64_t>> *cls_ids)
// {"source": "/path/to/image1.jpg", "usage":"train", annotation": ...}
// {"source": "/path/to/image2.jpg", "usage":"eval", "annotation": ...}
Status ManifestOp::ParseManifestFile() {
std::ifstream file_handle(file_);
auto realpath = Common::GetRealPath(file_);
if (!realpath.has_value()) {
MS_LOG(ERROR) << "Get real path failed, path=" << file_;
RETURN_STATUS_UNEXPECTED("Get real path failed, path=" + file_);
}
std::ifstream file_handle(realpath.value());
if (!file_handle.is_open()) {
RETURN_STATUS_UNEXPECTED("Invalid file, failed to open Manifest file: " + file_);
}
@ -235,10 +242,16 @@ Status ManifestOp::ParseManifestFile() {
// Only support JPEG/PNG/GIF/BMP
Status ManifestOp::CheckImageType(const std::string &file_name, bool *valid) {
auto realpath = Common::GetRealPath(file_name);
if (!realpath.has_value()) {
MS_LOG(ERROR) << "Get real path failed, path=" << file_name;
RETURN_STATUS_UNEXPECTED("Get real path failed, path=" + file_name);
}
std::ifstream file_handle;
constexpr int read_num = 3;
*valid = false;
file_handle.open(file_name, std::ios::binary | std::ios::in);
file_handle.open(realpath.value(), std::ios::binary | std::ios::in);
if (!file_handle.is_open()) {
RETURN_STATUS_UNEXPECTED("Invalid file, failed to open image file: " + file_name);
}

17
mindspore/ccsrc/minddata/dataset/engine/datasetops/source/text_file_op.cc
View File

@ -20,6 +20,7 @@
#include <string>
#include <utility>
#include "debug/common.h"
#include "minddata/dataset/engine/datasetops/source/text_file_op.h"
#include "minddata/dataset/core/config_manager.h"
#include "minddata/dataset/util/wait_post.h"
@ -78,7 +79,13 @@ Status TextFileOp::LoadTensor(const std::string &line, TensorRow *out_row) {
}
Status TextFileOp::LoadFile(const std::string &file, int64_t start_offset, int64_t end_offset, int32_t worker_id) {
std::ifstream handle(file);
auto realpath = Common::GetRealPath(file);
if (!realpath.has_value()) {
MS_LOG(ERROR) << "Get real path failed, path=" << file;
RETURN_STATUS_UNEXPECTED("Get real path failed, path=" + file);
}
std::ifstream handle(realpath.value());
if (!handle.is_open()) {
RETURN_STATUS_UNEXPECTED("Invalid file, failed to open file: " + file);
}
@ -162,7 +169,13 @@ Status TextFileOp::FillIOBlockQueue(const std::vector<int64_t> &i_keys) {
// Internal helper function to calculate rows
int64_t CountTotalRows(const std::string &file) {
std::ifstream handle(file);
auto realpath = Common::GetRealPath(file);
if (!realpath.has_value()) {
MS_LOG(ERROR) << "Get real path failed, path=" << file;
return 0;
}
std::ifstream handle(realpath.value());
if (!handle.is_open()) {
MS_LOG(ERROR) << "Invalid file, failed to open file: " << file;
return 0;

33
mindspore/ccsrc/minddata/dataset/engine/datasetops/source/tf_reader_op.cc
View File

@ -24,6 +24,7 @@
#include <utility>
#include <vector>
#include "debug/common.h"
#include "proto/example.pb.h"
#include "minddata/dataset/core/config_manager.h"
#include "minddata/dataset/core/global_context.h"
@ -42,8 +43,14 @@ namespace dataset {
const int64_t kTFRecordFileLimit = 0x140000000;
bool TFReaderOp::ValidateFirstRowCrc(const std::string &filename) {
auto realpath = Common::GetRealPath(filename);
if (!realpath.has_value()) {
MS_LOG(ERROR) << "Get real path failed, path=" << filename;
return false;
}
std::ifstream reader;
reader.open(filename);
reader.open(realpath.value());
if (!reader) {
return false;
}
@ -262,8 +269,14 @@ Status TFReaderOp::FillIOBlockNoShuffle() {
// Reads a tf_file file and loads the data into multiple TensorRows.
Status TFReaderOp::LoadFile(const std::string &filename, int64_t start_offset, int64_t end_offset, int32_t worker_id) {
auto realpath = Common::GetRealPath(filename);
if (!realpath.has_value()) {
MS_LOG(ERROR) << "Get real path failed, path=" << filename;
RETURN_STATUS_UNEXPECTED("Get real path failed, path=" + filename);
}
std::ifstream reader;
reader.open(filename);
reader.open(realpath.value());
if (!reader) {
RETURN_STATUS_UNEXPECTED("Invalid file, failed to open file: " + filename);
}
@ -540,8 +553,14 @@ Status TFReaderOp::LoadIntList(const ColDescriptor &current_col, const dataengin
}
Status TFReaderOp::CreateSchema(const std::string tf_file, std::vector<std::string> columns_to_load) {
auto realpath = Common::GetRealPath(tf_file);
if (!realpath.has_value()) {
MS_LOG(ERROR) << "Get real path failed, path=" << tf_file;
RETURN_STATUS_UNEXPECTED("Get real path failed, path=" + tf_file);
}
std::ifstream reader;
reader.open(tf_file);
reader.open(realpath.value());
// read length
int64_t record_length = 0;
@ -663,8 +682,14 @@ Status TFReaderOp::CountTotalRows(int64_t *out_total_rows, const std::vector<std
int64_t TFReaderOp::CountTotalRowsSectioned(const std::vector<std::string> &filenames, int64_t begin, int64_t end) {
int64_t rows_read = 0;
for (int i = begin; i < end; i++) {
auto realpath = Common::GetRealPath(filenames[i]);
if (!realpath.has_value()) {
MS_LOG(ERROR) << "Get real path failed, path=" << filenames[i];
continue;
}
std::ifstream reader;
reader.open(filenames[i]);
reader.open(realpath.value());
if (!reader) {
MS_LOG(DEBUG) << "TFReader operator failed to open file " << filenames[i] << ".";
}

10
mindspore/ccsrc/minddata/dataset/engine/datasetops/source/voc_op.cc
View File

@ -19,6 +19,7 @@
#include <fstream>
#include <iomanip>
#include "debug/common.h"
#include "minddata/dataset/core/config_manager.h"
#include "minddata/dataset/core/tensor_shape.h"
#include "minddata/dataset/engine/datasetops/source/sampler/sequential_sampler.h"
@ -166,8 +167,15 @@ Status VOCOp::ParseImageIds() {
} else if (task_type_ == TaskType::Detection) {
image_sets_file = folder_path_ + std::string(kImageSetsMain) + usage_ + std::string(kImageSetsExtension);
}
auto realpath = Common::GetRealPath(image_sets_file);
if (!realpath.has_value()) {
MS_LOG(ERROR) << "Get real path failed, path=" << image_sets_file;
RETURN_STATUS_UNEXPECTED("Get real path failed, path=" + image_sets_file);
}
std::ifstream in_file;
in_file.open(image_sets_file);
in_file.open(realpath.value());
if (in_file.fail()) {
RETURN_STATUS_UNEXPECTED("Invalid file, failed to open file: " + image_sets_file);
}

18
mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/celeba_node.cc
View File

@ -23,6 +23,7 @@
#include <vector>
#include <algorithm>
#include "debug/common.h"
#include "minddata/dataset/engine/datasetops/source/celeba_op.h"
#include "minddata/dataset/util/status.h"
namespace mindspore {
@ -92,7 +93,14 @@ Status CelebANode::GetDatasetSize(const std::shared_ptr<DatasetSizeGetter> &size
std::ifstream partition_file;
std::string line;
Path folder_path(dataset_dir_);
std::ifstream attr_file((folder_path / "list_attr_celeba.txt").toString());
auto realpath = Common::GetRealPath((folder_path / "list_attr_celeba.txt").toString());
if (!realpath.has_value()) {
MS_LOG(ERROR) << "Get real path failed, path=" << (folder_path / "list_attr_celeba.txt").toString();
RETURN_STATUS_UNEXPECTED("Get real path failed, path=" + (folder_path / "list_attr_celeba.txt").toString());
}
std::ifstream attr_file(realpath.value());
if (!attr_file.is_open()) {
std::string attr_file_name = (folder_path / "list_attr_celeba.txt").toString();
RETURN_STATUS_UNEXPECTED("Invalid file, failed to open Celeba attr file: " + attr_file_name);
@ -125,7 +133,13 @@ Status CelebANode::GetDatasetSize(const std::shared_ptr<DatasetSizeGetter> &size
}
}
if (!partition_file.is_open()) {
partition_file.open((folder_path / "list_eval_partition.txt").toString());
auto realpath_eval = Common::GetRealPath((folder_path / "list_eval_partition.txt").toString());
if (!realpath_eval.has_value()) {
MS_LOG(ERROR) << "Get real path failed, path=" << (folder_path / "list_eval_partition.txt").toString();
RETURN_STATUS_UNEXPECTED("Get real path failed, path=" + (folder_path / "list_eval_partition.txt").toString());
}
partition_file.open(realpath_eval.value());
}
if (partition_file.is_open()) {
while (getline(partition_file, line)) {

8
mindspore/ccsrc/minddata/dataset/engine/perf/cpu_sampling.cc
View File

@ -14,6 +14,7 @@
* limitations under the License.
*/
#include "minddata/dataset/engine/perf/cpu_sampling.h"
#if !defined(_WIN32) && !defined(_WIN64) && !defined(__ANDROID__) && !defined(ANDROID) && !defined(__APPLE__)
#include <sys/syscall.h>
#endif
@ -23,6 +24,7 @@
#include <fstream>
#include <memory>
#include <string>
#include "minddata/dataset/api/python/pybind_conversion.h"
#include "minddata/dataset/core/config_manager.h"
#include "minddata/dataset/engine/execution_tree.h"
@ -211,6 +213,7 @@ Status DeviceCpu::SaveToFile(const std::string &file_path) {
// Discard the content of the file when opening.
std::ofstream os(file_path, std::ios::trunc);
os << output;
os.close();
MS_LOG(INFO) << "Save device CPU success.";
return Status::OK();
@ -415,6 +418,7 @@ Status OperatorCpu::SaveToFile(const std::string &file_path) {
// Discard the content of the file when opening.
std::ofstream os(file_path, std::ios::trunc);
os << output;
os.close();
MS_LOG(INFO) << "Save device CPU success.";
return Status::OK();
@ -532,9 +536,11 @@ Status ProcessCpu::SaveToFile(const std::string &file_path) {
output["process_info"] = {{"user_utilization", user_util}, {"sys_utilization", sys_util}};
output["cpu_processor_num"] = cpu_processor_num_;
// Discard the content of the file when opening.
std::ofstream os(file_path, std::ios::trunc);
os << output;
os.close();
MS_LOG(INFO) << "Save process CPU success.";
return Status::OK();
@ -569,6 +575,7 @@ Status CpuSampling::SaveTimeStampToFile() {
output["time_stamp"] = time_stamp_;
std::ofstream os(file_path_, std::ios::trunc);
os << output;
os.close();
return Status::OK();
}
@ -584,6 +591,7 @@ Status CpuSampling::SaveSamplingItervalToFile() {
output["sampling_interval"] = GlobalContext::config_manager()->monitor_sampling_interval();
std::ofstream os(file_path_, std::ios::trunc);
os << output;
os.close();
return Status::OK();
}

14
mindspore/ccsrc/minddata/dataset/engine/serdes.cc
View File

@ -15,6 +15,9 @@
*/
#include "minddata/dataset/engine/serdes.h"
#include "debug/common.h"
#include "utils/utils.h"
namespace mindspore {
namespace dataset {
@ -46,8 +49,17 @@ Status Serdes::SaveToJSON(std::shared_ptr<DatasetNode> node, const std::string &
Status Serdes::SaveJSONToFile(nlohmann::json json_string, const std::string &file_name) {
try {
std::ofstream file(file_name);
auto realpath = Common::GetRealPath(file_name);
if (!realpath.has_value()) {
MS_LOG(ERROR) << "Get real path failed, path=" << file_name;
RETURN_STATUS_UNEXPECTED("Get real path failed, path=" + file_name);
}
std::ofstream file(realpath.value());
file << json_string;
file.close();
ChangeFileMode(realpath.value(), S_IRUSR | S_IWUSR);
} catch (const std::exception &err) {
RETURN_STATUS_UNEXPECTED("Save json string into " + file_name + " failed!");
}

1
mindspore/ccsrc/minddata/dataset/include/dataset/data_helper.h
View File

@ -25,6 +25,7 @@
#include <string>
#include <unordered_map>
#include <vector>
#include "include/api/dual_abi_helper.h"
#include "include/api/status.h"

12
mindspore/ccsrc/minddata/dataset/text/sentence_piece_vocab.cc
View File

@ -20,7 +20,9 @@
#include <sentencepiece_processor.h>
#include <fstream>
#include "debug/common.h"
#include "utils/ms_utils.h"
#include "utils/utils.h"
#include "minddata/dataset/util/path.h"
namespace mindspore {
@ -104,9 +106,17 @@ Status SentencePieceVocab::SaveModel(const std::shared_ptr<SentencePieceVocab> *
#endif
std::string abs_real_path = (Path(real_path) / Path(filename)).toString();
std::ofstream os_file(abs_real_path, std::ios::out);
auto realpath = Common::GetRealPath(abs_real_path);
if (!realpath.has_value()) {
RETURN_STATUS_UNEXPECTED("Get real path failed, path=" + abs_real_path);
}
std::ofstream os_file(realpath.value(), std::ios::out);
(void)os_file.write(vocab->get()->model_proto().data(), vocab->get()->model_proto().size());
os_file.close();
ChangeFileMode(realpath.value(), S_IRUSR | S_IWUSR);
return Status::OK();
}

23
mindspore/ccsrc/minddata/dataset/text/vocab.cc
View File

@ -13,13 +13,16 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "minddata/dataset/text/vocab.h"
#include <fstream>
#include <unordered_set>
#include <unordered_map>
#include <utility>
#include <algorithm>
#include "minddata/dataset/text/vocab.h"
#include "debug/common.h"
#ifndef ENABLE_ANDROID
#include "utils/log_adapter.h"
#else
@ -121,8 +124,9 @@ Status Vocab::BuildFromFileCpp(const std::string &path, const std::string &delim
const std::vector<WordType> &special_tokens, bool prepend_special,
std::shared_ptr<Vocab> *vocab) {
// Validate parameters
if (path.empty()) {
RETURN_STATUS_UNEXPECTED("from_file: vocab file path is not set!");
auto realpath = Common::GetRealPath(path);
if (!realpath.has_value()) {
RETURN_STATUS_UNEXPECTED("Get real path failed, path=" + path);
}
if (vocab_size < 0 && vocab_size != -1) {
@ -154,9 +158,10 @@ Status Vocab::BuildFromFileCpp(const std::string &path, const std::string &delim
}
WordIdType word_id = prepend_special ? static_cast<WordIdType>(special_tokens.size()) : 0;
std::unordered_map<WordType, WordIdType> word2id;
std::fstream handle(path, std::ios::in);
std::fstream handle(realpath.value(), std::ios::in);
if (!handle.good() || !handle.is_open()) {
RETURN_STATUS_UNEXPECTED("from_file: fail to open: " + path);
RETURN_STATUS_UNEXPECTED("from_file: fail to open: " + realpath.value());
}
std::string word;
while (std::getline(handle, word)) {
@ -198,7 +203,13 @@ Status Vocab::BuildFromFile(const std::string &path, const std::string &delimite
}
WordIdType word_id = prepend_special ? static_cast<WordIdType>(special_tokens.size()) : 0;
std::unordered_map<WordType, WordIdType> word2id;
std::fstream handle(path, std::ios::in);
auto realpath = Common::GetRealPath(path);
if (!realpath.has_value()) {
RETURN_STATUS_UNEXPECTED("Get real path failed, path=" + path);
}
std::fstream handle(realpath.value(), std::ios::in);
CHECK_FAIL_RETURN_UNEXPECTED(handle.good() && handle.is_open(), "from_file: fail to open:" + path);
std::string word;
while (std::getline(handle, word)) {

18
mindspore/ccsrc/minddata/mindrecord/io/shard_index_generator.cc
View File

@ -16,6 +16,8 @@
#include <thread>
#include "minddata/mindrecord/include/shard_index_generator.h"
#include "debug/common.h"
#include "utils/ms_utils.h"
using mindspore::LogStream;
@ -177,8 +179,14 @@ std::pair<MSRStatus, std::string> ShardIndexGenerator::GenerateFieldName(
}
std::pair<MSRStatus, sqlite3 *> ShardIndexGenerator::CheckDatabase(const std::string &shard_address) {
auto realpath = Common::GetRealPath(shard_address);
if (!realpath.has_value()) {
MS_LOG(ERROR) << "Get real path failed, path=" << shard_address;
return {FAILED, nullptr};
}
sqlite3 *db = nullptr;
std::ifstream fin(common::SafeCStr(shard_address));
std::ifstream fin(realpath.value());
if (!append_ && fin.good()) {
MS_LOG(ERROR) << "Invalid file, DB file already exist: " << shard_address;
fin.close();
@ -522,8 +530,14 @@ MSRStatus ShardIndexGenerator::ExecuteTransaction(const int &shard_no, std::pair
return FAILED;
}
auto realpath = Common::GetRealPath(shard_address);
if (!realpath.has_value()) {
MS_LOG(ERROR) << "Get real path failed, path=" << shard_address;
return FAILED;
}
std::fstream in;
in.open(common::SafeCStr(shard_address), std::ios::in | std::ios::binary);
in.open(realpath.value(), std::ios::in | std::ios::binary);
if (!in.good()) {
MS_LOG(ERROR) << "Invalid file, failed to open file: " << shard_address;
return FAILED;

38
mindspore/ccsrc/minddata/mindrecord/io/shard_reader.cc
View File

@ -14,11 +14,13 @@
* limitations under the License.
*/
#include "minddata/mindrecord/include/shard_reader.h"
#include <algorithm>
#include <thread>
#include "debug/common.h"
#include "minddata/mindrecord/include/shard_distributed_sample.h"
#include "minddata/mindrecord/include/shard_reader.h"
#include "utils/ms_utils.h"
using mindspore::LogStream;
@ -199,8 +201,14 @@ MSRStatus ShardReader::Open() {
file_streams_.clear();
for (const auto &file : file_paths_) {
auto realpath = Common::GetRealPath(file);
if (!realpath.has_value()) {
MS_LOG(ERROR) << "Get real path failed, path=" << file;
return FAILED;
}
std::shared_ptr<std::fstream> fs = std::make_shared<std::fstream>();
fs->open(common::SafeCStr(file), std::ios::in | std::ios::binary);
fs->open(realpath.value(), std::ios::in | std::ios::binary);
if (!fs->good()) {
MS_LOG(ERROR) << "Invalid file, failed to open file: " << file;
return FAILED;
@ -217,8 +225,14 @@ MSRStatus ShardReader::Open(int n_consumer) {
std::vector<std::vector<std::shared_ptr<std::fstream>>>(n_consumer, std::vector<std::shared_ptr<std::fstream>>());
for (const auto &file : file_paths_) {
for (int j = 0; j < n_consumer; ++j) {
auto realpath = Common::GetRealPath(file);
if (!realpath.has_value()) {
MS_LOG(ERROR) << "Get real path failed, path=" << file;
return FAILED;
}
std::shared_ptr<std::fstream> fs = std::make_shared<std::fstream>();
fs->open(common::SafeCStr(file), std::ios::in | std::ios::binary);
fs->open(realpath.value(), std::ios::in | std::ios::binary);
if (!fs->good()) {
MS_LOG(ERROR) << "Invalid file, failed to open file: " << file;
return FAILED;
@ -405,9 +419,16 @@ MSRStatus ShardReader::ReadAllRowsInShard(int shard_id, const std::string &sql,
MS_LOG(INFO) << "Get " << static_cast<int>(labels.size()) << " records from shard " << shard_id << " index.";
std::string file_name = file_paths_[shard_id];
auto realpath = Common::GetRealPath(file_name);
if (!realpath.has_value()) {
MS_LOG(ERROR) << "Get real path failed, path=" << file_name;
return FAILED;
}
std::shared_ptr<std::fstream> fs = std::make_shared<std::fstream>();
if (!all_in_index_) {
fs->open(common::SafeCStr(file_name), std::ios::in | std::ios::binary);
fs->open(realpath.value(), std::ios::in | std::ios::binary);
if (!fs->good()) {
MS_LOG(ERROR) << "Invalid file, failed to open file: " << file_name;
return FAILED;
@ -722,9 +743,16 @@ MSRStatus ShardReader::QueryWithCriteria(sqlite3 *db, const string &sql, const s
std::pair<MSRStatus, std::vector<json>> ShardReader::GetLabelsFromBinaryFile(
int shard_id, const std::vector<std::string> &columns, const std::vector<std::vector<std::string>> &label_offsets) {
std::string file_name = file_paths_[shard_id];
auto realpath = Common::GetRealPath(file_name);
if (!realpath.has_value()) {
MS_LOG(ERROR) << "Get real path failed, path=" << file_name;
return {FAILED, {}};
}
std::vector<json> res;
std::shared_ptr<std::fstream> fs = std::make_shared<std::fstream>();
fs->open(common::SafeCStr(file_name), std::ios::in | std::ios::binary);
fs->open(realpath.value(), std::ios::in | std::ios::binary);
if (!fs->good()) {
MS_LOG(ERROR) << "Invalid file, failed to open file: " << file_name;
return {FAILED, {}};

17
mindspore/ccsrc/minddata/mindrecord/io/shard_writer.cc
View File

@ -16,6 +16,7 @@
#include "minddata/dataset/util/random.h"
#include "minddata/mindrecord/include/shard_writer.h"
#include "debug/common.h"
#include "utils/ms_utils.h"
#include "minddata/mindrecord/include/common/shard_utils.h"
#include "./securec.h"
@ -77,10 +78,16 @@ MSRStatus ShardWriter::GetFullPathFromFileName(const std::vector<std::string> &p
MSRStatus ShardWriter::OpenDataFiles(bool append) {
// Open files
for (const auto &file : file_paths_) {
auto realpath = Common::GetRealPath(file);
if (!realpath.has_value()) {
MS_LOG(ERROR) << "Get real path failed, path=" << file;
return FAILED;
}
std::shared_ptr<std::fstream> fs = std::make_shared<std::fstream>();
if (!append) {
// if not append and mindrecord file exist, return FAILED
fs->open(common::SafeCStr(file), std::ios::in | std::ios::binary);
fs->open(realpath.value(), std::ios::in | std::ios::binary);
if (fs->good()) {
MS_LOG(ERROR) << "MindRecord file already existed, please delete file: " << common::SafeCStr(file);
fs->close();
@ -558,8 +565,14 @@ int ShardWriter::LockWriter(bool parallel_writer) {
// Open files
file_streams_.clear();
for (const auto &file : file_paths_) {
auto realpath = Common::GetRealPath(file);
if (!realpath.has_value()) {
MS_LOG(ERROR) << "Get real path failed, path=" << file;
return -1;
}
std::shared_ptr<std::fstream> fs = std::make_shared<std::fstream>();
fs->open(common::SafeCStr(file), std::ios::in | std::ios::out | std::ios::binary);
fs->open(realpath.value(), std::ios::in | std::ios::out | std::ios::binary);
if (fs->fail()) {
MS_LOG(ERROR) << "Invalid file, failed to open file: " << file;
return -1;

26
mindspore/ccsrc/minddata/mindrecord/meta/shard_header.cc
View File

@ -22,6 +22,7 @@
#include <utility>
#include <vector>
#include "debug/common.h"
#include "utils/ms_utils.h"
#include "minddata/mindrecord/include/shard_error.h"
#include "minddata/mindrecord/include/shard_page.h"
@ -67,7 +68,13 @@ MSRStatus ShardHeader::InitializeHeader(const std::vector<json> &headers, bool l
}
MSRStatus ShardHeader::CheckFileStatus(const std::string &path) {
std::ifstream fin(common::SafeCStr(path), std::ios::in | std::ios::binary);
auto realpath = Common::GetRealPath(path);
if (!realpath.has_value()) {
MS_LOG(ERROR) << "Get real path failed, path=" << path;
return FAILED;
}
std::ifstream fin(realpath.value(), std::ios::in | std::ios::binary);
if (!fin) {
MS_LOG(ERROR) << "File does not exist or permission denied. path: " << path;
return FAILED;
@ -700,8 +707,14 @@ std::pair<std::shared_ptr<Statistics>, MSRStatus> ShardHeader::GetStatisticByID(
}
MSRStatus ShardHeader::PagesToFile(const std::string dump_file_name) {
auto realpath = Common::GetRealPath(dump_file_name);
if (!realpath.has_value()) {
MS_LOG(ERROR) << "Get real path failed, path=" << dump_file_name;
return FAILED;
}
// write header content to file, dump whatever is in the file before
std::ofstream page_out_handle(dump_file_name.c_str(), std::ios_base::trunc | std::ios_base::out);
std::ofstream page_out_handle(realpath.value(), std::ios_base::trunc | std::ios_base::out);
if (page_out_handle.fail()) {
MS_LOG(ERROR) << "Failed in opening page file";
return FAILED;
@ -720,8 +733,15 @@ MSRStatus ShardHeader::FileToPages(const std::string dump_file_name) {
for (auto &v : pages_) { // clean pages
v.clear();
}
auto realpath = Common::GetRealPath(dump_file_name);
if (!realpath.has_value()) {
MS_LOG(ERROR) << "Get real path failed, path=" << dump_file_name;
return FAILED;
}
// attempt to open the file contains the page in json
std::ifstream page_in_handle(dump_file_name.c_str());
std::ifstream page_in_handle(realpath.value());
if (!page_in_handle.good()) {
MS_LOG(INFO) << "No page file exists.";

3
mindspore/dataset/datapreprocess/preprocess_imagenet_validate_dataset.py
View File

@ -36,7 +36,8 @@ def preprocess_imagenet_validation_dataset(train_dataset_path, validation_datase
if not os.path.exists(validate_sub_dir):
os.makedirs(validate_sub_dir)
mappings = [mapping.strip() for mapping in open(image_label_mapping_file).readlines()]
real_file_path = os.path.realpath(image_label_mapping_file)
mappings = [mapping.strip() for mapping in open(real_file_path).readlines()]
for mapping in mappings:
image_dir = mapping.split(':')
old_image_path = os.path.join(validation_dataset_path, image_dir[0])

3
mindspore/dataset/engine/serializer_deserializer.py
View File

@ -89,7 +89,8 @@ def deserialize(input_dict=None, json_filepath=None):
if json_filepath:
dict_pipeline = dict()
with open(json_filepath, 'r') as json_file:
real_file_path = os.path.realpath(json_filepath)
with open(real_file_path, 'r') as json_file:
dict_pipeline = json.load(json_file)
data = construct_pipeline(dict_pipeline)

6
mindspore/mindrecord/tools/cifar10.py
View File

@ -101,12 +101,14 @@ class Cifar10:
files = os.listdir(self.path)
for file in files:
if re.match("data_batch_*", file):
with open(os.path.join(self.path, file), 'rb') as f: # load train data
real_file_path = os.path.realpath(self.path)
with open(os.path.join(real_file_path, file), 'rb') as f: # load train data
dic = restricted_loads(f.read())
images = np.r_[images, dic[b"data"].reshape([-1, 3, 32, 32])]
labels.append(dic[b"labels"])
elif re.match("test_batch", file): # load test data
with open(os.path.join(self.path, file), 'rb') as f:
real_file_path = os.path.realpath(self.path)
with open(os.path.join(real_file_path, file), 'rb') as f:
dic = restricted_loads(f.read())
test_images = np.array(dic[b"data"].reshape([-1, 3, 32, 32]))
test_labels = np.array(dic[b"labels"])

6
mindspore/mindrecord/tools/cifar100.py
View File

@ -105,13 +105,15 @@ class Cifar100:
files = os.listdir(self.path)
for file in files:
if file == "train":
with open(os.path.join(self.path, file), 'rb') as f: # load train data
real_file_path = os.path.realpath(self.path)
with open(os.path.join(real_file_path, file), 'rb') as f: # load train data
dic = restricted_loads(f.read())
images = np.array(dic[b"data"].reshape([-1, 3, 32, 32]))
fine_labels.append(dic[b"fine_labels"])
coarse_labels.append(dic[b"coarse_labels"])
elif file == "test": # load test data
with open(os.path.join(self.path, file), 'rb') as f:
real_file_path = os.path.realpath(self.path)
with open(os.path.join(real_file_path, file), 'rb') as f:
dic = restricted_loads(f.read())
test_images = np.array(dic[b"data"].reshape([-1, 3, 32, 32]))
test_fine_labels = np.array(dic[b"fine_labels"])

8
mindspore/mindrecord/tools/imagenet_to_mr.py
View File

@ -74,11 +74,12 @@ class ImageNetToMR:
Yields:
data (dict of list): imagenet data list which contains dict.
"""
if not os.path.exists(self.map_file):
real_file_path = os.path.realpath(self.map_file)
if not os.path.exists(real_file_path):
raise IOError("map file {} not exists".format(self.map_file))
label_dict = {}
with open(self.map_file) as fp:
with open(real_file_path) as fp:
line = fp.readline()
while line:
labels = line.split(" ")
@ -109,7 +110,8 @@ class ImageNetToMR:
data["label"] = int(label)
# get the image data
image_file = open(file_name, "rb")
real_file_path = os.path.realpath(file_name)
image_file = open(real_file_path, "rb")
image_bytes = image_file.read()
image_file.close()
if not image_bytes:

6
mindspore/mindrecord/tools/mnist_to_mr.py
View File

@ -80,7 +80,8 @@ class MnistToMR:
def _extract_images(self, filename):
"""Extract the images into a 4D tensor [image index, y, x, channels]."""
with gzip.open(filename) as bytestream:
real_file_path = os.path.realpath(filename)
with gzip.open(real_file_path) as bytestream:
bytestream.read(16)
buf = bytestream.read()
data = np.frombuffer(buf, dtype=np.uint8)
@ -89,7 +90,8 @@ class MnistToMR:
def _extract_labels(self, filename):
"""Extract the labels into a vector of int64 label IDs."""
with gzip.open(filename) as bytestream:
real_file_path = os.path.realpath(filename)
with gzip.open(real_file_path) as bytestream:
bytestream.read(8)
buf = bytestream.read()
labels = np.frombuffer(buf, dtype=np.uint8).astype(np.int64)