forked from huawei/mindspore2022
make C++ API classes for samplers
This commit is contained in:
parent
a99fcf890a
commit
f73def5933
|
|
@ -1,5 +1,5 @@
|
|||
/**
|
||||
* Copyright 2020 Huawei Technologies Co., Ltd
|
||||
* Copyright 2020-2021 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
|
|
@ -285,7 +285,24 @@ std::shared_ptr<SchemaObj> Schema(const std::string &schema_file) {
|
|||
// Function to create a AlbumDataset.
|
||||
std::shared_ptr<AlbumDataset> Album(const std::string &dataset_dir, const std::string &data_schema,
|
||||
const std::vector<std::string> &column_names, bool decode,
|
||||
const std::shared_ptr<SamplerObj> &sampler,
|
||||
const std::shared_ptr<Sampler> &sampler,
|
||||
const std::shared_ptr<DatasetCache> &cache) {
|
||||
auto ds = std::make_shared<AlbumDataset>(dataset_dir, data_schema, column_names, decode, sampler, cache);
|
||||
|
||||
return ds;
|
||||
}
|
||||
// Function to create a AlbumDataset.
|
||||
std::shared_ptr<AlbumDataset> Album(const std::string &dataset_dir, const std::string &data_schema,
|
||||
const std::vector<std::string> &column_names, bool decode, Sampler *sampler,
|
||||
const std::shared_ptr<DatasetCache> &cache) {
|
||||
auto ds = std::make_shared<AlbumDataset>(dataset_dir, data_schema, column_names, decode, sampler, cache);
|
||||
|
||||
return ds;
|
||||
}
|
||||
// Function to create a AlbumDataset.
|
||||
std::shared_ptr<AlbumDataset> Album(const std::string &dataset_dir, const std::string &data_schema,
|
||||
const std::vector<std::string> &column_names, bool decode,
|
||||
const std::reference_wrapper<Sampler> sampler,
|
||||
const std::shared_ptr<DatasetCache> &cache) {
|
||||
auto ds = std::make_shared<AlbumDataset>(dataset_dir, data_schema, column_names, decode, sampler, cache);
|
||||
|
||||
|
|
@ -295,7 +312,24 @@ std::shared_ptr<AlbumDataset> Album(const std::string &dataset_dir, const std::s
|
|||
#ifndef ENABLE_ANDROID
|
||||
// Function to create a CelebADataset.
|
||||
std::shared_ptr<CelebADataset> CelebA(const std::string &dataset_dir, const std::string &usage,
|
||||
const std::shared_ptr<SamplerObj> &sampler, bool decode,
|
||||
const std::shared_ptr<Sampler> &sampler, bool decode,
|
||||
const std::set<std::string> &extensions,
|
||||
const std::shared_ptr<DatasetCache> &cache) {
|
||||
auto ds = std::make_shared<CelebADataset>(dataset_dir, usage, sampler, decode, extensions, cache);
|
||||
|
||||
return ds;
|
||||
}
|
||||
// Function to create a CelebADataset.
|
||||
std::shared_ptr<CelebADataset> CelebA(const std::string &dataset_dir, const std::string &usage, Sampler *sampler,
|
||||
bool decode, const std::set<std::string> &extensions,
|
||||
const std::shared_ptr<DatasetCache> &cache) {
|
||||
auto ds = std::make_shared<CelebADataset>(dataset_dir, usage, sampler, decode, extensions, cache);
|
||||
|
||||
return ds;
|
||||
}
|
||||
// Function to create a CelebADataset.
|
||||
std::shared_ptr<CelebADataset> CelebA(const std::string &dataset_dir, const std::string &usage,
|
||||
const std::reference_wrapper<Sampler> sampler, bool decode,
|
||||
const std::set<std::string> &extensions,
|
||||
const std::shared_ptr<DatasetCache> &cache) {
|
||||
auto ds = std::make_shared<CelebADataset>(dataset_dir, usage, sampler, decode, extensions, cache);
|
||||
|
|
@ -305,7 +339,22 @@ std::shared_ptr<CelebADataset> CelebA(const std::string &dataset_dir, const std:
|
|||
|
||||
// Function to create a Cifar10Dataset.
|
||||
std::shared_ptr<Cifar10Dataset> Cifar10(const std::string &dataset_dir, const std::string &usage,
|
||||
const std::shared_ptr<SamplerObj> &sampler,
|
||||
const std::shared_ptr<Sampler> &sampler,
|
||||
const std::shared_ptr<DatasetCache> &cache) {
|
||||
auto ds = std::make_shared<Cifar10Dataset>(dataset_dir, usage, sampler, cache);
|
||||
|
||||
return ds;
|
||||
}
|
||||
// Function to create a Cifar10Dataset.
|
||||
std::shared_ptr<Cifar10Dataset> Cifar10(const std::string &dataset_dir, const std::string &usage, Sampler *sampler,
|
||||
const std::shared_ptr<DatasetCache> &cache) {
|
||||
auto ds = std::make_shared<Cifar10Dataset>(dataset_dir, usage, sampler, cache);
|
||||
|
||||
return ds;
|
||||
}
|
||||
// Function to create a Cifar10Dataset.
|
||||
std::shared_ptr<Cifar10Dataset> Cifar10(const std::string &dataset_dir, const std::string &usage,
|
||||
const std::reference_wrapper<Sampler> sampler,
|
||||
const std::shared_ptr<DatasetCache> &cache) {
|
||||
auto ds = std::make_shared<Cifar10Dataset>(dataset_dir, usage, sampler, cache);
|
||||
|
||||
|
|
@ -314,7 +363,22 @@ std::shared_ptr<Cifar10Dataset> Cifar10(const std::string &dataset_dir, const st
|
|||
|
||||
// Function to create a Cifar100Dataset.
|
||||
std::shared_ptr<Cifar100Dataset> Cifar100(const std::string &dataset_dir, const std::string &usage,
|
||||
const std::shared_ptr<SamplerObj> &sampler,
|
||||
const std::shared_ptr<Sampler> &sampler,
|
||||
const std::shared_ptr<DatasetCache> &cache) {
|
||||
auto ds = std::make_shared<Cifar100Dataset>(dataset_dir, usage, sampler, cache);
|
||||
|
||||
return ds;
|
||||
}
|
||||
// Function to create a Cifar100Dataset.
|
||||
std::shared_ptr<Cifar100Dataset> Cifar100(const std::string &dataset_dir, const std::string &usage, Sampler *sampler,
|
||||
const std::shared_ptr<DatasetCache> &cache) {
|
||||
auto ds = std::make_shared<Cifar100Dataset>(dataset_dir, usage, sampler, cache);
|
||||
|
||||
return ds;
|
||||
}
|
||||
// Function to create a Cifar100Dataset.
|
||||
std::shared_ptr<Cifar100Dataset> Cifar100(const std::string &dataset_dir, const std::string &usage,
|
||||
const std::reference_wrapper<Sampler> sampler,
|
||||
const std::shared_ptr<DatasetCache> &cache) {
|
||||
auto ds = std::make_shared<Cifar100Dataset>(dataset_dir, usage, sampler, cache);
|
||||
|
||||
|
|
@ -330,10 +394,26 @@ std::shared_ptr<CLUEDataset> CLUE(const std::vector<std::string> &clue_files, co
|
|||
return ds;
|
||||
}
|
||||
|
||||
// Function to create a CocoDataset.
|
||||
std::shared_ptr<CocoDataset> Coco(const std::string &dataset_dir, const std::string &annotation_file,
|
||||
const std::string &task, const bool &decode, const std::shared_ptr<Sampler> &sampler,
|
||||
const std::shared_ptr<DatasetCache> &cache) {
|
||||
auto ds = std::make_shared<CocoDataset>(dataset_dir, annotation_file, task, decode, sampler, cache);
|
||||
|
||||
return ds;
|
||||
}
|
||||
// Function to create a CocoDataset.
|
||||
std::shared_ptr<CocoDataset> Coco(const std::string &dataset_dir, const std::string &annotation_file,
|
||||
const std::string &task, const bool &decode, Sampler *sampler,
|
||||
const std::shared_ptr<DatasetCache> &cache) {
|
||||
auto ds = std::make_shared<CocoDataset>(dataset_dir, annotation_file, task, decode, sampler, cache);
|
||||
|
||||
return ds;
|
||||
}
|
||||
// Function to create a CocoDataset.
|
||||
std::shared_ptr<CocoDataset> Coco(const std::string &dataset_dir, const std::string &annotation_file,
|
||||
const std::string &task, const bool &decode,
|
||||
const std::shared_ptr<SamplerObj> &sampler,
|
||||
const std::reference_wrapper<Sampler> sampler,
|
||||
const std::shared_ptr<DatasetCache> &cache) {
|
||||
auto ds = std::make_shared<CocoDataset>(dataset_dir, annotation_file, task, decode, sampler, cache);
|
||||
|
||||
|
|
@ -353,7 +433,26 @@ std::shared_ptr<CSVDataset> CSV(const std::vector<std::string> &dataset_files, c
|
|||
|
||||
// Function to create a ImageFolderDataset.
|
||||
std::shared_ptr<ImageFolderDataset> ImageFolder(const std::string &dataset_dir, bool decode,
|
||||
const std::shared_ptr<SamplerObj> &sampler,
|
||||
const std::shared_ptr<Sampler> &sampler,
|
||||
const std::set<std::string> &extensions,
|
||||
const std::map<std::string, int32_t> &class_indexing,
|
||||
const std::shared_ptr<DatasetCache> &cache) {
|
||||
auto ds = std::make_shared<ImageFolderDataset>(dataset_dir, decode, sampler, extensions, class_indexing, cache);
|
||||
|
||||
return ds;
|
||||
}
|
||||
// Function to create a ImageFolderDataset.
|
||||
std::shared_ptr<ImageFolderDataset> ImageFolder(const std::string &dataset_dir, bool decode, Sampler *sampler,
|
||||
const std::set<std::string> &extensions,
|
||||
const std::map<std::string, int32_t> &class_indexing,
|
||||
const std::shared_ptr<DatasetCache> &cache) {
|
||||
auto ds = std::make_shared<ImageFolderDataset>(dataset_dir, decode, sampler, extensions, class_indexing, cache);
|
||||
|
||||
return ds;
|
||||
}
|
||||
// Function to create a ImageFolderDataset.
|
||||
std::shared_ptr<ImageFolderDataset> ImageFolder(const std::string &dataset_dir, bool decode,
|
||||
const std::reference_wrapper<Sampler> sampler,
|
||||
const std::set<std::string> &extensions,
|
||||
const std::map<std::string, int32_t> &class_indexing,
|
||||
const std::shared_ptr<DatasetCache> &cache) {
|
||||
|
|
@ -364,7 +463,24 @@ std::shared_ptr<ImageFolderDataset> ImageFolder(const std::string &dataset_dir,
|
|||
|
||||
// Function to create a ManifestDataset.
|
||||
std::shared_ptr<ManifestDataset> Manifest(const std::string &dataset_file, const std::string &usage,
|
||||
const std::shared_ptr<SamplerObj> &sampler,
|
||||
const std::shared_ptr<Sampler> &sampler,
|
||||
const std::map<std::string, int32_t> &class_indexing, bool decode,
|
||||
const std::shared_ptr<DatasetCache> &cache) {
|
||||
auto ds = std::make_shared<ManifestDataset>(dataset_file, usage, sampler, class_indexing, decode, cache);
|
||||
|
||||
return ds;
|
||||
}
|
||||
// Function to create a ManifestDataset.
|
||||
std::shared_ptr<ManifestDataset> Manifest(const std::string &dataset_file, const std::string &usage, Sampler *sampler,
|
||||
const std::map<std::string, int32_t> &class_indexing, bool decode,
|
||||
const std::shared_ptr<DatasetCache> &cache) {
|
||||
auto ds = std::make_shared<ManifestDataset>(dataset_file, usage, sampler, class_indexing, decode, cache);
|
||||
|
||||
return ds;
|
||||
}
|
||||
// Function to create a ManifestDataset.
|
||||
std::shared_ptr<ManifestDataset> Manifest(const std::string &dataset_file, const std::string &usage,
|
||||
const std::reference_wrapper<Sampler> sampler,
|
||||
const std::map<std::string, int32_t> &class_indexing, bool decode,
|
||||
const std::shared_ptr<DatasetCache> &cache) {
|
||||
auto ds = std::make_shared<ManifestDataset>(dataset_file, usage, sampler, class_indexing, decode, cache);
|
||||
|
|
@ -374,7 +490,22 @@ std::shared_ptr<ManifestDataset> Manifest(const std::string &dataset_file, const
|
|||
|
||||
// Function to create a MindDataDataset.
|
||||
std::shared_ptr<MindDataDataset> MindData(const std::string &dataset_file, const std::vector<std::string> &columns_list,
|
||||
const std::shared_ptr<SamplerObj> &sampler, nlohmann::json padded_sample,
|
||||
const std::shared_ptr<Sampler> &sampler, nlohmann::json padded_sample,
|
||||
int64_t num_padded) {
|
||||
auto ds = std::make_shared<MindDataDataset>(dataset_file, columns_list, sampler, padded_sample, num_padded);
|
||||
|
||||
return ds;
|
||||
}
|
||||
// Function to create a MindDataDataset.
|
||||
std::shared_ptr<MindDataDataset> MindData(const std::string &dataset_file, const std::vector<std::string> &columns_list,
|
||||
Sampler *sampler, nlohmann::json padded_sample, int64_t num_padded) {
|
||||
auto ds = std::make_shared<MindDataDataset>(dataset_file, columns_list, sampler, padded_sample, num_padded);
|
||||
|
||||
return ds;
|
||||
}
|
||||
// Function to create a MindDataDataset.
|
||||
std::shared_ptr<MindDataDataset> MindData(const std::string &dataset_file, const std::vector<std::string> &columns_list,
|
||||
const std::reference_wrapper<Sampler> sampler, nlohmann::json padded_sample,
|
||||
int64_t num_padded) {
|
||||
auto ds = std::make_shared<MindDataDataset>(dataset_file, columns_list, sampler, padded_sample, num_padded);
|
||||
|
||||
|
|
@ -384,7 +515,24 @@ std::shared_ptr<MindDataDataset> MindData(const std::string &dataset_file, const
|
|||
// Function to create a MindDataDataset.
|
||||
std::shared_ptr<MindDataDataset> MindData(const std::vector<std::string> &dataset_files,
|
||||
const std::vector<std::string> &columns_list,
|
||||
const std::shared_ptr<SamplerObj> &sampler, nlohmann::json padded_sample,
|
||||
const std::shared_ptr<Sampler> &sampler, nlohmann::json padded_sample,
|
||||
int64_t num_padded) {
|
||||
auto ds = std::make_shared<MindDataDataset>(dataset_files, columns_list, sampler, padded_sample, num_padded);
|
||||
|
||||
return ds;
|
||||
}
|
||||
// Function to create a MindDataDataset.
|
||||
std::shared_ptr<MindDataDataset> MindData(const std::vector<std::string> &dataset_files,
|
||||
const std::vector<std::string> &columns_list, Sampler *sampler,
|
||||
nlohmann::json padded_sample, int64_t num_padded) {
|
||||
auto ds = std::make_shared<MindDataDataset>(dataset_files, columns_list, sampler, padded_sample, num_padded);
|
||||
|
||||
return ds;
|
||||
}
|
||||
// Function to create a MindDataDataset.
|
||||
std::shared_ptr<MindDataDataset> MindData(const std::vector<std::string> &dataset_files,
|
||||
const std::vector<std::string> &columns_list,
|
||||
const std::reference_wrapper<Sampler> sampler, nlohmann::json padded_sample,
|
||||
int64_t num_padded) {
|
||||
auto ds = std::make_shared<MindDataDataset>(dataset_files, columns_list, sampler, padded_sample, num_padded);
|
||||
|
||||
|
|
@ -393,7 +541,22 @@ std::shared_ptr<MindDataDataset> MindData(const std::vector<std::string> &datase
|
|||
#endif
|
||||
// Function to create a MnistDataset.
|
||||
std::shared_ptr<MnistDataset> Mnist(const std::string &dataset_dir, const std::string &usage,
|
||||
const std::shared_ptr<SamplerObj> &sampler,
|
||||
const std::shared_ptr<Sampler> &sampler,
|
||||
const std::shared_ptr<DatasetCache> &cache) {
|
||||
auto ds = std::make_shared<MnistDataset>(dataset_dir, usage, sampler, cache);
|
||||
|
||||
return ds;
|
||||
}
|
||||
// Function to create a MnistDataset.
|
||||
std::shared_ptr<MnistDataset> Mnist(const std::string &dataset_dir, const std::string &usage, Sampler *sampler,
|
||||
const std::shared_ptr<DatasetCache> &cache) {
|
||||
auto ds = std::make_shared<MnistDataset>(dataset_dir, usage, sampler, cache);
|
||||
|
||||
return ds;
|
||||
}
|
||||
// Function to create a MnistDataset.
|
||||
std::shared_ptr<MnistDataset> Mnist(const std::string &dataset_dir, const std::string &usage,
|
||||
const std::reference_wrapper<Sampler> sampler,
|
||||
const std::shared_ptr<DatasetCache> &cache) {
|
||||
auto ds = std::make_shared<MnistDataset>(dataset_dir, usage, sampler, cache);
|
||||
|
||||
|
|
@ -419,7 +582,23 @@ std::shared_ptr<TextFileDataset> TextFile(const std::vector<std::string> &datase
|
|||
// Function to create a VOCDataset.
|
||||
std::shared_ptr<VOCDataset> VOC(const std::string &dataset_dir, const std::string &task, const std::string &usage,
|
||||
const std::map<std::string, int32_t> &class_indexing, bool decode,
|
||||
const std::shared_ptr<SamplerObj> &sampler,
|
||||
const std::shared_ptr<Sampler> &sampler, const std::shared_ptr<DatasetCache> &cache) {
|
||||
auto ds = std::make_shared<VOCDataset>(dataset_dir, task, usage, class_indexing, decode, sampler, cache);
|
||||
|
||||
return ds;
|
||||
}
|
||||
// Function to create a VOCDataset.
|
||||
std::shared_ptr<VOCDataset> VOC(const std::string &dataset_dir, const std::string &task, const std::string &usage,
|
||||
const std::map<std::string, int32_t> &class_indexing, bool decode, Sampler *sampler,
|
||||
const std::shared_ptr<DatasetCache> &cache) {
|
||||
auto ds = std::make_shared<VOCDataset>(dataset_dir, task, usage, class_indexing, decode, sampler, cache);
|
||||
|
||||
return ds;
|
||||
}
|
||||
// Function to create a VOCDataset.
|
||||
std::shared_ptr<VOCDataset> VOC(const std::string &dataset_dir, const std::string &task, const std::string &usage,
|
||||
const std::map<std::string, int32_t> &class_indexing, bool decode,
|
||||
const std::reference_wrapper<Sampler> sampler,
|
||||
const std::shared_ptr<DatasetCache> &cache) {
|
||||
auto ds = std::make_shared<VOCDataset>(dataset_dir, task, usage, class_indexing, decode, sampler, cache);
|
||||
|
||||
|
|
@ -831,41 +1010,117 @@ std::shared_ptr<DatasetCache> CreateDatasetCache(session_id_type id, uint64_t me
|
|||
|
||||
AlbumDataset::AlbumDataset(const std::string &dataset_dir, const std::string &data_schema,
|
||||
const std::vector<std::string> &column_names, bool decode,
|
||||
const std::shared_ptr<SamplerObj> &sampler, const std::shared_ptr<DatasetCache> &cache) {
|
||||
auto ds = std::make_shared<AlbumNode>(dataset_dir, data_schema, column_names, decode, sampler, cache);
|
||||
const std::shared_ptr<Sampler> &sampler, const std::shared_ptr<DatasetCache> &cache) {
|
||||
auto sampler_obj = sampler ? sampler->Parse() : nullptr;
|
||||
auto ds = std::make_shared<AlbumNode>(dataset_dir, data_schema, column_names, decode, sampler_obj, cache);
|
||||
ir_node_ = std::static_pointer_cast<DatasetNode>(ds);
|
||||
}
|
||||
|
||||
AlbumDataset::AlbumDataset(const std::string &dataset_dir, const std::string &data_schema,
|
||||
const std::vector<std::string> &column_names, bool decode, Sampler *sampler,
|
||||
const std::shared_ptr<DatasetCache> &cache) {
|
||||
auto sampler_obj = sampler ? sampler->Parse() : nullptr;
|
||||
auto ds = std::make_shared<AlbumNode>(dataset_dir, data_schema, column_names, decode, sampler_obj, cache);
|
||||
ir_node_ = std::static_pointer_cast<DatasetNode>(ds);
|
||||
}
|
||||
AlbumDataset::AlbumDataset(const std::string &dataset_dir, const std::string &data_schema,
|
||||
const std::vector<std::string> &column_names, bool decode,
|
||||
const std::reference_wrapper<Sampler> sampler, const std::shared_ptr<DatasetCache> &cache) {
|
||||
auto sampler_obj = sampler.get().Parse();
|
||||
auto ds = std::make_shared<AlbumNode>(dataset_dir, data_schema, column_names, decode, sampler_obj, cache);
|
||||
ir_node_ = std::static_pointer_cast<DatasetNode>(ds);
|
||||
}
|
||||
|
||||
#ifndef ENABLE_ANDROID
|
||||
CelebADataset::CelebADataset(const std::string &dataset_dir, const std::string &usage,
|
||||
const std::shared_ptr<SamplerObj> &sampler, bool decode,
|
||||
const std::shared_ptr<Sampler> &sampler, bool decode,
|
||||
const std::set<std::string> &extensions, const std::shared_ptr<DatasetCache> &cache) {
|
||||
auto ds = std::make_shared<CelebANode>(dataset_dir, usage, sampler, decode, extensions, cache);
|
||||
auto sampler_obj = sampler ? sampler->Parse() : nullptr;
|
||||
auto ds = std::make_shared<CelebANode>(dataset_dir, usage, sampler_obj, decode, extensions, cache);
|
||||
ir_node_ = std::static_pointer_cast<DatasetNode>(ds);
|
||||
}
|
||||
CelebADataset::CelebADataset(const std::string &dataset_dir, const std::string &usage, Sampler *sampler, bool decode,
|
||||
const std::set<std::string> &extensions, const std::shared_ptr<DatasetCache> &cache) {
|
||||
auto sampler_obj = sampler ? sampler->Parse() : nullptr;
|
||||
auto ds = std::make_shared<CelebANode>(dataset_dir, usage, sampler_obj, decode, extensions, cache);
|
||||
ir_node_ = std::static_pointer_cast<DatasetNode>(ds);
|
||||
}
|
||||
CelebADataset::CelebADataset(const std::string &dataset_dir, const std::string &usage,
|
||||
const std::reference_wrapper<Sampler> sampler, bool decode,
|
||||
const std::set<std::string> &extensions, const std::shared_ptr<DatasetCache> &cache) {
|
||||
auto sampler_obj = sampler.get().Parse();
|
||||
auto ds = std::make_shared<CelebANode>(dataset_dir, usage, sampler_obj, decode, extensions, cache);
|
||||
ir_node_ = std::static_pointer_cast<DatasetNode>(ds);
|
||||
}
|
||||
|
||||
Cifar10Dataset::Cifar10Dataset(const std::string &dataset_dir, const std::string &usage,
|
||||
const std::shared_ptr<Sampler> &sampler, const std::shared_ptr<DatasetCache> &cache) {
|
||||
auto sampler_obj = sampler ? sampler->Parse() : nullptr;
|
||||
auto ds = std::make_shared<Cifar10Node>(dataset_dir, usage, sampler_obj, cache);
|
||||
ir_node_ = std::static_pointer_cast<DatasetNode>(ds);
|
||||
}
|
||||
Cifar10Dataset::Cifar10Dataset(const std::string &dataset_dir, const std::string &usage, Sampler *sampler,
|
||||
const std::shared_ptr<DatasetCache> &cache) {
|
||||
auto sampler_obj = sampler ? sampler->Parse() : nullptr;
|
||||
auto ds = std::make_shared<Cifar10Node>(dataset_dir, usage, sampler_obj, cache);
|
||||
ir_node_ = std::static_pointer_cast<DatasetNode>(ds);
|
||||
}
|
||||
Cifar10Dataset::Cifar10Dataset(const std::string &dataset_dir, const std::string &usage,
|
||||
const std::shared_ptr<SamplerObj> &sampler, const std::shared_ptr<DatasetCache> &cache) {
|
||||
auto ds = std::make_shared<Cifar10Node>(dataset_dir, usage, sampler, cache);
|
||||
const std::reference_wrapper<Sampler> sampler,
|
||||
const std::shared_ptr<DatasetCache> &cache) {
|
||||
auto sampler_obj = sampler.get().Parse();
|
||||
auto ds = std::make_shared<Cifar10Node>(dataset_dir, usage, sampler_obj, cache);
|
||||
ir_node_ = std::static_pointer_cast<DatasetNode>(ds);
|
||||
}
|
||||
|
||||
Cifar100Dataset::Cifar100Dataset(const std::string &dataset_dir, const std::string &usage,
|
||||
const std::shared_ptr<Sampler> &sampler, const std::shared_ptr<DatasetCache> &cache) {
|
||||
auto sampler_obj = sampler ? sampler->Parse() : nullptr;
|
||||
auto ds = std::make_shared<Cifar100Node>(dataset_dir, usage, sampler_obj, cache);
|
||||
ir_node_ = std::static_pointer_cast<DatasetNode>(ds);
|
||||
}
|
||||
Cifar100Dataset::Cifar100Dataset(const std::string &dataset_dir, const std::string &usage, Sampler *sampler,
|
||||
const std::shared_ptr<DatasetCache> &cache) {
|
||||
auto sampler_obj = sampler ? sampler->Parse() : nullptr;
|
||||
auto ds = std::make_shared<Cifar100Node>(dataset_dir, usage, sampler_obj, cache);
|
||||
ir_node_ = std::static_pointer_cast<DatasetNode>(ds);
|
||||
}
|
||||
Cifar100Dataset::Cifar100Dataset(const std::string &dataset_dir, const std::string &usage,
|
||||
const std::shared_ptr<SamplerObj> &sampler,
|
||||
const std::reference_wrapper<Sampler> sampler,
|
||||
const std::shared_ptr<DatasetCache> &cache) {
|
||||
auto ds = std::make_shared<Cifar100Node>(dataset_dir, usage, sampler, cache);
|
||||
auto sampler_obj = sampler.get().Parse();
|
||||
auto ds = std::make_shared<Cifar100Node>(dataset_dir, usage, sampler_obj, cache);
|
||||
ir_node_ = std::static_pointer_cast<DatasetNode>(ds);
|
||||
}
|
||||
|
||||
CLUEDataset::CLUEDataset(const std::vector<std::string> &dataset_files, const std::string &task,
|
||||
const std::string &usage, int64_t num_samples, ShuffleMode shuffle, int32_t num_shards,
|
||||
int32_t shard_id, const std::shared_ptr<DatasetCache> &cache) {
|
||||
auto ds = std::make_shared<CLUENode>(dataset_files, task, usage, num_samples, shuffle, num_shards, shard_id, cache);
|
||||
ir_node_ = std::static_pointer_cast<DatasetNode>(ds);
|
||||
}
|
||||
|
||||
CocoDataset::CocoDataset(const std::string &dataset_dir, const std::string &annotation_file, const std::string &task,
|
||||
const bool &decode, const std::shared_ptr<SamplerObj> &sampler,
|
||||
const bool &decode, const std::shared_ptr<Sampler> &sampler,
|
||||
const std::shared_ptr<DatasetCache> &cache) {
|
||||
auto ds = std::make_shared<CocoNode>(dataset_dir, annotation_file, task, decode, sampler, cache);
|
||||
auto sampler_obj = sampler ? sampler->Parse() : nullptr;
|
||||
auto ds = std::make_shared<CocoNode>(dataset_dir, annotation_file, task, decode, sampler_obj, cache);
|
||||
ir_node_ = std::static_pointer_cast<DatasetNode>(ds);
|
||||
}
|
||||
CocoDataset::CocoDataset(const std::string &dataset_dir, const std::string &annotation_file, const std::string &task,
|
||||
const bool &decode, Sampler *sampler, const std::shared_ptr<DatasetCache> &cache) {
|
||||
auto sampler_obj = sampler ? sampler->Parse() : nullptr;
|
||||
auto ds = std::make_shared<CocoNode>(dataset_dir, annotation_file, task, decode, sampler_obj, cache);
|
||||
ir_node_ = std::static_pointer_cast<DatasetNode>(ds);
|
||||
}
|
||||
CocoDataset::CocoDataset(const std::string &dataset_dir, const std::string &annotation_file, const std::string &task,
|
||||
const bool &decode, const std::reference_wrapper<Sampler> sampler,
|
||||
const std::shared_ptr<DatasetCache> &cache) {
|
||||
auto sampler_obj = sampler.get().Parse();
|
||||
auto ds = std::make_shared<CocoNode>(dataset_dir, annotation_file, task, decode, sampler_obj, cache);
|
||||
ir_node_ = std::static_pointer_cast<DatasetNode>(ds);
|
||||
}
|
||||
|
||||
CSVDataset::CSVDataset(const std::vector<std::string> &dataset_files, char field_delim,
|
||||
const std::vector<std::shared_ptr<CsvBase>> &column_defaults,
|
||||
const std::vector<std::string> &column_names, int64_t num_samples, ShuffleMode shuffle,
|
||||
|
|
@ -874,8 +1129,21 @@ CSVDataset::CSVDataset(const std::vector<std::string> &dataset_files, char field
|
|||
num_shards, shard_id, cache);
|
||||
ir_node_ = std::static_pointer_cast<DatasetNode>(ds);
|
||||
}
|
||||
|
||||
ImageFolderDataset::ImageFolderDataset(const std::string &dataset_dir, bool decode,
|
||||
const std::shared_ptr<SamplerObj> &sampler,
|
||||
const std::shared_ptr<Sampler> &sampler, const std::set<std::string> &extensions,
|
||||
const std::map<std::string, int32_t> &class_indexing,
|
||||
const std::shared_ptr<DatasetCache> &cache) {
|
||||
// This arg exists in ImageFolderOp, but not externalized (in Python API). The default value is false.
|
||||
bool recursive = false;
|
||||
|
||||
// Create logical representation of ImageFolderDataset.
|
||||
auto sampler_obj = sampler ? sampler->Parse() : nullptr;
|
||||
auto ds =
|
||||
std::make_shared<ImageFolderNode>(dataset_dir, decode, sampler_obj, recursive, extensions, class_indexing, cache);
|
||||
ir_node_ = std::static_pointer_cast<DatasetNode>(ds);
|
||||
}
|
||||
ImageFolderDataset::ImageFolderDataset(const std::string &dataset_dir, bool decode, Sampler *sampler,
|
||||
const std::set<std::string> &extensions,
|
||||
const std::map<std::string, int32_t> &class_indexing,
|
||||
const std::shared_ptr<DatasetCache> &cache) {
|
||||
|
|
@ -883,36 +1151,110 @@ ImageFolderDataset::ImageFolderDataset(const std::string &dataset_dir, bool deco
|
|||
bool recursive = false;
|
||||
|
||||
// Create logical representation of ImageFolderDataset.
|
||||
auto sampler_obj = sampler ? sampler->Parse() : nullptr;
|
||||
auto ds =
|
||||
std::make_shared<ImageFolderNode>(dataset_dir, decode, sampler, recursive, extensions, class_indexing, cache);
|
||||
std::make_shared<ImageFolderNode>(dataset_dir, decode, sampler_obj, recursive, extensions, class_indexing, cache);
|
||||
ir_node_ = std::static_pointer_cast<DatasetNode>(ds);
|
||||
}
|
||||
ImageFolderDataset::ImageFolderDataset(const std::string &dataset_dir, bool decode,
|
||||
const std::reference_wrapper<Sampler> sampler,
|
||||
const std::set<std::string> &extensions,
|
||||
const std::map<std::string, int32_t> &class_indexing,
|
||||
const std::shared_ptr<DatasetCache> &cache) {
|
||||
// This arg exists in ImageFolderOp, but not externalized (in Python API). The default value is false.
|
||||
bool recursive = false;
|
||||
|
||||
// Create logical representation of ImageFolderDataset.
|
||||
auto sampler_obj = sampler.get().Parse();
|
||||
auto ds =
|
||||
std::make_shared<ImageFolderNode>(dataset_dir, decode, sampler_obj, recursive, extensions, class_indexing, cache);
|
||||
ir_node_ = std::static_pointer_cast<DatasetNode>(ds);
|
||||
}
|
||||
|
||||
ManifestDataset::ManifestDataset(const std::string &dataset_file, const std::string &usage,
|
||||
const std::shared_ptr<SamplerObj> &sampler,
|
||||
const std::shared_ptr<Sampler> &sampler,
|
||||
const std::map<std::string, int32_t> &class_indexing, bool decode,
|
||||
const std::shared_ptr<DatasetCache> &cache) {
|
||||
auto ds = std::make_shared<ManifestNode>(dataset_file, usage, sampler, class_indexing, decode, cache);
|
||||
auto sampler_obj = sampler ? sampler->Parse() : nullptr;
|
||||
auto ds = std::make_shared<ManifestNode>(dataset_file, usage, sampler_obj, class_indexing, decode, cache);
|
||||
ir_node_ = std::static_pointer_cast<DatasetNode>(ds);
|
||||
}
|
||||
ManifestDataset::ManifestDataset(const std::string &dataset_file, const std::string &usage, Sampler *sampler,
|
||||
const std::map<std::string, int32_t> &class_indexing, bool decode,
|
||||
const std::shared_ptr<DatasetCache> &cache) {
|
||||
auto sampler_obj = sampler ? sampler->Parse() : nullptr;
|
||||
auto ds = std::make_shared<ManifestNode>(dataset_file, usage, sampler_obj, class_indexing, decode, cache);
|
||||
ir_node_ = std::static_pointer_cast<DatasetNode>(ds);
|
||||
}
|
||||
ManifestDataset::ManifestDataset(const std::string &dataset_file, const std::string &usage,
|
||||
const std::reference_wrapper<Sampler> sampler,
|
||||
const std::map<std::string, int32_t> &class_indexing, bool decode,
|
||||
const std::shared_ptr<DatasetCache> &cache) {
|
||||
auto sampler_obj = sampler.get().Parse();
|
||||
auto ds = std::make_shared<ManifestNode>(dataset_file, usage, sampler_obj, class_indexing, decode, cache);
|
||||
ir_node_ = std::static_pointer_cast<DatasetNode>(ds);
|
||||
}
|
||||
|
||||
MindDataDataset::MindDataDataset(const std::string &dataset_file, const std::vector<std::string> &columns_list,
|
||||
const std::shared_ptr<Sampler> &sampler, nlohmann::json padded_sample,
|
||||
int64_t num_padded) {
|
||||
auto sampler_obj = sampler ? sampler->Parse() : nullptr;
|
||||
auto ds = std::make_shared<MindDataNode>(dataset_file, columns_list, sampler_obj, padded_sample, num_padded);
|
||||
ir_node_ = std::static_pointer_cast<DatasetNode>(ds);
|
||||
}
|
||||
MindDataDataset::MindDataDataset(const std::string &dataset_file, const std::vector<std::string> &columns_list,
|
||||
const std::shared_ptr<SamplerObj> &sampler, nlohmann::json padded_sample,
|
||||
Sampler *sampler, nlohmann::json padded_sample, int64_t num_padded) {
|
||||
auto sampler_obj = sampler ? sampler->Parse() : nullptr;
|
||||
auto ds = std::make_shared<MindDataNode>(dataset_file, columns_list, sampler_obj, padded_sample, num_padded);
|
||||
ir_node_ = std::static_pointer_cast<DatasetNode>(ds);
|
||||
}
|
||||
MindDataDataset::MindDataDataset(const std::string &dataset_file, const std::vector<std::string> &columns_list,
|
||||
const std::reference_wrapper<Sampler> sampler, nlohmann::json padded_sample,
|
||||
int64_t num_padded) {
|
||||
auto ds = std::make_shared<MindDataNode>(dataset_file, columns_list, sampler, padded_sample, num_padded);
|
||||
auto sampler_obj = sampler.get().Parse();
|
||||
auto ds = std::make_shared<MindDataNode>(dataset_file, columns_list, sampler_obj, padded_sample, num_padded);
|
||||
ir_node_ = std::static_pointer_cast<DatasetNode>(ds);
|
||||
}
|
||||
MindDataDataset::MindDataDataset(const std::vector<std::string> &dataset_files,
|
||||
const std::vector<std::string> &columns_list, const std::shared_ptr<Sampler> &sampler,
|
||||
nlohmann::json padded_sample, int64_t num_padded) {
|
||||
auto sampler_obj = sampler ? sampler->Parse() : nullptr;
|
||||
auto ds = std::make_shared<MindDataNode>(dataset_files, columns_list, sampler_obj, padded_sample, num_padded);
|
||||
ir_node_ = std::static_pointer_cast<DatasetNode>(ds);
|
||||
}
|
||||
MindDataDataset::MindDataDataset(const std::vector<std::string> &dataset_files,
|
||||
const std::vector<std::string> &columns_list, Sampler *sampler,
|
||||
nlohmann::json padded_sample, int64_t num_padded) {
|
||||
auto sampler_obj = sampler ? sampler->Parse() : nullptr;
|
||||
auto ds = std::make_shared<MindDataNode>(dataset_files, columns_list, sampler_obj, padded_sample, num_padded);
|
||||
ir_node_ = std::static_pointer_cast<DatasetNode>(ds);
|
||||
}
|
||||
MindDataDataset::MindDataDataset(const std::vector<std::string> &dataset_files,
|
||||
const std::vector<std::string> &columns_list,
|
||||
const std::shared_ptr<SamplerObj> &sampler, nlohmann::json padded_sample,
|
||||
const std::reference_wrapper<Sampler> sampler, nlohmann::json padded_sample,
|
||||
int64_t num_padded) {
|
||||
auto ds = std::make_shared<MindDataNode>(dataset_files, columns_list, sampler, padded_sample, num_padded);
|
||||
auto sampler_obj = sampler.get().Parse();
|
||||
auto ds = std::make_shared<MindDataNode>(dataset_files, columns_list, sampler_obj, padded_sample, num_padded);
|
||||
ir_node_ = std::static_pointer_cast<DatasetNode>(ds);
|
||||
}
|
||||
#endif
|
||||
|
||||
MnistDataset::MnistDataset(const std::string &dataset_dir, const std::string &usage,
|
||||
const std::shared_ptr<SamplerObj> &sampler, const std::shared_ptr<DatasetCache> &cache) {
|
||||
auto ds = std::make_shared<MnistNode>(dataset_dir, usage, sampler, cache);
|
||||
const std::shared_ptr<Sampler> &sampler, const std::shared_ptr<DatasetCache> &cache) {
|
||||
auto sampler_obj = sampler ? sampler->Parse() : nullptr;
|
||||
auto ds = std::make_shared<MnistNode>(dataset_dir, usage, sampler_obj, cache);
|
||||
ir_node_ = std::static_pointer_cast<DatasetNode>(ds);
|
||||
}
|
||||
MnistDataset::MnistDataset(const std::string &dataset_dir, const std::string &usage, Sampler *sampler,
|
||||
const std::shared_ptr<DatasetCache> &cache) {
|
||||
auto sampler_obj = sampler ? sampler->Parse() : nullptr;
|
||||
auto ds = std::make_shared<MnistNode>(dataset_dir, usage, sampler_obj, cache);
|
||||
ir_node_ = std::static_pointer_cast<DatasetNode>(ds);
|
||||
}
|
||||
MnistDataset::MnistDataset(const std::string &dataset_dir, const std::string &usage,
|
||||
const std::reference_wrapper<Sampler> sampler, const std::shared_ptr<DatasetCache> &cache) {
|
||||
auto sampler_obj = sampler.get().Parse();
|
||||
auto ds = std::make_shared<MnistNode>(dataset_dir, usage, sampler_obj, cache);
|
||||
ir_node_ = std::static_pointer_cast<DatasetNode>(ds);
|
||||
}
|
||||
|
||||
|
|
@ -926,8 +1268,23 @@ TextFileDataset::TextFileDataset(const std::vector<std::string> &dataset_files,
|
|||
|
||||
VOCDataset::VOCDataset(const std::string &dataset_dir, const std::string &task, const std::string &usage,
|
||||
const std::map<std::string, int32_t> &class_indexing, bool decode,
|
||||
const std::shared_ptr<SamplerObj> &sampler, const std::shared_ptr<DatasetCache> &cache) {
|
||||
auto ds = std::make_shared<VOCNode>(dataset_dir, task, usage, class_indexing, decode, sampler, cache);
|
||||
const std::shared_ptr<Sampler> &sampler, const std::shared_ptr<DatasetCache> &cache) {
|
||||
auto sampler_obj = sampler ? sampler->Parse() : nullptr;
|
||||
auto ds = std::make_shared<VOCNode>(dataset_dir, task, usage, class_indexing, decode, sampler_obj, cache);
|
||||
ir_node_ = std::static_pointer_cast<DatasetNode>(ds);
|
||||
}
|
||||
VOCDataset::VOCDataset(const std::string &dataset_dir, const std::string &task, const std::string &usage,
|
||||
const std::map<std::string, int32_t> &class_indexing, bool decode, Sampler *sampler,
|
||||
const std::shared_ptr<DatasetCache> &cache) {
|
||||
auto sampler_obj = sampler ? sampler->Parse() : nullptr;
|
||||
auto ds = std::make_shared<VOCNode>(dataset_dir, task, usage, class_indexing, decode, sampler_obj, cache);
|
||||
ir_node_ = std::static_pointer_cast<DatasetNode>(ds);
|
||||
}
|
||||
VOCDataset::VOCDataset(const std::string &dataset_dir, const std::string &task, const std::string &usage,
|
||||
const std::map<std::string, int32_t> &class_indexing, bool decode,
|
||||
const std::reference_wrapper<Sampler> sampler, const std::shared_ptr<DatasetCache> &cache) {
|
||||
auto sampler_obj = sampler.get().Parse();
|
||||
auto ds = std::make_shared<VOCNode>(dataset_dir, task, usage, class_indexing, decode, sampler_obj, cache);
|
||||
ir_node_ = std::static_pointer_cast<DatasetNode>(ds);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -20,78 +20,68 @@
|
|||
namespace mindspore {
|
||||
namespace dataset {
|
||||
|
||||
/// Function to create a Distributed Sampler.
|
||||
std::shared_ptr<DistributedSamplerObj> DistributedSampler(int64_t num_shards, int64_t shard_id, bool shuffle,
|
||||
int64_t num_samples, uint32_t seed, int64_t offset,
|
||||
bool even_dist) {
|
||||
auto sampler =
|
||||
std::make_shared<DistributedSamplerObj>(num_shards, shard_id, shuffle, num_samples, seed, offset, even_dist);
|
||||
// Input validation
|
||||
if (sampler->ValidateParams().IsError()) {
|
||||
return nullptr;
|
||||
}
|
||||
return sampler;
|
||||
// DistributedSampler
|
||||
DistributedSampler::DistributedSampler(int64_t num_shards, int64_t shard_id, bool shuffle, int64_t num_samples,
|
||||
uint32_t seed, int64_t offset, bool even_dist)
|
||||
: num_shards_(num_shards),
|
||||
shard_id_(shard_id),
|
||||
shuffle_(shuffle),
|
||||
num_samples_(num_samples),
|
||||
seed_(seed),
|
||||
offset_(offset),
|
||||
even_dist_(even_dist) {}
|
||||
|
||||
std::shared_ptr<SamplerObj> DistributedSampler::Parse() {
|
||||
return std::make_shared<DistributedSamplerObj>(num_shards_, shard_id_, shuffle_, num_samples_, seed_, offset_,
|
||||
even_dist_);
|
||||
}
|
||||
|
||||
/// Function to create a PK Sampler.
|
||||
std::shared_ptr<PKSamplerObj> PKSampler(int64_t num_val, bool shuffle, int64_t num_samples) {
|
||||
auto sampler = std::make_shared<PKSamplerObj>(num_val, shuffle, num_samples);
|
||||
// Input validation
|
||||
if (sampler->ValidateParams().IsError()) {
|
||||
return nullptr;
|
||||
}
|
||||
return sampler;
|
||||
// PKSampler
|
||||
PKSampler::PKSampler(int64_t num_val, bool shuffle, int64_t num_samples)
|
||||
: num_val_(num_val), shuffle_(shuffle), num_samples_(num_samples) {}
|
||||
|
||||
std::shared_ptr<SamplerObj> PKSampler::Parse() {
|
||||
return std::make_shared<PKSamplerObj>(num_val_, shuffle_, num_samples_);
|
||||
}
|
||||
|
||||
/// Function to create a Random Sampler.
|
||||
std::shared_ptr<RandomSamplerObj> RandomSampler(bool replacement, int64_t num_samples) {
|
||||
auto sampler = std::make_shared<RandomSamplerObj>(replacement, num_samples);
|
||||
// Input validation
|
||||
if (sampler->ValidateParams().IsError()) {
|
||||
return nullptr;
|
||||
}
|
||||
return sampler;
|
||||
// RandomSampler
|
||||
RandomSampler::RandomSampler(bool replacement, int64_t num_samples)
|
||||
: replacement_(replacement), num_samples_(num_samples) {}
|
||||
|
||||
std::shared_ptr<SamplerObj> RandomSampler::Parse() {
|
||||
return std::make_shared<RandomSamplerObj>(replacement_, num_samples_);
|
||||
}
|
||||
|
||||
/// Function to create a Sequential Sampler.
|
||||
std::shared_ptr<SequentialSamplerObj> SequentialSampler(int64_t start_index, int64_t num_samples) {
|
||||
auto sampler = std::make_shared<SequentialSamplerObj>(start_index, num_samples);
|
||||
// Input validation
|
||||
if (sampler->ValidateParams().IsError()) {
|
||||
return nullptr;
|
||||
}
|
||||
return sampler;
|
||||
// SequentialSampler
|
||||
SequentialSampler::SequentialSampler(int64_t start_index, int64_t num_samples)
|
||||
: start_index_(start_index), num_samples_(num_samples) {}
|
||||
|
||||
std::shared_ptr<SamplerObj> SequentialSampler::Parse() {
|
||||
return std::make_shared<SequentialSamplerObj>(start_index_, num_samples_);
|
||||
}
|
||||
|
||||
/// Function to create a Subset Random Sampler.
|
||||
std::shared_ptr<SubsetSamplerObj> SubsetSampler(std::vector<int64_t> indices, int64_t num_samples) {
|
||||
auto sampler = std::make_shared<SubsetSamplerObj>(std::move(indices), num_samples);
|
||||
// Input validation
|
||||
if (sampler->ValidateParams().IsError()) {
|
||||
return nullptr;
|
||||
}
|
||||
return sampler;
|
||||
// SubsetSampler
|
||||
SubsetSampler::SubsetSampler(std::vector<int64_t> indices, int64_t num_samples)
|
||||
: indices_(indices), num_samples_(num_samples) {}
|
||||
|
||||
std::shared_ptr<SamplerObj> SubsetSampler::Parse() {
|
||||
return std::make_shared<SubsetSamplerObj>(indices_, num_samples_);
|
||||
}
|
||||
|
||||
/// Function to create a Subset Random Sampler.
|
||||
std::shared_ptr<SubsetRandomSamplerObj> SubsetRandomSampler(std::vector<int64_t> indices, int64_t num_samples) {
|
||||
auto sampler = std::make_shared<SubsetRandomSamplerObj>(std::move(indices), num_samples);
|
||||
// Input validation
|
||||
if (sampler->ValidateParams().IsError()) {
|
||||
return nullptr;
|
||||
}
|
||||
return sampler;
|
||||
// SubsetRandomSampler
|
||||
SubsetRandomSampler::SubsetRandomSampler(std::vector<int64_t> indices, int64_t num_samples)
|
||||
: SubsetSampler(indices, num_samples) {}
|
||||
|
||||
std::shared_ptr<SamplerObj> SubsetRandomSampler::Parse() {
|
||||
return std::make_shared<SubsetRandomSamplerObj>(indices_, num_samples_);
|
||||
}
|
||||
|
||||
/// Function to create a Weighted Random Sampler.
|
||||
std::shared_ptr<WeightedRandomSamplerObj> WeightedRandomSampler(std::vector<double> weights, int64_t num_samples,
|
||||
bool replacement) {
|
||||
auto sampler = std::make_shared<WeightedRandomSamplerObj>(std::move(weights), num_samples, replacement);
|
||||
// Input validation
|
||||
if (sampler->ValidateParams().IsError()) {
|
||||
return nullptr;
|
||||
}
|
||||
return sampler;
|
||||
// WeightedRandomSampler
|
||||
WeightedRandomSampler::WeightedRandomSampler(std::vector<double> weights, int64_t num_samples, bool replacement)
|
||||
: weights_(weights), num_samples_(num_samples), replacement_(replacement) {}
|
||||
|
||||
std::shared_ptr<SamplerObj> WeightedRandomSampler::Parse() {
|
||||
return std::make_shared<WeightedRandomSamplerObj>(weights_, num_samples_, replacement_);
|
||||
}
|
||||
|
||||
} // namespace dataset
|
||||
|
|
|
|||
|
|
@ -138,7 +138,7 @@ Status ValidateDatasetShardParams(const std::string &dataset_name, int32_t num_s
|
|||
|
||||
// Helper function to validate dataset sampler parameter
|
||||
Status ValidateDatasetSampler(const std::string &dataset_name, const std::shared_ptr<SamplerObj> &sampler) {
|
||||
if (sampler == nullptr) {
|
||||
if (sampler == nullptr || sampler->ValidateParams().IsError()) {
|
||||
std::string err_msg = dataset_name + ": Sampler is not constructed correctly, sampler: nullptr";
|
||||
MS_LOG(ERROR) << err_msg;
|
||||
RETURN_STATUS_SYNTAX_ERROR(err_msg);
|
||||
|
|
@ -191,17 +191,17 @@ std::shared_ptr<SamplerObj> SelectSampler(int64_t num_samples, bool shuffle, int
|
|||
if (shuffle) {
|
||||
if (num_shards > 1) {
|
||||
// If shuffle enabled, sharding enabled, use distributed random sampler
|
||||
return DistributedSampler(num_shards, shard_id, shuffle, num_samples);
|
||||
return DistributedSampler(num_shards, shard_id, shuffle, num_samples).Parse();
|
||||
}
|
||||
// If shuffle enabled, sharding disabled, use random sampler
|
||||
return RandomSampler(num_samples >= 0, num_samples);
|
||||
return RandomSampler(num_samples >= 0, num_samples).Parse();
|
||||
}
|
||||
if (num_shards > 1) {
|
||||
// If shuffle disabled, sharding enabled, use distributed sequential sampler
|
||||
return DistributedSampler(num_shards, shard_id, shuffle, num_samples);
|
||||
return DistributedSampler(num_shards, shard_id, shuffle, num_samples).Parse();
|
||||
}
|
||||
// If shuffle disabled, sharding disabled, use sequential sampler
|
||||
return SequentialSampler(0, num_samples);
|
||||
return SequentialSampler(0, num_samples).Parse();
|
||||
}
|
||||
|
||||
// Constructor to initialize the cache
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
/**
|
||||
* Copyright 2020 Huawei Technologies Co., Ltd
|
||||
* Copyright 2020-2021 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
|
|
@ -585,7 +585,13 @@ class AlbumDataset : public Dataset {
|
|||
public:
|
||||
AlbumDataset(const std::string &dataset_dir, const std::string &data_schema,
|
||||
const std::vector<std::string> &column_names = {}, bool decode = false,
|
||||
const std::shared_ptr<SamplerObj> &sampler = RandomSampler(),
|
||||
const std::shared_ptr<Sampler> &sampler = std::make_shared<RandomSampler>(),
|
||||
const std::shared_ptr<DatasetCache> &cache = nullptr);
|
||||
AlbumDataset(const std::string &dataset_dir, const std::string &data_schema,
|
||||
const std::vector<std::string> &column_names, bool decode, Sampler *sampler,
|
||||
const std::shared_ptr<DatasetCache> &cache = nullptr);
|
||||
AlbumDataset(const std::string &dataset_dir, const std::string &data_schema,
|
||||
const std::vector<std::string> &column_names, bool decode, const std::reference_wrapper<Sampler> sampler,
|
||||
const std::shared_ptr<DatasetCache> &cache = nullptr);
|
||||
~AlbumDataset() = default;
|
||||
};
|
||||
|
|
@ -597,20 +603,53 @@ class AlbumDataset : public Dataset {
|
|||
/// \param[in] column_names Column names used to specify columns to load, if empty, will read all columns.
|
||||
/// (default = {})
|
||||
/// \param[in] decode the option to decode the images in dataset (default = false)
|
||||
/// \param[in] sampler Object used to choose samples from the dataset. If sampler is not given,
|
||||
/// \param[in] sampler Shared pointer to a sampler object used to choose samples from the dataset. If sampler is not
|
||||
/// given,
|
||||
/// a `RandomSampler` will be used to randomly iterate the entire dataset (default = RandomSampler())
|
||||
/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used).
|
||||
/// \return Shared pointer to the current Dataset
|
||||
std::shared_ptr<AlbumDataset> Album(const std::string &dataset_dir, const std::string &data_schema,
|
||||
const std::vector<std::string> &column_names = {}, bool decode = false,
|
||||
const std::shared_ptr<SamplerObj> &sampler = RandomSampler(),
|
||||
const std::shared_ptr<Sampler> &sampler = std::make_shared<RandomSampler>(),
|
||||
const std::shared_ptr<DatasetCache> &cache = nullptr);
|
||||
/// \brief Function to create an AlbumDataset
|
||||
/// \notes The generated dataset is specified through setting a schema
|
||||
/// \param[in] dataset_dir Path to the root directory that contains the dataset
|
||||
/// \param[in] data_schema Path to dataset schema file
|
||||
/// \param[in] column_names Column names used to specify columns to load
|
||||
/// \param[in] decode the option to decode the images in dataset
|
||||
/// \param[in] sampler Raw pointer to a sampler object used to choose samples from the dataset.
|
||||
/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used).
|
||||
/// \return Shared pointer to the current Dataset
|
||||
std::shared_ptr<AlbumDataset> Album(const std::string &dataset_dir, const std::string &data_schema,
|
||||
const std::vector<std::string> &column_names, bool decode, Sampler *sampler,
|
||||
const std::shared_ptr<DatasetCache> &cache = nullptr);
|
||||
/// \brief Function to create an AlbumDataset
|
||||
/// \notes The generated dataset is specified through setting a schema
|
||||
/// \param[in] dataset_dir Path to the root directory that contains the dataset
|
||||
/// \param[in] data_schema Path to dataset schema file
|
||||
/// \param[in] column_names Column names used to specify columns to load
|
||||
/// \param[in] decode the option to decode the images in dataset
|
||||
/// \param[in] sampler Sampler object used to choose samples from the dataset.
|
||||
/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used).
|
||||
/// \return Shared pointer to the current Dataset
|
||||
std::shared_ptr<AlbumDataset> Album(const std::string &dataset_dir, const std::string &data_schema,
|
||||
const std::vector<std::string> &column_names, bool decode,
|
||||
const std::reference_wrapper<Sampler> sampler,
|
||||
const std::shared_ptr<DatasetCache> &cache = nullptr);
|
||||
|
||||
#ifndef ENABLE_ANDROID
|
||||
class CelebADataset : public Dataset {
|
||||
public:
|
||||
explicit CelebADataset(const std::string &dataset_dir, const std::string &usage = "all",
|
||||
const std::shared_ptr<SamplerObj> &sampler = RandomSampler(), bool decode = false,
|
||||
const std::shared_ptr<Sampler> &sampler = std::make_shared<RandomSampler>(),
|
||||
bool decode = false, const std::set<std::string> &extensions = {},
|
||||
const std::shared_ptr<DatasetCache> &cache = nullptr);
|
||||
explicit CelebADataset(const std::string &dataset_dir, const std::string &usage, Sampler *sampler,
|
||||
bool decode = false, const std::set<std::string> &extensions = {},
|
||||
const std::shared_ptr<DatasetCache> &cache = nullptr);
|
||||
explicit CelebADataset(const std::string &dataset_dir, const std::string &usage,
|
||||
const std::reference_wrapper<Sampler> sampler, bool decode = false,
|
||||
const std::set<std::string> &extensions = {},
|
||||
const std::shared_ptr<DatasetCache> &cache = nullptr);
|
||||
~CelebADataset() = default;
|
||||
|
|
@ -621,21 +660,54 @@ class CelebADataset : public Dataset {
|
|||
/// The type of the image tensor is uint8. The attr tensor is uint32 and one hot type.
|
||||
/// \param[in] dataset_dir Path to the root directory that contains the dataset.
|
||||
/// \param[in] usage One of "all", "train", "valid" or "test" (default = "all").
|
||||
/// \param[in] sampler Object used to choose samples from the dataset. If sampler is not given,
|
||||
/// \param[in] sampler Shared pointer to a sampler object used to choose samples from the dataset. If sampler is not
|
||||
/// given,
|
||||
/// a `RandomSampler` will be used to randomly iterate the entire dataset (default = RandomSampler())
|
||||
/// \param[in] decode Decode the images after reading (default=false).
|
||||
/// \param[in] extensions Set of file extensions to be included in the dataset (default={}).
|
||||
/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used).
|
||||
/// \return Shared pointer to the current Dataset
|
||||
std::shared_ptr<CelebADataset> CelebA(const std::string &dataset_dir, const std::string &usage = "all",
|
||||
const std::shared_ptr<SamplerObj> &sampler = RandomSampler(), bool decode = false,
|
||||
const std::shared_ptr<Sampler> &sampler = std::make_shared<RandomSampler>(),
|
||||
bool decode = false, const std::set<std::string> &extensions = {},
|
||||
const std::shared_ptr<DatasetCache> &cache = nullptr);
|
||||
/// \brief Function to create a CelebADataset
|
||||
/// \notes The generated dataset has two columns ['image', 'attr'].
|
||||
/// The type of the image tensor is uint8. The attr tensor is uint32 and one hot type.
|
||||
/// \param[in] dataset_dir Path to the root directory that contains the dataset.
|
||||
/// \param[in] usage One of "all", "train", "valid" or "test"
|
||||
/// \param[in] sampler Raw pointer to a sampler object used to choose samples from the dataset.
|
||||
/// \param[in] decode Decode the images after reading (default=false).
|
||||
/// \param[in] extensions Set of file extensions to be included in the dataset (default={}).
|
||||
/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used).
|
||||
/// \return Shared pointer to the current Dataset
|
||||
std::shared_ptr<CelebADataset> CelebA(const std::string &dataset_dir, const std::string &usage, Sampler *sampler,
|
||||
bool decode = false, const std::set<std::string> &extensions = {},
|
||||
const std::shared_ptr<DatasetCache> &cache = nullptr);
|
||||
/// \brief Function to create a CelebADataset
|
||||
/// \notes The generated dataset has two columns ['image', 'attr'].
|
||||
/// The type of the image tensor is uint8. The attr tensor is uint32 and one hot type.
|
||||
/// \param[in] dataset_dir Path to the root directory that contains the dataset.
|
||||
/// \param[in] usage One of "all", "train", "valid" or "test"
|
||||
/// \param[in] sampler Sampler object used to choose samples from the dataset.
|
||||
/// \param[in] decode Decode the images after reading (default=false).
|
||||
/// \param[in] extensions Set of file extensions to be included in the dataset (default={}).
|
||||
/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used).
|
||||
/// \return Shared pointer to the current Dataset
|
||||
std::shared_ptr<CelebADataset> CelebA(const std::string &dataset_dir, const std::string &usage,
|
||||
const std::reference_wrapper<Sampler> sampler, bool decode = false,
|
||||
const std::set<std::string> &extensions = {},
|
||||
const std::shared_ptr<DatasetCache> &cache = nullptr);
|
||||
|
||||
class Cifar10Dataset : public Dataset {
|
||||
public:
|
||||
explicit Cifar10Dataset(const std::string &dataset_dir, const std::string &usage = "all",
|
||||
const std::shared_ptr<SamplerObj> &sampler = RandomSampler(),
|
||||
const std::shared_ptr<Sampler> &sampler = std::make_shared<RandomSampler>(),
|
||||
const std::shared_ptr<DatasetCache> &cache = nullptr);
|
||||
explicit Cifar10Dataset(const std::string &dataset_dir, const std::string &usage, Sampler *sampler,
|
||||
const std::shared_ptr<DatasetCache> &cache = nullptr);
|
||||
explicit Cifar10Dataset(const std::string &dataset_dir, const std::string &usage,
|
||||
const std::reference_wrapper<Sampler> sampler,
|
||||
const std::shared_ptr<DatasetCache> &cache = nullptr);
|
||||
~Cifar10Dataset() = default;
|
||||
};
|
||||
|
|
@ -644,18 +716,43 @@ class Cifar10Dataset : public Dataset {
|
|||
/// \notes The generated dataset has two columns ["image", "label"]
|
||||
/// \param[in] dataset_dir Path to the root directory that contains the dataset
|
||||
/// \param[in] usage of CIFAR10, can be "train", "test" or "all" (default = "all").
|
||||
/// \param[in] sampler Object used to choose samples from the dataset. If sampler is not given,
|
||||
/// \param[in] sampler Shared pointer to a sampler object used to choose samples from the dataset. If sampler is not
|
||||
/// given,
|
||||
/// a `RandomSampler` will be used to randomly iterate the entire dataset (default = RandomSampler())
|
||||
/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used).
|
||||
/// \return Shared pointer to the current Dataset
|
||||
std::shared_ptr<Cifar10Dataset> Cifar10(const std::string &dataset_dir, const std::string &usage = "all",
|
||||
const std::shared_ptr<SamplerObj> &sampler = RandomSampler(),
|
||||
const std::shared_ptr<Sampler> &sampler = std::make_shared<RandomSampler>(),
|
||||
const std::shared_ptr<DatasetCache> &cache = nullptr);
|
||||
/// \brief Function to create a Cifar10 Dataset
|
||||
/// \notes The generated dataset has two columns ["image", "label"]
|
||||
/// \param[in] dataset_dir Path to the root directory that contains the dataset
|
||||
/// \param[in] usage of CIFAR10, can be "train", "test" or "all"
|
||||
/// \param[in] sampler Raw pointer to a sampler object used to choose samples from the dataset.
|
||||
/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used).
|
||||
/// \return Shared pointer to the current Dataset
|
||||
std::shared_ptr<Cifar10Dataset> Cifar10(const std::string &dataset_dir, const std::string &usage, Sampler *sampler,
|
||||
const std::shared_ptr<DatasetCache> &cache = nullptr);
|
||||
/// \brief Function to create a Cifar10 Dataset
|
||||
/// \notes The generated dataset has two columns ["image", "label"]
|
||||
/// \param[in] dataset_dir Path to the root directory that contains the dataset
|
||||
/// \param[in] usage of CIFAR10, can be "train", "test" or "all"
|
||||
/// \param[in] sampler Sampler object used to choose samples from the dataset.
|
||||
/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used).
|
||||
/// \return Shared pointer to the current Dataset
|
||||
std::shared_ptr<Cifar10Dataset> Cifar10(const std::string &dataset_dir, const std::string &usage,
|
||||
const std::reference_wrapper<Sampler> sampler,
|
||||
const std::shared_ptr<DatasetCache> &cache = nullptr);
|
||||
|
||||
class Cifar100Dataset : public Dataset {
|
||||
public:
|
||||
explicit Cifar100Dataset(const std::string &dataset_dir, const std::string &usage = "all",
|
||||
const std::shared_ptr<SamplerObj> &sampler = RandomSampler(),
|
||||
const std::shared_ptr<Sampler> &sampler = std::make_shared<RandomSampler>(),
|
||||
const std::shared_ptr<DatasetCache> &cache = nullptr);
|
||||
explicit Cifar100Dataset(const std::string &dataset_dir, const std::string &usage, Sampler *sampler,
|
||||
const std::shared_ptr<DatasetCache> &cache = nullptr);
|
||||
explicit Cifar100Dataset(const std::string &dataset_dir, const std::string &usage,
|
||||
const std::reference_wrapper<Sampler> sampler,
|
||||
const std::shared_ptr<DatasetCache> &cache = nullptr);
|
||||
~Cifar100Dataset() = default;
|
||||
};
|
||||
|
|
@ -664,12 +761,32 @@ class Cifar100Dataset : public Dataset {
|
|||
/// \notes The generated dataset has three columns ["image", "coarse_label", "fine_label"]
|
||||
/// \param[in] dataset_dir Path to the root directory that contains the dataset
|
||||
/// \param[in] usage of CIFAR100, can be "train", "test" or "all" (default = "all").
|
||||
/// \param[in] sampler Object used to choose samples from the dataset. If sampler is not given,
|
||||
/// \param[in] sampler Shared pointer to a sampler object used to choose samples from the dataset. If sampler is not
|
||||
/// given,
|
||||
/// a `RandomSampler` will be used to randomly iterate the entire dataset (default = RandomSampler())
|
||||
/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used).
|
||||
/// \return Shared pointer to the current Dataset
|
||||
std::shared_ptr<Cifar100Dataset> Cifar100(const std::string &dataset_dir, const std::string &usage = "all",
|
||||
const std::shared_ptr<SamplerObj> &sampler = RandomSampler(),
|
||||
const std::shared_ptr<Sampler> &sampler = std::make_shared<RandomSampler>(),
|
||||
const std::shared_ptr<DatasetCache> &cache = nullptr);
|
||||
/// \brief Function to create a Cifar100 Dataset
|
||||
/// \notes The generated dataset has three columns ["image", "coarse_label", "fine_label"]
|
||||
/// \param[in] dataset_dir Path to the root directory that contains the dataset
|
||||
/// \param[in] usage of CIFAR100, can be "train", "test" or "all".
|
||||
/// \param[in] sampler Raw pointer to a sampler object used to choose samples from the dataset.
|
||||
/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used).
|
||||
/// \return Shared pointer to the current Dataset
|
||||
std::shared_ptr<Cifar100Dataset> Cifar100(const std::string &dataset_dir, const std::string &usage, Sampler *sampler,
|
||||
const std::shared_ptr<DatasetCache> &cache = nullptr);
|
||||
/// \brief Function to create a Cifar100 Dataset
|
||||
/// \notes The generated dataset has three columns ["image", "coarse_label", "fine_label"]
|
||||
/// \param[in] dataset_dir Path to the root directory that contains the dataset
|
||||
/// \param[in] usage of CIFAR100, can be "train", "test" or "all".
|
||||
/// \param[in] sampler Sampler object used to choose samples from the dataset.
|
||||
/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used).
|
||||
/// \return Shared pointer to the current Dataset
|
||||
std::shared_ptr<Cifar100Dataset> Cifar100(const std::string &dataset_dir, const std::string &usage,
|
||||
const std::reference_wrapper<Sampler> sampler,
|
||||
const std::shared_ptr<DatasetCache> &cache = nullptr);
|
||||
|
||||
class CLUEDataset : public Dataset {
|
||||
|
|
@ -707,7 +824,12 @@ std::shared_ptr<CLUEDataset> CLUE(const std::vector<std::string> &dataset_files,
|
|||
class CocoDataset : public Dataset {
|
||||
public:
|
||||
CocoDataset(const std::string &dataset_dir, const std::string &annotation_file, const std::string &task = "Detection",
|
||||
const bool &decode = false, const std::shared_ptr<SamplerObj> &sampler = RandomSampler(),
|
||||
const bool &decode = false, const std::shared_ptr<Sampler> &sampler = std::make_shared<RandomSampler>(),
|
||||
const std::shared_ptr<DatasetCache> &cache = nullptr);
|
||||
CocoDataset(const std::string &dataset_dir, const std::string &annotation_file, const std::string &task,
|
||||
const bool &decode, Sampler *sampler, const std::shared_ptr<DatasetCache> &cache = nullptr);
|
||||
CocoDataset(const std::string &dataset_dir, const std::string &annotation_file, const std::string &task,
|
||||
const bool &decode, const std::reference_wrapper<Sampler> sampler,
|
||||
const std::shared_ptr<DatasetCache> &cache = nullptr);
|
||||
~CocoDataset() = default;
|
||||
};
|
||||
|
|
@ -725,13 +847,53 @@ class CocoDataset : public Dataset {
|
|||
/// \param[in] annotation_file Path to the annotation json
|
||||
/// \param[in] task Set the task type of reading coco data, now support 'Detection'/'Stuff'/'Panoptic'/'Keypoint'
|
||||
/// \param[in] decode Decode the images after reading
|
||||
/// \param[in] sampler Object used to choose samples from the dataset. If sampler is not given,
|
||||
/// \param[in] sampler Shared pointer to a sampler object used to choose samples from the dataset. If sampler is not
|
||||
/// given,
|
||||
/// a `RandomSampler` will be used to randomly iterate the entire dataset (default = RandomSampler())
|
||||
/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used).
|
||||
/// \return Shared pointer to the current Dataset
|
||||
std::shared_ptr<CocoDataset> Coco(const std::string &dataset_dir, const std::string &annotation_file,
|
||||
const std::string &task = "Detection", const bool &decode = false,
|
||||
const std::shared_ptr<SamplerObj> &sampler = RandomSampler(),
|
||||
const std::shared_ptr<Sampler> &sampler = std::make_shared<RandomSampler>(),
|
||||
const std::shared_ptr<DatasetCache> &cache = nullptr);
|
||||
/// \brief Function to create a CocoDataset
|
||||
/// \notes The generated dataset has multi-columns :
|
||||
/// - task='Detection', column: [['image', dtype=uint8], ['bbox', dtype=float32], ['category_id', dtype=uint32],
|
||||
/// ['iscrowd', dtype=uint32]].
|
||||
/// - task='Stuff', column: [['image', dtype=uint8], ['segmentation',dtype=float32], ['iscrowd', dtype=uint32]].
|
||||
/// - task='Keypoint', column: [['image', dtype=uint8], ['keypoints', dtype=float32],
|
||||
/// ['num_keypoints', dtype=uint32]].
|
||||
/// - task='Panoptic', column: [['image', dtype=uint8], ['bbox', dtype=float32], ['category_id', dtype=uint32],
|
||||
/// ['iscrowd', dtype=uint32], ['area', dtype=uitn32]].
|
||||
/// \param[in] dataset_dir Path to the root directory that contains the dataset
|
||||
/// \param[in] annotation_file Path to the annotation json
|
||||
/// \param[in] task Set the task type of reading coco data, now support 'Detection'/'Stuff'/'Panoptic'/'Keypoint'
|
||||
/// \param[in] decode Decode the images after reading
|
||||
/// \param[in] sampler Raw pointer to a sampler object used to choose samples from the dataset.
|
||||
/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used).
|
||||
/// \return Shared pointer to the current Dataset
|
||||
std::shared_ptr<CocoDataset> Coco(const std::string &dataset_dir, const std::string &annotation_file,
|
||||
const std::string &task, const bool &decode, Sampler *sampler,
|
||||
const std::shared_ptr<DatasetCache> &cache = nullptr);
|
||||
/// \brief Function to create a CocoDataset
|
||||
/// \notes The generated dataset has multi-columns :
|
||||
/// - task='Detection', column: [['image', dtype=uint8], ['bbox', dtype=float32], ['category_id', dtype=uint32],
|
||||
/// ['iscrowd', dtype=uint32]].
|
||||
/// - task='Stuff', column: [['image', dtype=uint8], ['segmentation',dtype=float32], ['iscrowd', dtype=uint32]].
|
||||
/// - task='Keypoint', column: [['image', dtype=uint8], ['keypoints', dtype=float32],
|
||||
/// ['num_keypoints', dtype=uint32]].
|
||||
/// - task='Panoptic', column: [['image', dtype=uint8], ['bbox', dtype=float32], ['category_id', dtype=uint32],
|
||||
/// ['iscrowd', dtype=uint32], ['area', dtype=uitn32]].
|
||||
/// \param[in] dataset_dir Path to the root directory that contains the dataset
|
||||
/// \param[in] annotation_file Path to the annotation json
|
||||
/// \param[in] task Set the task type of reading coco data, now support 'Detection'/'Stuff'/'Panoptic'/'Keypoint'
|
||||
/// \param[in] decode Decode the images after reading
|
||||
/// \param[in] sampler Sampler object used to choose samples from the dataset.
|
||||
/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used).
|
||||
/// \return Shared pointer to the current Dataset
|
||||
std::shared_ptr<CocoDataset> Coco(const std::string &dataset_dir, const std::string &annotation_file,
|
||||
const std::string &task, const bool &decode,
|
||||
const std::reference_wrapper<Sampler> sampler,
|
||||
const std::shared_ptr<DatasetCache> &cache = nullptr);
|
||||
|
||||
class CSVDataset : public Dataset {
|
||||
|
|
@ -774,7 +936,16 @@ std::shared_ptr<CSVDataset> CSV(const std::vector<std::string> &dataset_files, c
|
|||
class ImageFolderDataset : public Dataset {
|
||||
public:
|
||||
explicit ImageFolderDataset(const std::string &dataset_dir, bool decode = false,
|
||||
const std::shared_ptr<SamplerObj> &sampler = RandomSampler(),
|
||||
const std::shared_ptr<Sampler> &sampler = std::make_shared<RandomSampler>(),
|
||||
const std::set<std::string> &extensions = {},
|
||||
const std::map<std::string, int32_t> &class_indexing = {},
|
||||
const std::shared_ptr<DatasetCache> &cache = nullptr);
|
||||
explicit ImageFolderDataset(const std::string &dataset_dir, bool decode, Sampler *sampler,
|
||||
const std::set<std::string> &extensions = {},
|
||||
const std::map<std::string, int32_t> &class_indexing = {},
|
||||
const std::shared_ptr<DatasetCache> &cache = nullptr);
|
||||
explicit ImageFolderDataset(const std::string &dataset_dir, bool decode,
|
||||
const std::reference_wrapper<Sampler> sampler,
|
||||
const std::set<std::string> &extensions = {},
|
||||
const std::map<std::string, int32_t> &class_indexing = {},
|
||||
const std::shared_ptr<DatasetCache> &cache = nullptr);
|
||||
|
|
@ -787,14 +958,46 @@ class ImageFolderDataset : public Dataset {
|
|||
/// The generated dataset has two columns ["image", "label"]
|
||||
/// \param[in] dataset_dir Path to the root directory that contains the dataset
|
||||
/// \param[in] decode A flag to decode in ImageFolder
|
||||
/// \param[in] sampler Object used to choose samples from the dataset. If sampler is not given,
|
||||
/// \param[in] sampler Shared pointer to a sampler object used to choose samples from the dataset. If sampler is not
|
||||
/// given,
|
||||
/// a `RandomSampler` will be used to randomly iterate the entire dataset (default = RandomSampler())
|
||||
/// \param[in] extensions File extensions to be read
|
||||
/// \param[in] class_indexing a class name to label map
|
||||
/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used).
|
||||
/// \return Shared pointer to the current ImageFolderDataset
|
||||
std::shared_ptr<ImageFolderDataset> ImageFolder(const std::string &dataset_dir, bool decode = false,
|
||||
const std::shared_ptr<SamplerObj> &sampler = RandomSampler(),
|
||||
std::shared_ptr<ImageFolderDataset> ImageFolder(
|
||||
const std::string &dataset_dir, bool decode = false,
|
||||
const std::shared_ptr<Sampler> &sampler = std::make_shared<RandomSampler>(),
|
||||
const std::set<std::string> &extensions = {}, const std::map<std::string, int32_t> &class_indexing = {},
|
||||
const std::shared_ptr<DatasetCache> &cache = nullptr);
|
||||
/// \brief Function to create an ImageFolderDataset
|
||||
/// \notes A source dataset that reads images from a tree of directories
|
||||
/// All images within one folder have the same label
|
||||
/// The generated dataset has two columns ["image", "label"]
|
||||
/// \param[in] dataset_dir Path to the root directory that contains the dataset
|
||||
/// \param[in] decode A flag to decode in ImageFolder
|
||||
/// \param[in] sampler Raw pointer to a sampler object used to choose samples from the dataset.
|
||||
/// \param[in] extensions File extensions to be read
|
||||
/// \param[in] class_indexing a class name to label map
|
||||
/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used).
|
||||
/// \return Shared pointer to the current ImageFolderDataset
|
||||
std::shared_ptr<ImageFolderDataset> ImageFolder(const std::string &dataset_dir, bool decode, Sampler *sampler,
|
||||
const std::set<std::string> &extensions = {},
|
||||
const std::map<std::string, int32_t> &class_indexing = {},
|
||||
const std::shared_ptr<DatasetCache> &cache = nullptr);
|
||||
/// \brief Function to create an ImageFolderDataset
|
||||
/// \notes A source dataset that reads images from a tree of directories
|
||||
/// All images within one folder have the same label
|
||||
/// The generated dataset has two columns ["image", "label"]
|
||||
/// \param[in] dataset_dir Path to the root directory that contains the dataset
|
||||
/// \param[in] decode A flag to decode in ImageFolder
|
||||
/// \param[in] sampler Sampler object used to choose samples from the dataset.
|
||||
/// \param[in] extensions File extensions to be read
|
||||
/// \param[in] class_indexing a class name to label map
|
||||
/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used).
|
||||
/// \return Shared pointer to the current ImageFolderDataset
|
||||
std::shared_ptr<ImageFolderDataset> ImageFolder(const std::string &dataset_dir, bool decode,
|
||||
const std::reference_wrapper<Sampler> sampler,
|
||||
const std::set<std::string> &extensions = {},
|
||||
const std::map<std::string, int32_t> &class_indexing = {},
|
||||
const std::shared_ptr<DatasetCache> &cache = nullptr);
|
||||
|
|
@ -802,7 +1005,14 @@ std::shared_ptr<ImageFolderDataset> ImageFolder(const std::string &dataset_dir,
|
|||
class ManifestDataset : public Dataset {
|
||||
public:
|
||||
explicit ManifestDataset(const std::string &dataset_file, const std::string &usage = "train",
|
||||
const std::shared_ptr<SamplerObj> &sampler = RandomSampler(),
|
||||
const std::shared_ptr<Sampler> &sampler = std::make_shared<RandomSampler>(),
|
||||
const std::map<std::string, int32_t> &class_indexing = {}, bool decode = false,
|
||||
const std::shared_ptr<DatasetCache> &cache = nullptr);
|
||||
explicit ManifestDataset(const std::string &dataset_file, const std::string &usage, Sampler *sampler,
|
||||
const std::map<std::string, int32_t> &class_indexing = {}, bool decode = false,
|
||||
const std::shared_ptr<DatasetCache> &cache = nullptr);
|
||||
explicit ManifestDataset(const std::string &dataset_file, const std::string &usage,
|
||||
const std::reference_wrapper<Sampler> sampler,
|
||||
const std::map<std::string, int32_t> &class_indexing = {}, bool decode = false,
|
||||
const std::shared_ptr<DatasetCache> &cache = nullptr);
|
||||
~ManifestDataset() = default;
|
||||
|
|
@ -812,7 +1022,8 @@ class ManifestDataset : public Dataset {
|
|||
/// \notes The generated dataset has two columns ["image", "label"]
|
||||
/// \param[in] dataset_file The dataset file to be read
|
||||
/// \param[in] usage Need "train", "eval" or "inference" data (default="train")
|
||||
/// \param[in] sampler Object used to choose samples from the dataset. If sampler is not given,
|
||||
/// \param[in] sampler Shared pointer to a sampler object used to choose samples from the dataset. If sampler is not
|
||||
/// given,
|
||||
/// a `RandomSampler` will be used to randomly iterate the entire dataset (default = RandomSampler())
|
||||
/// \param[in] class_indexing A str-to-int mapping from label name to index (default={}, the folder
|
||||
/// names will be sorted alphabetically and each class will be given a unique index starting from 0).
|
||||
|
|
@ -820,19 +1031,56 @@ class ManifestDataset : public Dataset {
|
|||
/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used).
|
||||
/// \return Shared pointer to the current ManifestDataset
|
||||
std::shared_ptr<ManifestDataset> Manifest(const std::string &dataset_file, const std::string &usage = "train",
|
||||
const std::shared_ptr<SamplerObj> &sampler = RandomSampler(),
|
||||
const std::shared_ptr<Sampler> &sampler = std::make_shared<RandomSampler>(),
|
||||
const std::map<std::string, int32_t> &class_indexing = {},
|
||||
bool decode = false, const std::shared_ptr<DatasetCache> &cache = nullptr);
|
||||
/// \brief Function to create a ManifestDataset
|
||||
/// \notes The generated dataset has two columns ["image", "label"]
|
||||
/// \param[in] dataset_file The dataset file to be read
|
||||
/// \param[in] usage Need "train", "eval" or "inference" data
|
||||
/// \param[in] sampler Raw pointer to a sampler object used to choose samples from the dataset.
|
||||
/// \param[in] class_indexing A str-to-int mapping from label name to index (default={}, the folder
|
||||
/// names will be sorted alphabetically and each class will be given a unique index starting from 0).
|
||||
/// \param[in] decode Decode the images after reading (default=false).
|
||||
/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used).
|
||||
/// \return Shared pointer to the current ManifestDataset
|
||||
std::shared_ptr<ManifestDataset> Manifest(const std::string &dataset_file, const std::string &usage, Sampler *sampler,
|
||||
const std::map<std::string, int32_t> &class_indexing = {},
|
||||
bool decode = false, const std::shared_ptr<DatasetCache> &cache = nullptr);
|
||||
/// \brief Function to create a ManifestDataset
|
||||
/// \notes The generated dataset has two columns ["image", "label"]
|
||||
/// \param[in] dataset_file The dataset file to be read
|
||||
/// \param[in] usage Need "train", "eval" or "inference" data
|
||||
/// \param[in] sampler Sampler object used to choose samples from the dataset.
|
||||
/// \param[in] class_indexing A str-to-int mapping from label name to index (default={}, the folder
|
||||
/// names will be sorted alphabetically and each class will be given a unique index starting from 0).
|
||||
/// \param[in] decode Decode the images after reading (default=false).
|
||||
/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used).
|
||||
/// \return Shared pointer to the current ManifestDataset
|
||||
std::shared_ptr<ManifestDataset> Manifest(const std::string &dataset_file, const std::string &usage,
|
||||
const std::reference_wrapper<Sampler> sampler,
|
||||
const std::map<std::string, int32_t> &class_indexing = {},
|
||||
bool decode = false, const std::shared_ptr<DatasetCache> &cache = nullptr);
|
||||
|
||||
class MindDataDataset : public Dataset {
|
||||
public:
|
||||
explicit MindDataDataset(const std::string &dataset_file, const std::vector<std::string> &columns_list = {},
|
||||
const std::shared_ptr<SamplerObj> &sampler = RandomSampler(),
|
||||
const std::shared_ptr<Sampler> &sampler = std::make_shared<RandomSampler>(),
|
||||
nlohmann::json padded_sample = nullptr, int64_t num_padded = 0);
|
||||
explicit MindDataDataset(const std::string &dataset_file, const std::vector<std::string> &columns_list,
|
||||
Sampler *sampler, nlohmann::json padded_sample = nullptr, int64_t num_padded = 0);
|
||||
explicit MindDataDataset(const std::string &dataset_file, const std::vector<std::string> &columns_list,
|
||||
const std::reference_wrapper<Sampler> sampler, nlohmann::json padded_sample = nullptr,
|
||||
int64_t num_padded = 0);
|
||||
explicit MindDataDataset(const std::vector<std::string> &dataset_files,
|
||||
const std::vector<std::string> &columns_list = {},
|
||||
const std::shared_ptr<SamplerObj> &sampler = RandomSampler(),
|
||||
const std::shared_ptr<Sampler> &sampler = std::make_shared<RandomSampler>(),
|
||||
nlohmann::json padded_sample = nullptr, int64_t num_padded = 0);
|
||||
explicit MindDataDataset(const std::vector<std::string> &dataset_files, const std::vector<std::string> &columns_list,
|
||||
Sampler *sampler, nlohmann::json padded_sample = nullptr, int64_t num_padded = 0);
|
||||
explicit MindDataDataset(const std::vector<std::string> &dataset_files, const std::vector<std::string> &columns_list,
|
||||
const std::reference_wrapper<Sampler> sampler, nlohmann::json padded_sample = nullptr,
|
||||
int64_t num_padded = 0);
|
||||
~MindDataDataset() = default;
|
||||
};
|
||||
|
||||
|
|
@ -840,7 +1088,8 @@ class MindDataDataset : public Dataset {
|
|||
/// \param[in] dataset_file File name of one component of a mindrecord source. Other files with identical source
|
||||
/// in the same path will be found and loaded automatically.
|
||||
/// \param[in] columns_list List of columns to be read (default={})
|
||||
/// \param[in] sampler Object used to choose samples from the dataset. If sampler is not given,
|
||||
/// \param[in] sampler Shared pointer to a sampler object used to choose samples from the dataset. If sampler is not
|
||||
/// given,
|
||||
/// a `RandomSampler` will be used to randomly iterate the entire dataset (default = RandomSampler()),
|
||||
/// supported sampler list: SubsetRandomSampler, PkSampler, RandomSampler, SequentialSampler, DistributedSampler.
|
||||
/// \param[in] padded_sample Samples will be appended to dataset, where keys are the same as column_list.
|
||||
|
|
@ -848,13 +1097,38 @@ class MindDataDataset : public Dataset {
|
|||
/// \return Shared pointer to the current MindDataDataset
|
||||
std::shared_ptr<MindDataDataset> MindData(const std::string &dataset_file,
|
||||
const std::vector<std::string> &columns_list = {},
|
||||
const std::shared_ptr<SamplerObj> &sampler = RandomSampler(),
|
||||
const std::shared_ptr<Sampler> &sampler = std::make_shared<RandomSampler>(),
|
||||
nlohmann::json padded_sample = nullptr, int64_t num_padded = 0);
|
||||
/// \brief Function to create a MindDataDataset
|
||||
/// \param[in] dataset_file File name of one component of a mindrecord source. Other files with identical source
|
||||
/// in the same path will be found and loaded automatically.
|
||||
/// \param[in] columns_list List of columns to be read
|
||||
/// \param[in] sampler Raw pointer to a sampler object used to choose samples from the dataset.
|
||||
/// supported sampler list: SubsetRandomSampler, PkSampler, RandomSampler, SequentialSampler, DistributedSampler.
|
||||
/// \param[in] padded_sample Samples will be appended to dataset, where keys are the same as column_list.
|
||||
/// \param[in] num_padded Number of padding samples. Dataset size plus num_padded should be divisible by num_shards.
|
||||
/// \return Shared pointer to the current MindDataDataset
|
||||
std::shared_ptr<MindDataDataset> MindData(const std::string &dataset_file, const std::vector<std::string> &columns_list,
|
||||
Sampler *sampler, nlohmann::json padded_sample = nullptr,
|
||||
int64_t num_padded = 0);
|
||||
/// \brief Function to create a MindDataDataset
|
||||
/// \param[in] dataset_file File name of one component of a mindrecord source. Other files with identical source
|
||||
/// in the same path will be found and loaded automatically.
|
||||
/// \param[in] columns_list List of columns to be read
|
||||
/// \param[in] sampler Sampler object used to choose samples from the dataset.
|
||||
/// supported sampler list: SubsetRandomSampler, PkSampler, RandomSampler, SequentialSampler, DistributedSampler.
|
||||
/// \param[in] padded_sample Samples will be appended to dataset, where keys are the same as column_list.
|
||||
/// \param[in] num_padded Number of padding samples. Dataset size plus num_padded should be divisible by num_shards.
|
||||
/// \return Shared pointer to the current MindDataDataset
|
||||
std::shared_ptr<MindDataDataset> MindData(const std::string &dataset_file, const std::vector<std::string> &columns_list,
|
||||
const std::reference_wrapper<Sampler> sampler,
|
||||
nlohmann::json padded_sample = nullptr, int64_t num_padded = 0);
|
||||
|
||||
/// \brief Function to create a MindDataDataset
|
||||
/// \param[in] dataset_files List of dataset files to be read directly.
|
||||
/// \param[in] columns_list List of columns to be read (default={})
|
||||
/// \param[in] sampler Object used to choose samples from the dataset. If sampler is not given,
|
||||
/// \param[in] sampler Shared pointer to a sampler object used to choose samples from the dataset. If sampler is not
|
||||
/// given,
|
||||
/// a `RandomSampler` will be used to randomly iterate the entire dataset (default = RandomSampler()),
|
||||
/// supported sampler list: SubsetRandomSampler, PkSampler, RandomSampler, SequentialSampler, DistributedSampler.
|
||||
/// \param[in] padded_sample Samples will be appended to dataset, where keys are the same as column_list.
|
||||
|
|
@ -862,14 +1136,42 @@ std::shared_ptr<MindDataDataset> MindData(const std::string &dataset_file,
|
|||
/// \return Shared pointer to the current MindDataDataset
|
||||
std::shared_ptr<MindDataDataset> MindData(const std::vector<std::string> &dataset_files,
|
||||
const std::vector<std::string> &columns_list = {},
|
||||
const std::shared_ptr<SamplerObj> &sampler = RandomSampler(),
|
||||
const std::shared_ptr<Sampler> &sampler = std::make_shared<RandomSampler>(),
|
||||
nlohmann::json padded_sample = nullptr, int64_t num_padded = 0);
|
||||
/// \brief Function to create a MindDataDataset
|
||||
/// \param[in] dataset_files List of dataset files to be read directly.
|
||||
/// \param[in] columns_list List of columns to be read
|
||||
/// \param[in] sampler Raw pointer to a sampler object used to choose samples from the dataset.
|
||||
/// supported sampler list: SubsetRandomSampler, PkSampler, RandomSampler, SequentialSampler, DistributedSampler.
|
||||
/// \param[in] padded_sample Samples will be appended to dataset, where keys are the same as column_list.
|
||||
/// \param[in] num_padded Number of padding samples. Dataset size plus num_padded should be divisible by num_shards.
|
||||
/// \return Shared pointer to the current MindDataDataset
|
||||
std::shared_ptr<MindDataDataset> MindData(const std::vector<std::string> &dataset_files,
|
||||
const std::vector<std::string> &columns_list, Sampler *sampler,
|
||||
nlohmann::json padded_sample = nullptr, int64_t num_padded = 0);
|
||||
/// \brief Function to create a MindDataDataset
|
||||
/// \param[in] dataset_files List of dataset files to be read directly.
|
||||
/// \param[in] columns_list List of columns to be read
|
||||
/// \param[in] sampler Sampler object used to choose samples from the dataset.
|
||||
/// supported sampler list: SubsetRandomSampler, PkSampler, RandomSampler, SequentialSampler, DistributedSampler.
|
||||
/// \param[in] padded_sample Samples will be appended to dataset, where keys are the same as column_list.
|
||||
/// \param[in] num_padded Number of padding samples. Dataset size plus num_padded should be divisible by num_shards.
|
||||
/// \return Shared pointer to the current MindDataDataset
|
||||
std::shared_ptr<MindDataDataset> MindData(const std::vector<std::string> &dataset_files,
|
||||
const std::vector<std::string> &columns_list,
|
||||
const std::reference_wrapper<Sampler> sampler,
|
||||
nlohmann::json padded_sample = nullptr, int64_t num_padded = 0);
|
||||
#endif
|
||||
|
||||
class MnistDataset : public Dataset {
|
||||
public:
|
||||
explicit MnistDataset(const std::string &dataset_dir, const std::string &usage = "all",
|
||||
const std::shared_ptr<SamplerObj> &sampler = RandomSampler(),
|
||||
const std::shared_ptr<Sampler> &sampler = std::make_shared<RandomSampler>(),
|
||||
const std::shared_ptr<DatasetCache> &cache = nullptr);
|
||||
explicit MnistDataset(const std::string &dataset_dir, const std::string &usage, Sampler *sampler,
|
||||
const std::shared_ptr<DatasetCache> &cache = nullptr);
|
||||
explicit MnistDataset(const std::string &dataset_dir, const std::string &usage,
|
||||
const std::reference_wrapper<Sampler> sampler,
|
||||
const std::shared_ptr<DatasetCache> &cache = nullptr);
|
||||
~MnistDataset() = default;
|
||||
};
|
||||
|
|
@ -878,12 +1180,32 @@ class MnistDataset : public Dataset {
|
|||
/// \notes The generated dataset has two columns ["image", "label"]
|
||||
/// \param[in] dataset_dir Path to the root directory that contains the dataset
|
||||
/// \param[in] usage of MNIST, can be "train", "test" or "all" (default = "all").
|
||||
/// \param[in] sampler Object used to choose samples from the dataset. If sampler is not given,
|
||||
/// \param[in] sampler Shared pointer to a sampler object used to choose samples from the dataset. If sampler is not
|
||||
/// given,
|
||||
/// a `RandomSampler` will be used to randomly iterate the entire dataset (default = RandomSampler())
|
||||
/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used).
|
||||
/// \return Shared pointer to the current MnistDataset
|
||||
std::shared_ptr<MnistDataset> Mnist(const std::string &dataset_dir, const std::string &usage = "all",
|
||||
const std::shared_ptr<SamplerObj> &sampler = RandomSampler(),
|
||||
const std::shared_ptr<Sampler> &sampler = std::make_shared<RandomSampler>(),
|
||||
const std::shared_ptr<DatasetCache> &cache = nullptr);
|
||||
/// \brief Function to create a MnistDataset
|
||||
/// \notes The generated dataset has two columns ["image", "label"]
|
||||
/// \param[in] dataset_dir Path to the root directory that contains the dataset
|
||||
/// \param[in] usage of MNIST, can be "train", "test" or "all"
|
||||
/// \param[in] sampler Raw pointer to a sampler object used to choose samples from the dataset.
|
||||
/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used).
|
||||
/// \return Shared pointer to the current MnistDataset
|
||||
std::shared_ptr<MnistDataset> Mnist(const std::string &dataset_dir, const std::string &usage, Sampler *sampler,
|
||||
const std::shared_ptr<DatasetCache> &cache = nullptr);
|
||||
/// \brief Function to create a MnistDataset
|
||||
/// \notes The generated dataset has two columns ["image", "label"]
|
||||
/// \param[in] dataset_dir Path to the root directory that contains the dataset
|
||||
/// \param[in] usage of MNIST, can be "train", "test" or "all"
|
||||
/// \param[in] sampler Sampler object used to choose samples from the dataset.
|
||||
/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used).
|
||||
/// \return Shared pointer to the current MnistDataset
|
||||
std::shared_ptr<MnistDataset> Mnist(const std::string &dataset_dir, const std::string &usage,
|
||||
const std::reference_wrapper<Sampler> sampler,
|
||||
const std::shared_ptr<DatasetCache> &cache = nullptr);
|
||||
#ifndef ENABLE_ANDROID
|
||||
/// \brief Function to create a ConcatDataset
|
||||
|
|
@ -909,8 +1231,6 @@ class RandomDataDataset : public Dataset {
|
|||
/// \param[in] total_rows Number of rows for the dataset to generate (default=0, number of rows is random)
|
||||
/// \param[in] schema SchemaObj to set column type, data type and data shape
|
||||
/// \param[in] columns_list List of columns to be read (default={}, read all columns)
|
||||
/// \param[in] sampler Object used to choose samples from the dataset. If sampler is not given,
|
||||
/// a `RandomSampler` will be used to randomly iterate the entire dataset (default = RandomSampler())
|
||||
/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used).
|
||||
/// \return Shared pointer to the current Dataset
|
||||
template <typename T = std::shared_ptr<SchemaObj>>
|
||||
|
|
@ -1027,7 +1347,14 @@ class VOCDataset : public Dataset {
|
|||
public:
|
||||
explicit VOCDataset(const std::string &dataset_dir, const std::string &task = "Segmentation",
|
||||
const std::string &usage = "train", const std::map<std::string, int32_t> &class_indexing = {},
|
||||
bool decode = false, const std::shared_ptr<SamplerObj> &sampler = RandomSampler(),
|
||||
bool decode = false, const std::shared_ptr<Sampler> &sampler = std::make_shared<RandomSampler>(),
|
||||
const std::shared_ptr<DatasetCache> &cache = nullptr);
|
||||
explicit VOCDataset(const std::string &dataset_dir, const std::string &task, const std::string &usage,
|
||||
const std::map<std::string, int32_t> &class_indexing, bool decode, Sampler *sampler,
|
||||
const std::shared_ptr<DatasetCache> &cache = nullptr);
|
||||
explicit VOCDataset(const std::string &dataset_dir, const std::string &task, const std::string &usage,
|
||||
const std::map<std::string, int32_t> &class_indexing, bool decode,
|
||||
const std::reference_wrapper<Sampler> sampler,
|
||||
const std::shared_ptr<DatasetCache> &cache = nullptr);
|
||||
~VOCDataset() = default;
|
||||
};
|
||||
|
|
@ -1042,14 +1369,48 @@ class VOCDataset : public Dataset {
|
|||
/// \param[in] usage The type of data list text file to be read (default = "train").
|
||||
/// \param[in] class_indexing A str-to-int mapping from label name to index, only valid in "Detection" task
|
||||
/// \param[in] decode Decode the images after reading
|
||||
/// \param[in] sampler Object used to choose samples from the dataset. If sampler is not given,
|
||||
/// \param[in] sampler Shared pointer to a sampler object used to choose samples from the dataset. If sampler is not
|
||||
/// given,
|
||||
/// a `RandomSampler` will be used to randomly iterate the entire dataset (default = RandomSampler())
|
||||
/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used).
|
||||
/// \return Shared pointer to the current Dataset
|
||||
std::shared_ptr<VOCDataset> VOC(const std::string &dataset_dir, const std::string &task = "Segmentation",
|
||||
const std::string &usage = "train",
|
||||
const std::map<std::string, int32_t> &class_indexing = {}, bool decode = false,
|
||||
const std::shared_ptr<SamplerObj> &sampler = RandomSampler(),
|
||||
const std::shared_ptr<Sampler> &sampler = std::make_shared<RandomSampler>(),
|
||||
const std::shared_ptr<DatasetCache> &cache = nullptr);
|
||||
/// \brief Function to create a VOCDataset
|
||||
/// \notes The generated dataset has multi-columns :
|
||||
/// - task='Detection', column: [['image', dtype=uint8], ['bbox', dtype=float32], ['label', dtype=uint32],
|
||||
/// ['difficult', dtype=uint32], ['truncate', dtype=uint32]].
|
||||
/// - task='Segmentation', column: [['image', dtype=uint8], ['target',dtype=uint8]].
|
||||
/// \param[in] dataset_dir Path to the root directory that contains the dataset
|
||||
/// \param[in] task Set the task type of reading voc data, now only support "Segmentation" or "Detection"
|
||||
/// \param[in] usage The type of data list text file to be read.
|
||||
/// \param[in] class_indexing A str-to-int mapping from label name to index, only valid in "Detection" task
|
||||
/// \param[in] decode Decode the images after reading
|
||||
/// \param[in] sampler Raw pointer to a sampler object used to choose samples from the dataset.
|
||||
/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used).
|
||||
/// \return Shared pointer to the current Dataset
|
||||
std::shared_ptr<VOCDataset> VOC(const std::string &dataset_dir, const std::string &task, const std::string &usage,
|
||||
const std::map<std::string, int32_t> &class_indexing, bool decode, Sampler *sampler,
|
||||
const std::shared_ptr<DatasetCache> &cache = nullptr);
|
||||
/// \brief Function to create a VOCDataset
|
||||
/// \notes The generated dataset has multi-columns :
|
||||
/// - task='Detection', column: [['image', dtype=uint8], ['bbox', dtype=float32], ['label', dtype=uint32],
|
||||
/// ['difficult', dtype=uint32], ['truncate', dtype=uint32]].
|
||||
/// - task='Segmentation', column: [['image', dtype=uint8], ['target',dtype=uint8]].
|
||||
/// \param[in] dataset_dir Path to the root directory that contains the dataset
|
||||
/// \param[in] task Set the task type of reading voc data, now only support "Segmentation" or "Detection"
|
||||
/// \param[in] usage The type of data list text file to be read.
|
||||
/// \param[in] class_indexing A str-to-int mapping from label name to index, only valid in "Detection" task
|
||||
/// \param[in] decode Decode the images after reading
|
||||
/// \param[in] sampler Sampler object used to choose samples from the dataset.
|
||||
/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used).
|
||||
/// \return Shared pointer to the current Dataset
|
||||
std::shared_ptr<VOCDataset> VOC(const std::string &dataset_dir, const std::string &task, const std::string &usage,
|
||||
const std::map<std::string, int32_t> &class_indexing, bool decode,
|
||||
const std::reference_wrapper<Sampler> sampler,
|
||||
const std::shared_ptr<DatasetCache> &cache = nullptr);
|
||||
|
||||
/// \brief Function the create a cache to be attached to a dataset
|
||||
|
|
|
|||
|
|
@ -26,76 +26,225 @@
|
|||
namespace mindspore {
|
||||
namespace dataset {
|
||||
|
||||
class DistributedSamplerObj;
|
||||
class PKSamplerObj;
|
||||
class PreBuiltSamplerObj;
|
||||
class RandomSamplerObj;
|
||||
class SequentialSamplerObj;
|
||||
class SubsetSamplerObj;
|
||||
class SubsetRandomSamplerObj;
|
||||
class WeightedRandomSamplerObj;
|
||||
// Abstract class to represent a sampler in the data pipeline.
|
||||
/// \class Sampler samplers.h
|
||||
/// \brief An abstract base class to represent a sampler in the data pipeline.
|
||||
class Sampler : std::enable_shared_from_this<Sampler> {
|
||||
friend class AlbumDataset;
|
||||
friend class CelebADataset;
|
||||
friend class Cifar10Dataset;
|
||||
friend class Cifar100Dataset;
|
||||
friend class CLUEDataset;
|
||||
friend class CocoDataset;
|
||||
friend class CSVDataset;
|
||||
friend class ImageFolderDataset;
|
||||
friend class ManifestDataset;
|
||||
friend class MindDataDataset;
|
||||
friend class MnistDataset;
|
||||
friend class RandomDataDataset;
|
||||
friend class TextFileDataset;
|
||||
friend class TFRecordDataset;
|
||||
friend class VOCDataset;
|
||||
friend std::shared_ptr<SamplerObj> SelectSampler(int64_t, bool, int32_t, int32_t);
|
||||
|
||||
/// Function to create a Distributed Sampler.
|
||||
/// \notes A Sampler that access a shard of the dataset.
|
||||
/// \param[in] num_shards - Number of shards to divide the dataset into.
|
||||
/// \param[in] shard_id - Shard ID of the current shard within num_shards.
|
||||
/// \param[in] shuffle - If true, the indices are shuffled.
|
||||
/// \param[in] num_samples - The number of samples to draw (default to all elements).
|
||||
/// \param[in] seed - The seed in use when shuffle is true.
|
||||
/// \param[in] offset - The starting position where access to elements in the dataset begins.
|
||||
/// \param[in] even_dist - If true, each shard would return the same number of rows (default to true).
|
||||
/// If false the total rows returned by all the shards would not have overlap.
|
||||
/// \return Shared pointer to the current Sampler.
|
||||
std::shared_ptr<DistributedSamplerObj> DistributedSampler(int64_t num_shards, int64_t shard_id, bool shuffle = true,
|
||||
int64_t num_samples = 0, uint32_t seed = 1,
|
||||
int64_t offset = -1, bool even_dist = true);
|
||||
public:
|
||||
/// \brief Constructor
|
||||
Sampler() {}
|
||||
|
||||
/// Function to create a PK Sampler.
|
||||
/// \brief Destructor
|
||||
~Sampler() = default;
|
||||
|
||||
/// \brief A virtual function to add a child sampler.
|
||||
/// \param[in] child The child sampler to be added as a children of this sampler.
|
||||
virtual void AddChild(std::shared_ptr<Sampler> child) { children_.push_back(child); }
|
||||
|
||||
protected:
|
||||
/// \brief Pure virtual function to convert a Sampler class into an IR Sampler object.
|
||||
/// \return shared pointer to the newly created TensorOperation.
|
||||
virtual std::shared_ptr<SamplerObj> Parse() = 0;
|
||||
|
||||
std::vector<std::shared_ptr<Sampler>> children_;
|
||||
};
|
||||
|
||||
/// \brief A class to represent a Distributed Sampler in the data pipeline.
|
||||
/// \notes A Sampler that accesses a shard of the dataset.
|
||||
class DistributedSampler : public Sampler {
|
||||
friend std::shared_ptr<SamplerObj> SelectSampler(int64_t, bool, int32_t, int32_t);
|
||||
|
||||
public:
|
||||
/// \brief Constructor
|
||||
/// \param[in] num_shards - Number of shards to divide the dataset into.
|
||||
/// \param[in] shard_id - Shard ID of the current shard within num_shards.
|
||||
/// \param[in] shuffle - If true, the indices are shuffled.
|
||||
/// \param[in] num_samples - The number of samples to draw (default to all elements).
|
||||
/// \param[in] seed - The seed in use when shuffle is true.
|
||||
/// \param[in] offset - The starting position where access to elements in the dataset begins.
|
||||
/// \param[in] even_dist - If true, each shard would return the same number of rows (default to true).
|
||||
/// If false the total rows returned by all the shards would not have overlap.
|
||||
explicit DistributedSampler(int64_t num_shards, int64_t shard_id, bool shuffle = true, int64_t num_samples = 0,
|
||||
uint32_t seed = 1, int64_t offset = -1, bool even_dist = true);
|
||||
/// \brief Destructor.
|
||||
~DistributedSampler() = default;
|
||||
|
||||
protected:
|
||||
/// \brief Function to convert a Sampler into an IR SamplerObj.
|
||||
/// \return shared pointer to the newly created SamplerObj.
|
||||
std::shared_ptr<SamplerObj> Parse() override;
|
||||
|
||||
private:
|
||||
int64_t num_shards_;
|
||||
int64_t shard_id_;
|
||||
bool shuffle_;
|
||||
int64_t num_samples_;
|
||||
uint32_t seed_;
|
||||
int64_t offset_;
|
||||
bool even_dist_;
|
||||
};
|
||||
|
||||
/// \brief A class to represent a PK Sampler in the data pipeline.
|
||||
/// \notes Samples K elements for each P class in the dataset.
|
||||
/// This will sample all classes.
|
||||
/// \param[in] num_val - Number of elements to sample for each class.
|
||||
/// \param[in] shuffle - If true, the class IDs are shuffled.
|
||||
/// \param[in] num_samples - The number of samples to draw (default to all elements).
|
||||
/// \return Shared pointer to the current Sampler.
|
||||
std::shared_ptr<PKSamplerObj> PKSampler(int64_t num_val, bool shuffle = false, int64_t num_samples = 0);
|
||||
class PKSampler : public Sampler {
|
||||
friend std::shared_ptr<SamplerObj> SelectSampler(int64_t, bool, int32_t, int32_t);
|
||||
|
||||
/// Function to create a Random Sampler.
|
||||
public:
|
||||
/// \brief Constructor
|
||||
/// \param[in] num_val - Number of elements to sample for each class.
|
||||
/// \param[in] shuffle - If true, the class IDs are shuffled.
|
||||
/// \param[in] num_samples - The number of samples to draw (default to all elements).
|
||||
explicit PKSampler(int64_t num_val, bool shuffle = false, int64_t num_samples = 0);
|
||||
|
||||
/// \brief Destructor.
|
||||
~PKSampler() = default;
|
||||
|
||||
protected:
|
||||
/// \brief Function to convert a Sampler into an IR SamplerObj.
|
||||
/// \return shared pointer to the newly created SamplerObj.
|
||||
std::shared_ptr<SamplerObj> Parse() override;
|
||||
|
||||
private:
|
||||
int64_t num_val_;
|
||||
bool shuffle_;
|
||||
int64_t num_samples_;
|
||||
};
|
||||
|
||||
/// \brief A class to represent a Random Sampler in the data pipeline.
|
||||
/// \notes Samples the elements randomly.
|
||||
/// \param[in] replacement - If true, put the sample ID back for the next draw.
|
||||
/// \param[in] num_samples - The number of samples to draw (default to all elements).
|
||||
/// \return Shared pointer to the current Sampler.
|
||||
std::shared_ptr<RandomSamplerObj> RandomSampler(bool replacement = false, int64_t num_samples = 0);
|
||||
class RandomSampler : public Sampler {
|
||||
friend std::shared_ptr<SamplerObj> SelectSampler(int64_t, bool, int32_t, int32_t);
|
||||
|
||||
/// Function to create a Sequential Sampler.
|
||||
public:
|
||||
/// \brief Constructor
|
||||
/// \param[in] replacement - If true, put the sample ID back for the next draw.
|
||||
/// \param[in] num_samples - The number of samples to draw (default to all elements).
|
||||
explicit RandomSampler(bool replacement = false, int64_t num_samples = 0);
|
||||
|
||||
/// \brief Destructor.
|
||||
~RandomSampler() = default;
|
||||
|
||||
protected:
|
||||
/// \brief Function to convert a Sampler into an IR SamplerObj.
|
||||
/// \return shared pointer to the newly created SamplerObj.
|
||||
std::shared_ptr<SamplerObj> Parse() override;
|
||||
|
||||
private:
|
||||
bool replacement_;
|
||||
int64_t num_samples_;
|
||||
};
|
||||
|
||||
/// \brief A class to represent a Sequential Sampler in the data pipeline.
|
||||
/// \notes Samples the dataset elements sequentially, same as not having a sampler.
|
||||
/// \param[in] start_index - Index to start sampling at (default to start at first id).
|
||||
/// \param[in] num_samples - The number of samples to draw (default to all elements).
|
||||
/// \return Shared pointer to the current Sampler.
|
||||
std::shared_ptr<SequentialSamplerObj> SequentialSampler(int64_t start_index = 0, int64_t num_samples = 0);
|
||||
class SequentialSampler : public Sampler {
|
||||
friend std::shared_ptr<SamplerObj> SelectSampler(int64_t, bool, int32_t, int32_t);
|
||||
|
||||
/// Function to create a Subset Sampler.
|
||||
public:
|
||||
/// \brief Constructor
|
||||
/// \param[in] start_index - Index to start sampling at (default to start at first id).
|
||||
/// \param[in] num_samples - The number of samples to draw (default to all elements).
|
||||
explicit SequentialSampler(int64_t start_index = 0, int64_t num_samples = 0);
|
||||
|
||||
/// \brief Destructor.
|
||||
~SequentialSampler() = default;
|
||||
|
||||
protected:
|
||||
/// \brief Function to convert a Sampler into an IR SamplerObj.
|
||||
/// \return shared pointer to the newly created SamplerObj.
|
||||
std::shared_ptr<SamplerObj> Parse() override;
|
||||
|
||||
private:
|
||||
int64_t start_index_;
|
||||
int64_t num_samples_;
|
||||
};
|
||||
|
||||
/// \brief A class to represent a Subset Sampler in the data pipeline.
|
||||
/// \notes Samples the elements from a sequence of indices.
|
||||
/// \param[in] indices - A vector sequence of indices.
|
||||
/// \param[in] num_samples - The number of samples to draw (default to all elements).
|
||||
/// \return Shared pointer to the current Sampler.
|
||||
std::shared_ptr<SubsetSamplerObj> SubsetSampler(std::vector<int64_t> indices, int64_t num_samples = 0);
|
||||
class SubsetSampler : public Sampler {
|
||||
friend std::shared_ptr<SamplerObj> SelectSampler(int64_t, bool, int32_t, int32_t);
|
||||
|
||||
/// Function to create a Subset Random Sampler.
|
||||
public:
|
||||
/// \brief Constructor
|
||||
/// \param[in] indices - A vector sequence of indices.
|
||||
/// \param[in] num_samples - The number of samples to draw (default to all elements).
|
||||
explicit SubsetSampler(std::vector<int64_t> indices, int64_t num_samples = 0);
|
||||
|
||||
/// \brief Destructor.
|
||||
~SubsetSampler() = default;
|
||||
|
||||
protected:
|
||||
/// \brief Function to convert a Sampler into an IR SamplerObj.
|
||||
/// \return shared pointer to the newly created SamplerObj.
|
||||
std::shared_ptr<SamplerObj> Parse() override;
|
||||
|
||||
std::vector<int64_t> indices_;
|
||||
int64_t num_samples_;
|
||||
};
|
||||
|
||||
/// \brief A class to represent a Subset Random Sampler in the data pipeline.
|
||||
/// \notes Samples the elements randomly from a sequence of indices.
|
||||
/// \param[in] indices - A vector sequence of indices.
|
||||
/// \param[in] num_samples - The number of samples to draw (default to all elements).
|
||||
/// \return Shared pointer to the current Sampler.
|
||||
std::shared_ptr<SubsetRandomSamplerObj> SubsetRandomSampler(std::vector<int64_t> indices, int64_t num_samples = 0);
|
||||
class SubsetRandomSampler : public SubsetSampler {
|
||||
friend std::shared_ptr<SamplerObj> SelectSampler(int64_t, bool, int32_t, int32_t);
|
||||
|
||||
/// Function to create a Weighted Random Sampler.
|
||||
public:
|
||||
/// \brief Constructor
|
||||
/// \param[in] indices - A vector sequence of indices.
|
||||
/// \param[in] num_samples - The number of samples to draw (default to all elements).
|
||||
explicit SubsetRandomSampler(std::vector<int64_t> indices, int64_t num_samples = 0);
|
||||
|
||||
/// \brief Destructor.
|
||||
~SubsetRandomSampler() = default;
|
||||
|
||||
protected:
|
||||
/// \brief Function to convert a Sampler into an IR SamplerObj.
|
||||
/// \return shared pointer to the newly created SamplerObj.
|
||||
std::shared_ptr<SamplerObj> Parse() override;
|
||||
};
|
||||
|
||||
/// \brief A class to represent a Weighted Random Sampler in the data pipeline.
|
||||
/// \notes Samples the elements from [0, len(weights) - 1] randomly with the given
|
||||
/// weights (probabilities).
|
||||
/// \param[in] weights - A vector sequence of weights, not necessarily summing up to 1.
|
||||
/// \param[in] num_samples - The number of samples to draw (default to all elements).
|
||||
/// \param[in] replacement - If true, put the sample ID back for the next draw.
|
||||
/// \return Shared pointer to the current Sampler.
|
||||
std::shared_ptr<WeightedRandomSamplerObj> WeightedRandomSampler(std::vector<double> weights, int64_t num_samples = 0,
|
||||
bool replacement = true);
|
||||
class WeightedRandomSampler : public Sampler {
|
||||
friend std::shared_ptr<SamplerObj> SelectSampler(int64_t, bool, int32_t, int32_t);
|
||||
|
||||
public:
|
||||
/// \brief Constructor
|
||||
/// \param[in] weights - A vector sequence of weights, not necessarily summing up to 1.
|
||||
/// \param[in] num_samples - The number of samples to draw (default to all elements).
|
||||
/// \param[in] replacement - If true, put the sample ID back for the next draw.
|
||||
explicit WeightedRandomSampler(std::vector<double> weights, int64_t num_samples = 0, bool replacement = true);
|
||||
|
||||
/// \brief Destructor.
|
||||
~WeightedRandomSampler() = default;
|
||||
|
||||
protected:
|
||||
/// \brief Function to convert a Sampler into an IR SamplerObj.
|
||||
/// \return shared pointer to the newly created SamplerObj.
|
||||
std::shared_ptr<SamplerObj> Parse() override;
|
||||
|
||||
private:
|
||||
std::vector<double> weights_;
|
||||
int64_t num_samples_;
|
||||
bool replacement_;
|
||||
};
|
||||
|
||||
} // namespace dataset
|
||||
} // namespace mindspore
|
||||
|
|
|
|||
|
|
@ -57,7 +57,7 @@ TEST_F(MindDataTestCacheOp, DISABLED_TestCacheCApiNestedCache) {
|
|||
|
||||
// Create an ImageFolder Dataset, this folder_path only has 2 images in it
|
||||
std::string folder_path = datasets_root_path_ + "/testImageNetData/train/";
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, false, RandomSampler(), {}, {}, some_cache);
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, false, std::make_shared<RandomSampler>(), {}, {}, some_cache);
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create objects for the tensor ops
|
||||
|
|
@ -85,7 +85,7 @@ TEST_F(MindDataTestCacheOp, DISABLED_TestCacheImageFolderCApi) {
|
|||
|
||||
// Create an ImageFolder Dataset, this folder_path only has 2 images in it
|
||||
std::string folder_path = datasets_root_path_ + "/testImageNetData/train/";
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, false, RandomSampler(), {}, {}, some_cache);
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, false, std::make_shared<RandomSampler>(), {}, {}, some_cache);
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create a Repeat operation on ds
|
||||
|
|
@ -128,7 +128,7 @@ TEST_F(MindDataTestCacheOp, DISABLED_TestCacheCocoCApi) {
|
|||
std::string folder_path = datasets_root_path_ + "/testCOCO/train/";
|
||||
std::string annotation_file_path = datasets_root_path_ + "/testCOCO/annotations/train.json";
|
||||
std::shared_ptr<Dataset> ds =
|
||||
Coco(folder_path, annotation_file_path, "Detection", false, RandomSampler(), some_cache);
|
||||
Coco(folder_path, annotation_file_path, "Detection", false, std::make_shared<RandomSampler>(), some_cache);
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create a Repeat operation on ds
|
||||
|
|
@ -169,7 +169,7 @@ TEST_F(MindDataTestCacheOp, DISABLED_TestCacheMnistCApi) {
|
|||
|
||||
// Create a Mnist Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testMnistData/";
|
||||
std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", RandomSampler(false, 10), some_cache);
|
||||
std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", std::make_shared<RandomSampler>(false, 10), some_cache);
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create a Repeat operation on ds
|
||||
|
|
@ -210,7 +210,7 @@ TEST_F(MindDataTestCacheOp, DISABLED_TestCacheCelebaCApi) {
|
|||
|
||||
// Create a CelebA Dataset, this folder_path has 4 records in it
|
||||
std::string folder_path = datasets_root_path_ + "/testCelebAData/";
|
||||
std::shared_ptr<Dataset> ds = CelebA(folder_path, "all", RandomSampler(false, 10), false, {}, some_cache);
|
||||
std::shared_ptr<Dataset> ds = CelebA(folder_path, "all", std::make_shared<RandomSampler>(false, 10), false, {}, some_cache);
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create a Repeat operation on ds
|
||||
|
|
@ -251,7 +251,7 @@ TEST_F(MindDataTestCacheOp, DISABLED_TestCacheManifestCApi) {
|
|||
|
||||
// Create a Manifest Dataset, this file_path has 2 records in it
|
||||
std::string file_path = datasets_root_path_ + "/testManifestData/cpp.json";
|
||||
std::shared_ptr<Dataset> ds = Manifest(file_path, "train", RandomSampler(), {}, false, some_cache);
|
||||
std::shared_ptr<Dataset> ds = Manifest(file_path, "train", std::make_shared<RandomSampler>(), {}, false, some_cache);
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create a Repeat operation on ds
|
||||
|
|
@ -292,7 +292,7 @@ TEST_F(MindDataTestCacheOp, DISABLED_TestCacheCifar10CApi) {
|
|||
|
||||
// Create a Cifar10 Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
|
||||
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10), some_cache);
|
||||
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10), some_cache);
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create a Repeat operation on ds
|
||||
|
|
@ -333,7 +333,7 @@ TEST_F(MindDataTestCacheOp, DISABLED_TestCacheCifar100CApi) {
|
|||
|
||||
// Create a Cifar100 Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testCifar100Data/";
|
||||
std::shared_ptr<Dataset> ds = Cifar100(folder_path, "all", RandomSampler(false, 10), some_cache);
|
||||
std::shared_ptr<Dataset> ds = Cifar100(folder_path, "all", std::make_shared<RandomSampler>(false, 10), some_cache);
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create a Repeat operation on ds
|
||||
|
|
@ -374,7 +374,7 @@ TEST_F(MindDataTestCacheOp, DISABLED_TestCacheVocCApi) {
|
|||
|
||||
// Create a VOC Dataset, this folder_path has 9 records in it
|
||||
std::string folder_path = datasets_root_path_ + "/testVOC2012/";
|
||||
std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", {}, false, RandomSampler(), some_cache);
|
||||
std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", {}, false, std::make_shared<RandomSampler>(), some_cache);
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create a Repeat operation on ds
|
||||
|
|
@ -417,7 +417,7 @@ TEST_F(MindDataTestCacheOp, DISABLED_TestCacheAlbumCApi) {
|
|||
std::string schema_file = datasets_root_path_ + "/testAlbum/datasetSchema.json";
|
||||
std::vector<std::string> column_names = {"image", "label", "id"};
|
||||
// Create a Album Dataset, 7 records in it
|
||||
std::shared_ptr<Dataset> ds = Album(folder_path, schema_file, column_names, false, RandomSampler(), some_cache);
|
||||
std::shared_ptr<Dataset> ds = Album(folder_path, schema_file, column_names, false, std::make_shared<RandomSampler>(), some_cache);
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create a Repeat operation on ds
|
||||
|
|
@ -770,9 +770,9 @@ TEST_F(MindDataTestCacheOp, DISABLED_TestCApiCacheShare1) {
|
|||
|
||||
// Create an ImageFolder Dataset, this folder_path only has 2 images in it
|
||||
std::string folder_path = datasets_root_path_ + "/testImageNetData/train/";
|
||||
std::shared_ptr<Dataset> ds1 = ImageFolder(folder_path, false, RandomSampler(), {}, {}, some_cache);
|
||||
std::shared_ptr<Dataset> ds1 = ImageFolder(folder_path, false, std::make_shared<RandomSampler>(), {}, {}, some_cache);
|
||||
EXPECT_NE(ds1, nullptr);
|
||||
std::shared_ptr<Dataset> ds2 = ImageFolder(folder_path, false, RandomSampler(), {}, {}, some_cache);
|
||||
std::shared_ptr<Dataset> ds2 = ImageFolder(folder_path, false, std::make_shared<RandomSampler>(), {}, {}, some_cache);
|
||||
EXPECT_NE(ds2, nullptr);
|
||||
|
||||
// Create and launch the Execution Tree for ds1
|
||||
|
|
@ -824,9 +824,9 @@ TEST_F(MindDataTestCacheOp, DISABLED_TestCApiCacheShare2) {
|
|||
std::string folder_path = datasets_root_path_ + "/testImageNetData/train/";
|
||||
// The first pipeline is ImageFolder with RandomSampler, the second pipeline is ImageFolder with SequentialSampler
|
||||
// Since sampler does not influence the data in the source, these two pipelines can share a common cache.
|
||||
std::shared_ptr<Dataset> ds1 = ImageFolder(folder_path, true, RandomSampler(), {}, {}, some_cache);
|
||||
std::shared_ptr<Dataset> ds1 = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(), {}, {}, some_cache);
|
||||
EXPECT_NE(ds1, nullptr);
|
||||
std::shared_ptr<Dataset> ds2 = ImageFolder(folder_path, true, SequentialSampler(), {}, {}, some_cache);
|
||||
std::shared_ptr<Dataset> ds2 = ImageFolder(folder_path, true, std::make_shared<SequentialSampler>(), {}, {}, some_cache);
|
||||
EXPECT_NE(ds2, nullptr);
|
||||
|
||||
// Create and launch the Execution Tree for ds1
|
||||
|
|
@ -874,9 +874,9 @@ TEST_F(MindDataTestCacheOp, DISABLED_TestCApiCacheShareFailure1) {
|
|||
|
||||
// Create an ImageFolder Dataset, this folder_path only has 2 images in it
|
||||
std::string folder_path = datasets_root_path_ + "/testImageNetData/train/";
|
||||
std::shared_ptr<Dataset> ds1 = ImageFolder(folder_path, true, RandomSampler(), {}, {}, some_cache);
|
||||
std::shared_ptr<Dataset> ds1 = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(), {}, {}, some_cache);
|
||||
EXPECT_NE(ds1, nullptr);
|
||||
std::shared_ptr<Dataset> ds2 = ImageFolder(folder_path, false, RandomSampler(), {}, {}, some_cache);
|
||||
std::shared_ptr<Dataset> ds2 = ImageFolder(folder_path, false, std::make_shared<RandomSampler>(), {}, {}, some_cache);
|
||||
EXPECT_NE(ds2, nullptr);
|
||||
|
||||
// Create and launch the Execution Tree for ds1
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
/**
|
||||
* Copyright 2020 Huawei Technologies Co., Ltd
|
||||
* Copyright 2020-2021 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
|
|
@ -174,7 +174,7 @@ TEST_F(MindDataTestPipeline, TestAlbumNumSamplers) {
|
|||
std::string schema_file = datasets_root_path_ + "/testAlbum/datasetSchema.json";
|
||||
std::vector<std::string> column_names = {"image", "label", "id"};
|
||||
// Create a Album Dataset
|
||||
std::shared_ptr<Dataset> ds = Album(folder_path, schema_file, column_names, true, SequentialSampler(0, 1));
|
||||
std::shared_ptr<Dataset> ds = Album(folder_path, schema_file, column_names, true, std::make_shared<SequentialSampler>(0, 1));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create an iterator over the result of the above dataset
|
||||
|
|
@ -206,7 +206,7 @@ TEST_F(MindDataTestPipeline, TestAlbumError) {
|
|||
std::string schema_file = datasets_root_path_ + "/testAlbum/datasetSchema.json";
|
||||
std::vector<std::string> column_names = {"image", "label", "id"};
|
||||
// Create an Album Dataset
|
||||
std::shared_ptr<Dataset> ds = Album(folder_path, schema_file, column_names, true, SequentialSampler(0, 1));
|
||||
std::shared_ptr<Dataset> ds = Album(folder_path, schema_file, column_names, true, std::make_shared<SequentialSampler>(0, 1));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create an iterator over the result of the above dataset
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
/**
|
||||
* Copyright 2020 Huawei Technologies Co., Ltd
|
||||
* Copyright 2020-2021 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
|
|
@ -30,7 +30,7 @@ TEST_F(MindDataTestPipeline, TestCifar10Dataset) {
|
|||
|
||||
// Create a Cifar10 Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
|
||||
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10));
|
||||
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create an iterator over the result of the above dataset
|
||||
|
|
@ -64,8 +64,8 @@ TEST_F(MindDataTestPipeline, TestCifar10DatasetWithPipeline) {
|
|||
|
||||
// Create two Cifar10 Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
|
||||
std::shared_ptr<Dataset> ds1 = Cifar10(folder_path, "all", RandomSampler(false, 10));
|
||||
std::shared_ptr<Dataset> ds2 = Cifar10(folder_path, "all", RandomSampler(false, 10));
|
||||
std::shared_ptr<Dataset> ds1 = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
|
||||
std::shared_ptr<Dataset> ds2 = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
|
||||
EXPECT_NE(ds1, nullptr);
|
||||
EXPECT_NE(ds2, nullptr);
|
||||
|
||||
|
|
@ -168,7 +168,7 @@ TEST_F(MindDataTestPipeline, TestCifar100Dataset) {
|
|||
|
||||
// Create a Cifar100 Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testCifar100Data/";
|
||||
std::shared_ptr<Dataset> ds = Cifar100(folder_path, "all", RandomSampler(false, 10));
|
||||
std::shared_ptr<Dataset> ds = Cifar100(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create an iterator over the result of the above dataset
|
||||
|
|
@ -203,7 +203,7 @@ TEST_F(MindDataTestPipeline, TestCifar100Getters) {
|
|||
|
||||
// Create a Cifar100 Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testCifar100Data/";
|
||||
std::shared_ptr<Dataset> ds = Cifar100(folder_path, "all", RandomSampler(false, 10));
|
||||
std::shared_ptr<Dataset> ds = Cifar100(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
std::vector<std::string> column_names = {"image", "coarse_label", "fine_label"};
|
||||
|
|
@ -230,7 +230,7 @@ TEST_F(MindDataTestPipeline, TestCifar100DatasetFail) {
|
|||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestCifar100DatasetFail.";
|
||||
|
||||
// Create a Cifar100 Dataset
|
||||
std::shared_ptr<Dataset> ds = Cifar100("", "all", RandomSampler(false, 10));
|
||||
std::shared_ptr<Dataset> ds = Cifar100("", "all", std::make_shared<RandomSampler>(false, 10));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create an iterator over the result of the above dataset
|
||||
|
|
@ -243,7 +243,7 @@ TEST_F(MindDataTestPipeline, TestCifar10DatasetFail) {
|
|||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestCifar10DatasetFail.";
|
||||
|
||||
// Create a Cifar10 Dataset
|
||||
std::shared_ptr<Dataset> ds = Cifar10("", "all", RandomSampler(false, 10));
|
||||
std::shared_ptr<Dataset> ds = Cifar10("", "all", std::make_shared<RandomSampler>(false, 10));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create an iterator over the result of the above dataset
|
||||
|
|
@ -299,7 +299,7 @@ TEST_F(MindDataTestPipeline, TestCifar100DatasetWithWrongSamplerFail) {
|
|||
|
||||
// Create a Cifar100 Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testCifar100Data/";
|
||||
std::shared_ptr<Dataset> ds = Cifar100(folder_path, "all", RandomSampler(false, -10));
|
||||
std::shared_ptr<Dataset> ds = Cifar100(folder_path, "all", std::make_shared<RandomSampler>(false, -10));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create an iterator over the result of the above dataset
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
/**
|
||||
* Copyright 2020 Huawei Technologies Co., Ltd
|
||||
* Copyright 2020-2021 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
|
|
@ -138,7 +138,7 @@ TEST_F(MindDataTestPipeline, TestCocoDetection) {
|
|||
std::string folder_path = datasets_root_path_ + "/testCOCO/train";
|
||||
std::string annotation_file = datasets_root_path_ + "/testCOCO/annotations/train.json";
|
||||
|
||||
std::shared_ptr<Dataset> ds = Coco(folder_path, annotation_file, "Detection", false, SequentialSampler(0, 6));
|
||||
std::shared_ptr<Dataset> ds = Coco(folder_path, annotation_file, "Detection", false, std::make_shared<SequentialSampler>(0, 6));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create an iterator over the result of the above dataset
|
||||
|
|
@ -220,7 +220,7 @@ TEST_F(MindDataTestPipeline, TestCocoKeypoint) {
|
|||
std::string folder_path = datasets_root_path_ + "/testCOCO/train";
|
||||
std::string annotation_file = datasets_root_path_ + "/testCOCO/annotations/key_point.json";
|
||||
|
||||
std::shared_ptr<Dataset> ds = Coco(folder_path, annotation_file, "Keypoint", false, SequentialSampler(0, 2));
|
||||
std::shared_ptr<Dataset> ds = Coco(folder_path, annotation_file, "Keypoint", false, std::make_shared<SequentialSampler>(0, 2));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create an iterator over the result of the above dataset
|
||||
|
|
@ -275,7 +275,7 @@ TEST_F(MindDataTestPipeline, TestCocoPanoptic) {
|
|||
std::string folder_path = datasets_root_path_ + "/testCOCO/train";
|
||||
std::string annotation_file = datasets_root_path_ + "/testCOCO/annotations/panoptic.json";
|
||||
|
||||
std::shared_ptr<Dataset> ds = Coco(folder_path, annotation_file, "Panoptic", false, SequentialSampler(0, 2));
|
||||
std::shared_ptr<Dataset> ds = Coco(folder_path, annotation_file, "Panoptic", false, std::make_shared<SequentialSampler>(0, 2));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create an iterator over the result of the above dataset
|
||||
|
|
@ -333,7 +333,7 @@ TEST_F(MindDataTestPipeline, TestCocoPanopticGetClassIndex) {
|
|||
std::string folder_path = datasets_root_path_ + "/testCOCO/train";
|
||||
std::string annotation_file = datasets_root_path_ + "/testCOCO/annotations/panoptic.json";
|
||||
|
||||
std::shared_ptr<Dataset> ds = Coco(folder_path, annotation_file, "Panoptic", false, SequentialSampler(0, 2));
|
||||
std::shared_ptr<Dataset> ds = Coco(folder_path, annotation_file, "Panoptic", false, std::make_shared<SequentialSampler>(0, 2));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
std::vector<std::pair<std::string, std::vector<int32_t>>> class_index1 = ds->GetClassIndexing();
|
||||
|
|
@ -355,7 +355,7 @@ TEST_F(MindDataTestPipeline, TestCocoStuff) {
|
|||
std::string folder_path = datasets_root_path_ + "/testCOCO/train";
|
||||
std::string annotation_file = datasets_root_path_ + "/testCOCO/annotations/train.json";
|
||||
|
||||
std::shared_ptr<Dataset> ds = Coco(folder_path, annotation_file, "Stuff", false, SequentialSampler(0, 6));
|
||||
std::shared_ptr<Dataset> ds = Coco(folder_path, annotation_file, "Stuff", false, std::make_shared<SequentialSampler>(0, 6));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create an iterator over the result of the above dataset
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
|
||||
/**
|
||||
* Copyright 2020 Huawei Technologies Co., Ltd
|
||||
* Copyright 2020-2021 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
|
|
@ -29,7 +29,7 @@ TEST_F(MindDataTestPipeline, TestIteratorEmptyColumn) {
|
|||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestIteratorEmptyColumn.";
|
||||
// Create a Cifar10 Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
|
||||
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 5));
|
||||
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 5));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create a Rename operation on ds
|
||||
|
|
@ -65,7 +65,7 @@ TEST_F(MindDataTestPipeline, TestIteratorOneColumn) {
|
|||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestIteratorOneColumn.";
|
||||
// Create a Mnist Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testMnistData/";
|
||||
std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", RandomSampler(false, 4));
|
||||
std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", std::make_shared<RandomSampler>(false, 4));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create a Batch operation on ds
|
||||
|
|
@ -104,7 +104,7 @@ TEST_F(MindDataTestPipeline, TestIteratorReOrder) {
|
|||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestIteratorReOrder.";
|
||||
// Create a Cifar10 Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
|
||||
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", SequentialSampler(false, 4));
|
||||
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<SequentialSampler>(false, 4));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create a Take operation on ds
|
||||
|
|
@ -144,7 +144,7 @@ TEST_F(MindDataTestPipeline, TestIteratorTwoColumns) {
|
|||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestIteratorTwoColumns.";
|
||||
// Create a VOC Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testVOC2012_2";
|
||||
std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", {}, false, SequentialSampler(0, 4));
|
||||
std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", {}, false, std::make_shared<SequentialSampler>(0, 4));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create a Repeat operation on ds
|
||||
|
|
@ -187,7 +187,7 @@ TEST_F(MindDataTestPipeline, TestIteratorWrongColumn) {
|
|||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestIteratorOneColumn.";
|
||||
// Create a Mnist Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testMnistData/";
|
||||
std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", RandomSampler(false, 4));
|
||||
std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", std::make_shared<RandomSampler>(false, 4));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Pass wrong column name
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
/**
|
||||
* Copyright 2020 Huawei Technologies Co., Ltd
|
||||
* Copyright 2020-2021 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
|
|
@ -147,7 +147,7 @@ TEST_F(MindDataTestPipeline, TestManifestDecode) {
|
|||
|
||||
std::string file_path = datasets_root_path_ + "/testManifestData/cpp.json";
|
||||
// Create a Manifest Dataset
|
||||
std::shared_ptr<Dataset> ds = Manifest(file_path, "train", RandomSampler(), {}, true);
|
||||
std::shared_ptr<Dataset> ds = Manifest(file_path, "train", std::make_shared<RandomSampler>(), {}, true);
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create an iterator over the result of the above dataset
|
||||
|
|
@ -218,7 +218,7 @@ TEST_F(MindDataTestPipeline, TestManifestClassIndex) {
|
|||
std::vector<int64_t> expected_label = {111, 222};
|
||||
|
||||
// Create a Manifest Dataset
|
||||
std::shared_ptr<Dataset> ds = Manifest(file_path, "train", RandomSampler(), map, true);
|
||||
std::shared_ptr<Dataset> ds = Manifest(file_path, "train", std::make_shared<RandomSampler>(), map, true);
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
std::vector<std::pair<std::string, std::vector<int32_t>>> class_index1 = ds->GetClassIndexing();
|
||||
|
|
@ -261,7 +261,7 @@ TEST_F(MindDataTestPipeline, TestManifestNumSamplers) {
|
|||
|
||||
std::string file_path = datasets_root_path_ + "/testManifestData/cpp.json";
|
||||
// Create a Manifest Dataset
|
||||
std::shared_ptr<Dataset> ds = Manifest(file_path, "train", SequentialSampler(0, 1), {}, true);
|
||||
std::shared_ptr<Dataset> ds = Manifest(file_path, "train", std::make_shared<SequentialSampler>(0, 1), {}, true);
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create an iterator over the result of the above dataset
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
/**
|
||||
* Copyright 2020 Huawei Technologies Co., Ltd
|
||||
* Copyright 2020-2021 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
|
|
@ -182,7 +182,7 @@ TEST_F(MindDataTestPipeline, TestMindDataSuccess5) {
|
|||
// Pass one mindrecord shard file to parse dataset info, and search for other mindrecord files with same dataset info,
|
||||
// thus all records in imagenet.mindrecord0 ~ imagenet.mindrecord3 will be read
|
||||
std::string file_path1 = datasets_root_path_ + "/../mindrecord/testMindDataSet/testImageNetData/imagenet.mindrecord0";
|
||||
std::shared_ptr<Dataset> ds = MindData(file_path1, {}, SequentialSampler(0, 3));
|
||||
std::shared_ptr<Dataset> ds = MindData(file_path1, {}, std::make_shared<SequentialSampler>(0, 3));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create an iterator over the result of the above dataset
|
||||
|
|
@ -223,26 +223,26 @@ TEST_F(MindDataTestPipeline, TestMindDataSuccess6) {
|
|||
std::vector<std::string> file_list = {file_path1};
|
||||
|
||||
// Check sequential sampler, output number is 5
|
||||
std::shared_ptr<Dataset> ds1 = MindData(file_list, {}, SequentialSampler(0, 10));
|
||||
std::shared_ptr<Dataset> ds1 = MindData(file_list, {}, std::make_shared<SequentialSampler>(0, 10));
|
||||
EXPECT_NE(ds1, nullptr);
|
||||
|
||||
// Check random sampler, output number is 5, same rows with file
|
||||
std::shared_ptr<Dataset> ds2 = MindData(file_list, {}, RandomSampler(false, 10));
|
||||
std::shared_ptr<Dataset> ds2 = MindData(file_list, {}, std::make_shared<RandomSampler>(false, 10));
|
||||
EXPECT_NE(ds2, nullptr);
|
||||
|
||||
// Check pk sampler, output number is 2, get 2 samples with label 0
|
||||
std::shared_ptr<Dataset> ds3 = MindData(file_list, {}, PKSampler(2, false, 10));
|
||||
std::shared_ptr<Dataset> ds3 = MindData(file_list, {}, std::make_shared<PKSampler>(2, false, 10));
|
||||
EXPECT_NE(ds3, nullptr);
|
||||
|
||||
// Check distributed sampler, output number is 3, get 3 samples in shard 0
|
||||
std::shared_ptr<Dataset> ds4 = MindData(file_list, {}, DistributedSampler(2, 0, false, 10));
|
||||
std::shared_ptr<Dataset> ds4 = MindData(file_list, {}, std::make_shared<DistributedSampler>(2, 0, false, 10));
|
||||
EXPECT_NE(ds4, nullptr);
|
||||
|
||||
// Check distributed sampler get 3 samples with indice 0, 1 ,2
|
||||
std::shared_ptr<Dataset> ds5 = MindData(file_list, {}, SubsetRandomSampler({0, 1, 2}, 10));
|
||||
std::shared_ptr<Dataset> ds5 = MindData(file_list, {}, new SubsetRandomSampler({0, 1, 2}, 10));
|
||||
EXPECT_NE(ds5, nullptr);
|
||||
|
||||
std::shared_ptr<Dataset> ds6 = MindData(file_list, {}, SubsetSampler({1, 2}, 10));
|
||||
std::shared_ptr<Dataset> ds6 = MindData(file_list, {}, new SubsetSampler({1, 2}, 10));
|
||||
EXPECT_NE(ds5, nullptr);
|
||||
|
||||
std::vector<std::shared_ptr<Dataset>> ds = {ds1, ds2, ds3, ds4, ds5, ds6};
|
||||
|
|
@ -283,7 +283,7 @@ TEST_F(MindDataTestPipeline, TestMindDataSuccess7) {
|
|||
// Pass a list of mindrecord file name, files in list will be read directly but not search for related files
|
||||
std::string file_path1 = datasets_root_path_ + "/../mindrecord/testMindDataSet/testImageNetData/imagenet.mindrecord0";
|
||||
std::vector<std::string> file_list = {file_path1};
|
||||
std::shared_ptr<Dataset> ds = MindData(file_list, {"file_name", "label"}, SequentialSampler(), pad, 4);
|
||||
std::shared_ptr<Dataset> ds = MindData(file_list, {"file_name", "label"}, std::make_shared<SequentialSampler>(), pad, 4);
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create a Skip operation on ds, skip original data in mindrecord and get padded samples
|
||||
|
|
@ -332,7 +332,7 @@ TEST_F(MindDataTestPipeline, TestMindDataSuccess8) {
|
|||
// Pass a list of mindrecord file name, files in list will be read directly but not search for related files
|
||||
std::string file_path1 = datasets_root_path_ + "/../mindrecord/testMindDataSet/testImageNetData/imagenet.mindrecord0";
|
||||
std::vector<std::string> file_list = {file_path1};
|
||||
std::shared_ptr<Dataset> ds = MindData(file_list, {"file_name", "label"}, SequentialSampler(), pad, 4);
|
||||
std::shared_ptr<Dataset> ds = MindData(file_list, {"file_name", "label"}, std::make_shared<SequentialSampler>(), pad, 4);
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
std::vector<DataType> types = ds->GetOutputTypes();
|
||||
|
|
@ -400,12 +400,12 @@ TEST_F(MindDataTestPipeline, TestMindDataSuccess9) {
|
|||
// Pass a list of mindrecord file name, files in list will be read directly but not search for related files
|
||||
std::string file_path1 = datasets_root_path_ + "/../mindrecord/testMindDataSet/testImageNetData/imagenet.mindrecord0";
|
||||
std::vector<std::string> file_list = {file_path1};
|
||||
std::shared_ptr<Dataset> ds1 = MindData(file_list, {"file_name", "label"}, SequentialSampler(), pad, 4);
|
||||
std::shared_ptr<Dataset> ds1 = MindData(file_list, {"file_name", "label"}, std::make_shared<SequentialSampler>(), pad, 4);
|
||||
EXPECT_NE(ds1, nullptr);
|
||||
ds1 = ds1->Skip(5);
|
||||
EXPECT_NE(ds1, nullptr);
|
||||
|
||||
std::shared_ptr<Dataset> ds2 = MindData(file_list, {"file_name", "label"}, SequentialSampler(), pad, 4);
|
||||
std::shared_ptr<Dataset> ds2 = MindData(file_list, {"file_name", "label"}, std::make_shared<SequentialSampler>(), pad, 4);
|
||||
EXPECT_NE(ds2, nullptr);
|
||||
ds2 = ds2->Skip(5);
|
||||
EXPECT_NE(ds2, nullptr);
|
||||
|
|
@ -534,7 +534,7 @@ TEST_F(MindDataTestPipeline, TestMindDataFail3) {
|
|||
|
||||
// Create a MindData Dataset with unsupported sampler
|
||||
std::string file_path1 = datasets_root_path_ + "/../mindrecord/testMindDataSet/testImageNetData/imagenet.mindrecord0";
|
||||
std::shared_ptr<Dataset> ds1 = MindData(file_path1, {}, WeightedRandomSampler({1, 1, 1, 1}));
|
||||
std::shared_ptr<Dataset> ds1 = MindData(file_path1, {}, new WeightedRandomSampler({1, 1, 1, 1}));
|
||||
EXPECT_NE(ds1, nullptr);
|
||||
|
||||
// Create an iterator over the result of the above dataset
|
||||
|
|
@ -558,7 +558,7 @@ TEST_F(MindDataTestPipeline, TestMindDataFail4) {
|
|||
|
||||
// Create a MindData Dataset
|
||||
std::string file_path1 = datasets_root_path_ + "/../mindrecord/testMindDataSet/testImageNetData/imagenet.mindrecord0";
|
||||
std::shared_ptr<Dataset> ds1 = MindData(file_path1, {}, RandomSampler(), nullptr, 2);
|
||||
std::shared_ptr<Dataset> ds1 = MindData(file_path1, {}, std::make_shared<RandomSampler>(), nullptr, 2);
|
||||
EXPECT_NE(ds1, nullptr);
|
||||
|
||||
// Create an iterator over the result of the above dataset
|
||||
|
|
@ -573,7 +573,7 @@ TEST_F(MindDataTestPipeline, TestMindDataFail4) {
|
|||
|
||||
// Create a MindData Dataset
|
||||
std::string file_path2 = datasets_root_path_ + "/../mindrecord/testMindDataSet/testImageNetData/imagenet.mindrecord0";
|
||||
std::shared_ptr<Dataset> ds2 = MindData(file_path2, {"label"}, RandomSampler(), pad, -2);
|
||||
std::shared_ptr<Dataset> ds2 = MindData(file_path2, {"label"}, std::make_shared<RandomSampler>(), pad, -2);
|
||||
EXPECT_NE(ds2, nullptr);
|
||||
|
||||
// Create an iterator over the result of the above dataset
|
||||
|
|
@ -583,7 +583,7 @@ TEST_F(MindDataTestPipeline, TestMindDataFail4) {
|
|||
|
||||
// Create a MindData Dataset
|
||||
std::string file_path3 = datasets_root_path_ + "/../mindrecord/testMindDataSet/testImageNetData/imagenet.mindrecord0";
|
||||
std::shared_ptr<Dataset> ds3 = MindData(file_path3, {}, RandomSampler(), pad, 1);
|
||||
std::shared_ptr<Dataset> ds3 = MindData(file_path3, {}, std::make_shared<RandomSampler>(), pad, 1);
|
||||
EXPECT_NE(ds3, nullptr);
|
||||
|
||||
// Create an iterator over the result of the above dataset
|
||||
|
|
@ -598,7 +598,7 @@ TEST_F(MindDataTestPipeline, TestMindDataFail4) {
|
|||
|
||||
// Create a MindData Dataset
|
||||
std::string file_path4 = datasets_root_path_ + "/../mindrecord/testMindDataSet/testImageNetData/imagenet.mindrecord0";
|
||||
std::shared_ptr<Dataset> ds4 = MindData(file_path4, {"file_name", "label"}, RandomSampler(), pad2, 1);
|
||||
std::shared_ptr<Dataset> ds4 = MindData(file_path4, {"file_name", "label"}, std::make_shared<RandomSampler>(), pad2, 1);
|
||||
EXPECT_NE(ds4, nullptr);
|
||||
|
||||
// Create an iterator over the result of the above dataset
|
||||
|
|
|
|||
|
|
@ -76,7 +76,7 @@ TEST_F(MindDataTestPipeline, TestBatchAndRepeat) {
|
|||
|
||||
// Create a Mnist Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testMnistData/";
|
||||
std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", RandomSampler(false, 10));
|
||||
std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create a Repeat operation on ds
|
||||
|
|
@ -118,7 +118,7 @@ TEST_F(MindDataTestPipeline, TestBucketBatchByLengthSuccess1) {
|
|||
|
||||
// Create a Mnist Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testMnistData/";
|
||||
std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", RandomSampler(false, 10));
|
||||
std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create a BucketBatchByLength operation on ds
|
||||
|
|
@ -154,7 +154,7 @@ TEST_F(MindDataTestPipeline, TestBucketBatchByLengthSuccess2) {
|
|||
|
||||
// Create a Mnist Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testMnistData/";
|
||||
std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", RandomSampler(false, 10));
|
||||
std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create a BucketBatchByLength operation on ds
|
||||
|
|
@ -192,7 +192,7 @@ TEST_F(MindDataTestPipeline, TestBucketBatchByLengthFail1) {
|
|||
|
||||
// Create a Mnist Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testMnistData/";
|
||||
std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", RandomSampler(false, 10));
|
||||
std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create a BucketBatchByLength operation on ds
|
||||
|
|
@ -212,7 +212,7 @@ TEST_F(MindDataTestPipeline, TestBucketBatchByLengthFail2) {
|
|||
|
||||
// Create a Mnist Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testMnistData/";
|
||||
std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", RandomSampler(false, 10));
|
||||
std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create a BucketBatchByLength operation on ds
|
||||
|
|
@ -232,7 +232,7 @@ TEST_F(MindDataTestPipeline, TestBucketBatchByLengthFail3) {
|
|||
|
||||
// Create a Mnist Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testMnistData/";
|
||||
std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", RandomSampler(false, 10));
|
||||
std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create a BucketBatchByLength operation on ds
|
||||
|
|
@ -252,7 +252,7 @@ TEST_F(MindDataTestPipeline, TestBucketBatchByLengthFail4) {
|
|||
|
||||
// Create a Mnist Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testMnistData/";
|
||||
std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", RandomSampler(false, 10));
|
||||
std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create a BucketBatchByLength operation on ds
|
||||
|
|
@ -272,7 +272,7 @@ TEST_F(MindDataTestPipeline, TestBucketBatchByLengthFail5) {
|
|||
|
||||
// Create a Mnist Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testMnistData/";
|
||||
std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", RandomSampler(false, 10));
|
||||
std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create a BucketBatchByLength operation on ds
|
||||
|
|
@ -292,7 +292,7 @@ TEST_F(MindDataTestPipeline, TestBucketBatchByLengthFail6) {
|
|||
|
||||
// Create a Mnist Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testMnistData/";
|
||||
std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", RandomSampler(false, 10));
|
||||
std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
// Create a BucketBatchByLength operation on ds
|
||||
ds = ds->BucketBatchByLength({"image"}, {1, 2}, {1, -2, 3});
|
||||
|
|
@ -311,7 +311,7 @@ TEST_F(MindDataTestPipeline, TestBucketBatchByLengthFail7) {
|
|||
|
||||
// Create a Mnist Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testMnistData/";
|
||||
std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", RandomSampler(false, 10));
|
||||
std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create a BucketBatchByLength operation on ds
|
||||
|
|
@ -331,9 +331,9 @@ TEST_F(MindDataTestPipeline, TestConcatFail1) {
|
|||
// Create an ImageFolder Dataset
|
||||
// Column names: {"image", "label"}
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data/";
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
std::shared_ptr<Dataset> ds2 = ImageFolder(folder_path, true, RandomSampler(false, 10));
|
||||
std::shared_ptr<Dataset> ds2 = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create a Rename operation on ds
|
||||
|
|
@ -362,7 +362,7 @@ TEST_F(MindDataTestPipeline, TestConcatFail2) {
|
|||
|
||||
// Create an ImageFolder Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data/";
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create a Concat operation on the ds
|
||||
|
|
@ -382,7 +382,7 @@ TEST_F(MindDataTestPipeline, TestConcatFail3) {
|
|||
|
||||
// Create an ImageFolder Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data/";
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create a Concat operation on the ds
|
||||
|
|
@ -402,7 +402,7 @@ TEST_F(MindDataTestPipeline, TestConcatFail4) {
|
|||
|
||||
// Create an ImageFolder Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data/";
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create a Concat operation on the ds
|
||||
|
|
@ -422,10 +422,10 @@ TEST_F(MindDataTestPipeline, TestConcatFail5) {
|
|||
|
||||
// Create an ImageFolder Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data/";
|
||||
std::shared_ptr<Dataset> ds1 = ImageFolder(folder_path, true, RandomSampler(false, 10));
|
||||
std::shared_ptr<Dataset> ds1 = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10));
|
||||
EXPECT_NE(ds1, nullptr);
|
||||
|
||||
std::shared_ptr<Dataset> ds2 = ImageFolder(folder_path, true, RandomSampler(false, 10));
|
||||
std::shared_ptr<Dataset> ds2 = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10));
|
||||
EXPECT_NE(ds2, nullptr);
|
||||
|
||||
// Create a Project operation on ds
|
||||
|
|
@ -451,13 +451,13 @@ TEST_F(MindDataTestPipeline, TestConcatSuccess) {
|
|||
// Create an ImageFolder Dataset
|
||||
// Column names: {"image", "label"}
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data/";
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create a Cifar10 Dataset
|
||||
// Column names: {"image", "label"}
|
||||
folder_path = datasets_root_path_ + "/testCifar10Data/";
|
||||
std::shared_ptr<Dataset> ds2 = Cifar10(folder_path, "all", RandomSampler(false, 9));
|
||||
std::shared_ptr<Dataset> ds2 = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 9));
|
||||
EXPECT_NE(ds2, nullptr);
|
||||
|
||||
// Create a Project operation on ds
|
||||
|
|
@ -502,13 +502,13 @@ TEST_F(MindDataTestPipeline, TestConcatGetDatasetSize) {
|
|||
// Create an ImageFolder Dataset
|
||||
// Column names: {"image", "label"}
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data/";
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create a Cifar10 Dataset
|
||||
// Column names: {"image", "label"}
|
||||
folder_path = datasets_root_path_ + "/testCifar10Data/";
|
||||
std::shared_ptr<Dataset> ds2 = Cifar10(folder_path, "all", RandomSampler(false, 9));
|
||||
std::shared_ptr<Dataset> ds2 = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 9));
|
||||
EXPECT_NE(ds2, nullptr);
|
||||
|
||||
// Create a Project operation on ds
|
||||
|
|
@ -531,13 +531,13 @@ TEST_F(MindDataTestPipeline, TestConcatSuccess2) {
|
|||
// Create an ImageFolder Dataset
|
||||
// Column names: {"image", "label"}
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data/";
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create a Cifar10 Dataset
|
||||
// Column names: {"image", "label"}
|
||||
folder_path = datasets_root_path_ + "/testCifar10Data/";
|
||||
std::shared_ptr<Dataset> ds2 = Cifar10(folder_path, "all", RandomSampler(false, 9));
|
||||
std::shared_ptr<Dataset> ds2 = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 9));
|
||||
EXPECT_NE(ds2, nullptr);
|
||||
|
||||
// Create a Project operation on ds
|
||||
|
|
@ -755,7 +755,7 @@ TEST_F(MindDataTestPipeline, TestImageFolderBatchAndRepeat) {
|
|||
|
||||
// Create an ImageFolder Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data/";
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create a Repeat operation on ds
|
||||
|
|
@ -796,7 +796,7 @@ TEST_F(MindDataTestPipeline, TestPipelineGetDatasetSize) {
|
|||
|
||||
// Create an ImageFolder Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data/";
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create a Repeat operation on ds
|
||||
|
|
@ -818,7 +818,7 @@ TEST_F(MindDataTestPipeline, TestDistributedGetDatasetSize1) {
|
|||
|
||||
// Create an ImageFolder Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data/";
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, DistributedSampler(4, 0, false, 10));
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<DistributedSampler>(4, 0, false, 10));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// num_per_shard is equal to 44/4 = 11 which is more than num_samples = 10, so the output is 10
|
||||
|
|
@ -849,7 +849,7 @@ TEST_F(MindDataTestPipeline, TestDistributedGetDatasetSize2) {
|
|||
|
||||
// Create an ImageFolder Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data/";
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, DistributedSampler(4, 0, false, 15));
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<DistributedSampler>(4, 0, false, 15));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// num_per_shard is equal to 44/4 = 11 which is less than num_samples = 15, so the output is 11
|
||||
|
|
@ -879,7 +879,7 @@ TEST_F(MindDataTestPipeline, TestProjectMap) {
|
|||
|
||||
// Create an ImageFolder Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data/";
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create a Repeat operation on ds
|
||||
|
|
@ -933,7 +933,7 @@ TEST_F(MindDataTestPipeline, TestProjectDuplicateColumnFail) {
|
|||
|
||||
// Create an ImageFolder Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data/";
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 3));
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 3));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create objects for the tensor ops
|
||||
|
|
@ -962,7 +962,7 @@ TEST_F(MindDataTestPipeline, TestMapDuplicateColumnFail) {
|
|||
|
||||
// Create an ImageFolder Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data/";
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create objects for the tensor ops
|
||||
|
|
@ -1002,7 +1002,7 @@ TEST_F(MindDataTestPipeline, TestProjectMapAutoInjection) {
|
|||
|
||||
// Create an ImageFolder Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data/";
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create a Repeat operation on ds
|
||||
|
|
@ -1054,7 +1054,7 @@ TEST_F(MindDataTestPipeline, TestRenameFail1) {
|
|||
|
||||
// Create an ImageFolder Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data/";
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create a Repeat operation on ds
|
||||
|
|
@ -1078,7 +1078,7 @@ TEST_F(MindDataTestPipeline, TestRenameFail2) {
|
|||
|
||||
// Create an ImageFolder Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data/";
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create a Rename operation on ds
|
||||
|
|
@ -1097,7 +1097,7 @@ TEST_F(MindDataTestPipeline, TestRenameFail3) {
|
|||
|
||||
// Create an ImageFolder Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data/";
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create a Rename operation on ds
|
||||
|
|
@ -1124,7 +1124,7 @@ TEST_F(MindDataTestPipeline, TestRenameSuccess) {
|
|||
|
||||
// Create an ImageFolder Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data/";
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create a Repeat operation on ds
|
||||
|
|
@ -1174,7 +1174,7 @@ TEST_F(MindDataTestPipeline, TestRepeatDefault) {
|
|||
|
||||
// Create an ImageFolder Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data/";
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create a Repeat operation on ds
|
||||
|
|
@ -1217,7 +1217,7 @@ TEST_F(MindDataTestPipeline, TestRepeatOne) {
|
|||
|
||||
// Create an ImageFolder Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data/";
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create a Repeat operation on ds
|
||||
|
|
@ -1256,7 +1256,7 @@ TEST_F(MindDataTestPipeline, TestRepeatFail1) {
|
|||
|
||||
// Create an ImageFolder Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data/";
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create a Repeat operation on ds
|
||||
|
|
@ -1276,7 +1276,7 @@ TEST_F(MindDataTestPipeline, TestRepeatFail2) {
|
|||
|
||||
// Create an ImageFolder Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data/";
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create a Repeat operation on ds
|
||||
|
|
@ -1295,7 +1295,7 @@ TEST_F(MindDataTestPipeline, TestShuffleDataset) {
|
|||
|
||||
// Create an ImageFolder Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data/";
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create a Shuffle operation on ds
|
||||
|
|
@ -1341,7 +1341,7 @@ TEST_F(MindDataTestPipeline, TestSkipDataset) {
|
|||
|
||||
// Create an ImageFolder Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data/";
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create a Skip operation on ds
|
||||
|
|
@ -1379,7 +1379,7 @@ TEST_F(MindDataTestPipeline, TestSkipTakeRepeat) {
|
|||
|
||||
// Create an ImageFolder Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data/";
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 6));
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 6));
|
||||
|
||||
// Create a Skip operation on ds
|
||||
int32_t count = 0;
|
||||
|
|
@ -1424,7 +1424,7 @@ TEST_F(MindDataTestPipeline, TestSkipGetDatasetSize) {
|
|||
|
||||
// Create an ImageFolder Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data/";
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create a Skip operation on ds
|
||||
|
|
@ -1440,7 +1440,7 @@ TEST_F(MindDataTestPipeline, TestSkipDatasetError1) {
|
|||
|
||||
// Create an ImageFolder Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data/";
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create a Skip operation on ds with invalid count input
|
||||
|
|
@ -1459,7 +1459,7 @@ TEST_F(MindDataTestPipeline, TestTakeDatasetDefault) {
|
|||
|
||||
// Create an ImageFolder Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data/";
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 7));
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 7));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create a Take operation on ds, default count = -1
|
||||
|
|
@ -1496,7 +1496,7 @@ TEST_F(MindDataTestPipeline, TestTakeGetDatasetSize) {
|
|||
|
||||
// Create an ImageFolder Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data/";
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 7));
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 7));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create a Take operation on ds, default count = -1
|
||||
|
|
@ -1511,7 +1511,7 @@ TEST_F(MindDataTestPipeline, TestTakeDatasetError1) {
|
|||
|
||||
// Create an ImageFolder Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data/";
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create a Take operation on ds with invalid count input
|
||||
|
|
@ -1540,7 +1540,7 @@ TEST_F(MindDataTestPipeline, TestTakeDatasetNormal) {
|
|||
|
||||
// Create an ImageFolder Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data/";
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 8));
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 8));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create a Take operation on ds
|
||||
|
|
@ -1577,7 +1577,7 @@ TEST_F(MindDataTestPipeline, TestTensorOpsAndMap) {
|
|||
|
||||
// Create a Mnist Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testMnistData/";
|
||||
std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", RandomSampler(false, 20));
|
||||
std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", std::make_shared<RandomSampler>(false, 20));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create a Repeat operation on ds
|
||||
|
|
@ -1631,11 +1631,11 @@ TEST_F(MindDataTestPipeline, TestZipFail) {
|
|||
|
||||
// Create an ImageFolder Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data/";
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create an ImageFolder Dataset
|
||||
std::shared_ptr<Dataset> ds1 = ImageFolder(folder_path, true, RandomSampler(false, 10));
|
||||
std::shared_ptr<Dataset> ds1 = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10));
|
||||
EXPECT_NE(ds1, nullptr);
|
||||
|
||||
// Create a Zip operation on the datasets
|
||||
|
|
@ -1659,7 +1659,7 @@ TEST_F(MindDataTestPipeline, TestZipFail2) {
|
|||
|
||||
// Create an ImageFolder Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data/";
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create a Zip operation on the datasets
|
||||
|
|
@ -1679,7 +1679,7 @@ TEST_F(MindDataTestPipeline, TestZipSuccess) {
|
|||
|
||||
// Create an ImageFolder Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data/";
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create a Project operation on ds
|
||||
|
|
@ -1688,7 +1688,7 @@ TEST_F(MindDataTestPipeline, TestZipSuccess) {
|
|||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create an ImageFolder Dataset
|
||||
std::shared_ptr<Dataset> ds1 = ImageFolder(folder_path, true, RandomSampler(false, 10));
|
||||
std::shared_ptr<Dataset> ds1 = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10));
|
||||
EXPECT_NE(ds1, nullptr);
|
||||
|
||||
// Create a Rename operation on ds (so that the 3 datasets we are going to zip have distinct column names)
|
||||
|
|
@ -1696,7 +1696,7 @@ TEST_F(MindDataTestPipeline, TestZipSuccess) {
|
|||
EXPECT_NE(ds1, nullptr);
|
||||
|
||||
folder_path = datasets_root_path_ + "/testCifar10Data/";
|
||||
std::shared_ptr<Dataset> ds2 = Cifar10(folder_path, "all", RandomSampler(false, 10));
|
||||
std::shared_ptr<Dataset> ds2 = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
|
||||
EXPECT_NE(ds2, nullptr);
|
||||
|
||||
// Create a Project operation on ds
|
||||
|
|
@ -1749,7 +1749,7 @@ TEST_F(MindDataTestPipeline, TestZipGetDatasetSize) {
|
|||
|
||||
// Create an ImageFolder Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data/";
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 2));
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 2));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create a Project operation on ds
|
||||
|
|
@ -1758,7 +1758,7 @@ TEST_F(MindDataTestPipeline, TestZipGetDatasetSize) {
|
|||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create an ImageFolder Dataset
|
||||
std::shared_ptr<Dataset> ds1 = ImageFolder(folder_path, true, RandomSampler(false, 3));
|
||||
std::shared_ptr<Dataset> ds1 = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 3));
|
||||
EXPECT_NE(ds1, nullptr);
|
||||
|
||||
// Create a Rename operation on ds (so that the 3 datasets we are going to zip have distinct column names)
|
||||
|
|
@ -1766,7 +1766,7 @@ TEST_F(MindDataTestPipeline, TestZipGetDatasetSize) {
|
|||
EXPECT_NE(ds1, nullptr);
|
||||
|
||||
folder_path = datasets_root_path_ + "/testCifar10Data/";
|
||||
std::shared_ptr<Dataset> ds2 = Cifar10(folder_path, "all", RandomSampler(false, 5));
|
||||
std::shared_ptr<Dataset> ds2 = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 5));
|
||||
EXPECT_NE(ds2, nullptr);
|
||||
|
||||
// Create a Project operation on ds
|
||||
|
|
@ -1787,9 +1787,9 @@ TEST_F(MindDataTestPipeline, TestZipSuccess2) {
|
|||
|
||||
// Create an ImageFolder Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data/";
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 9));
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 9));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
std::shared_ptr<Dataset> ds2 = ImageFolder(folder_path, true, RandomSampler(false, 10));
|
||||
std::shared_ptr<Dataset> ds2 = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10));
|
||||
EXPECT_NE(ds2, nullptr);
|
||||
|
||||
// Create a Rename operation on ds (so that the 2 datasets we are going to zip have distinct column names)
|
||||
|
|
@ -1840,7 +1840,7 @@ TEST_F(MindDataTestPipeline, TestNumWorkersValidate) {
|
|||
|
||||
// Create an ImageFolder Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data/";
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, false, SequentialSampler(0, 1));
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 1));
|
||||
|
||||
// ds needs to be non nullptr otherwise, the subsequent logic will core dump
|
||||
ASSERT_NE(ds, nullptr);
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
/**
|
||||
* Copyright 2020 Huawei Technologies Co., Ltd
|
||||
* Copyright 2020-2021 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
|
|
@ -30,7 +30,7 @@ TEST_F(MindDataTestPipeline, TestSaveCifar10AndLoad) {
|
|||
// Stage 1: load original dataset
|
||||
// Create a Cifar10 Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
|
||||
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", SequentialSampler(0, 10));
|
||||
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<SequentialSampler>(0, 10));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create an iterator over the result of the above dataset
|
||||
|
|
@ -70,7 +70,7 @@ TEST_F(MindDataTestPipeline, TestSaveCifar10AndLoad) {
|
|||
|
||||
// Stage 3: Load dataset from file output by stage 2
|
||||
// Create a MindData Dataset
|
||||
std::shared_ptr<Dataset> ds_minddata = MindData(temp_file, {}, SequentialSampler(0, 10));
|
||||
std::shared_ptr<Dataset> ds_minddata = MindData(temp_file, {}, std::make_shared<SequentialSampler>(0, 10));
|
||||
|
||||
// Create objects for the tensor ops
|
||||
// uint32 will be casted to int64 implicitly in mindrecord file, so we have to cast it back to uint32
|
||||
|
|
@ -119,7 +119,7 @@ TEST_F(MindDataTestPipeline, TestSaveFail) {
|
|||
|
||||
// Create a Cifar10 Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
|
||||
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", SequentialSampler(0, 10));
|
||||
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<SequentialSampler>(0, 10));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// fail with invalid dataset_path
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
/**
|
||||
* Copyright 2020 Huawei Technologies Co., Ltd
|
||||
* Copyright 2020-2021 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
|
|
@ -35,7 +35,7 @@ TEST_F(MindDataTestPipeline, TestVOCClassIndex) {
|
|||
class_index["cat"] = 1;
|
||||
class_index["train"] = 9;
|
||||
|
||||
std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", class_index, false, SequentialSampler(0, 6));
|
||||
std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", class_index, false, std::make_shared<SequentialSampler>(0, 6));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create an iterator over the result of the above dataset
|
||||
|
|
@ -81,7 +81,7 @@ TEST_F(MindDataTestPipeline, TestVOCGetClassIndex) {
|
|||
class_index["cat"] = 1;
|
||||
class_index["train"] = 9;
|
||||
|
||||
std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", class_index, false, SequentialSampler(0, 6));
|
||||
std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", class_index, false, std::make_shared<SequentialSampler>(0, 6));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
std::vector<std::pair<std::string, std::vector<int32_t>>> class_index1 = ds->GetClassIndexing();
|
||||
|
|
@ -104,7 +104,7 @@ TEST_F(MindDataTestPipeline, TestVOCGetters) {
|
|||
class_index["cat"] = 1;
|
||||
class_index["train"] = 9;
|
||||
|
||||
std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", class_index, false, SequentialSampler(0, 6));
|
||||
std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", class_index, false, std::make_shared<SequentialSampler>(0, 6));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
ds = ds->Batch(2);
|
||||
|
|
@ -120,7 +120,7 @@ TEST_F(MindDataTestPipeline, TestVOCDetection) {
|
|||
|
||||
// Create a VOC Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testVOC2012_2";
|
||||
std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", {}, false, SequentialSampler(0, 4));
|
||||
std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", {}, false, std::make_shared<SequentialSampler>(0, 4));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create an iterator over the result of the above dataset
|
||||
|
|
@ -166,7 +166,7 @@ TEST_F(MindDataTestPipeline, TestVOCInvalidTaskOrModeError1) {
|
|||
|
||||
// Create a VOC Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testVOC2012_2";
|
||||
std::shared_ptr<Dataset> ds1 = VOC(folder_path, "Classification", "train", {}, false, SequentialSampler(0, 3));
|
||||
std::shared_ptr<Dataset> ds1 = VOC(folder_path, "Classification", "train", {}, false, std::make_shared<SequentialSampler>(0, 3));
|
||||
EXPECT_NE(ds1, nullptr);
|
||||
|
||||
// Create an iterator over the result of the above dataset
|
||||
|
|
@ -174,7 +174,7 @@ TEST_F(MindDataTestPipeline, TestVOCInvalidTaskOrModeError1) {
|
|||
// Expect failure: invalid Manifest input, invalid task
|
||||
EXPECT_EQ(iter1, nullptr);
|
||||
|
||||
std::shared_ptr<Dataset> ds2 = VOC(folder_path, "Segmentation", "validation", {}, false, RandomSampler(false, 4));
|
||||
std::shared_ptr<Dataset> ds2 = VOC(folder_path, "Segmentation", "validation", {}, false, std::make_shared<RandomSampler>(false, 4));
|
||||
EXPECT_NE(ds2, nullptr);
|
||||
|
||||
// Create an iterator over the result of the above dataset
|
||||
|
|
@ -188,7 +188,7 @@ TEST_F(MindDataTestPipeline, TestVOCSegmentation) {
|
|||
|
||||
// Create a VOC Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testVOC2012_2";
|
||||
std::shared_ptr<Dataset> ds = VOC(folder_path, "Segmentation", "train", {}, false, SequentialSampler(0, 3));
|
||||
std::shared_ptr<Dataset> ds = VOC(folder_path, "Segmentation", "train", {}, false, std::make_shared<SequentialSampler>(0, 3));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create a Repeat operation on ds
|
||||
|
|
@ -240,7 +240,7 @@ TEST_F(MindDataTestPipeline, TestVOCSegmentationError2) {
|
|||
std::map<std::string, int32_t> class_index;
|
||||
class_index["car"] = 0;
|
||||
std::string folder_path = datasets_root_path_ + "/testVOC2012_2";
|
||||
std::shared_ptr<Dataset> ds = VOC(folder_path, "Segmentation", "train", class_index, false, RandomSampler(false, 6));
|
||||
std::shared_ptr<Dataset> ds = VOC(folder_path, "Segmentation", "train", class_index, false, std::make_shared<RandomSampler>(false, 6));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create an iterator over the result of the above dataset
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
/**
|
||||
* Copyright 2020 Huawei Technologies Co., Ltd
|
||||
* Copyright 2020-2021 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
|
|
@ -31,7 +31,7 @@ TEST_F(MindDataTestPipeline, TestCelebADataset) {
|
|||
|
||||
// Create a CelebA Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testCelebAData/";
|
||||
std::shared_ptr<Dataset> ds = CelebA(folder_path, "all", SequentialSampler(0, 2), false, {});
|
||||
std::shared_ptr<Dataset> ds = CelebA(folder_path, "all", std::make_shared<SequentialSampler>(0, 2), false, {});
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create an iterator over the result of the above dataset
|
||||
|
|
@ -208,7 +208,7 @@ TEST_F(MindDataTestPipeline, TestImageFolderFailWithWrongExtensionFail) {
|
|||
|
||||
// Create an ImageFolder Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data/";
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 2), {".JGP"});
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 2), {".JGP"});
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create an iterator over the result of the above dataset
|
||||
|
|
@ -260,7 +260,7 @@ TEST_F(MindDataTestPipeline, TestImageFolderFailWithWrongSamplerFail) {
|
|||
|
||||
// Create an ImageFolder Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data/";
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, SequentialSampler(-2, 5));
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<SequentialSampler>(-2, 5));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create an iterator over the result of the above dataset
|
||||
|
|
@ -274,7 +274,7 @@ TEST_F(MindDataTestPipeline, TestMnistGetDatasetSize) {
|
|||
|
||||
// Create a Mnist Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testMnistData/";
|
||||
std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", RandomSampler(false, 20));
|
||||
std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", std::make_shared<RandomSampler>(false, 20));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
EXPECT_EQ(ds->GetDatasetSize(), 20);
|
||||
}
|
||||
|
|
@ -283,7 +283,7 @@ TEST_F(MindDataTestPipeline, TestMnistFailWithWrongDatasetDirFail) {
|
|||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestMnistFailWithWrongDatasetDirFail.";
|
||||
|
||||
// Create a Mnist Dataset
|
||||
std::shared_ptr<Dataset> ds = Mnist("", "all", RandomSampler(false, 10));
|
||||
std::shared_ptr<Dataset> ds = Mnist("", "all", std::make_shared<RandomSampler>(false, 10));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create an iterator over the result of the above dataset
|
||||
|
|
@ -311,7 +311,7 @@ TEST_F(MindDataTestPipeline, TestImageFolderClassIndexDatasetSize) {
|
|||
std::map<std::string, int32_t> class_index;
|
||||
class_index["class1"] = 111;
|
||||
class_index["class2"] = 333;
|
||||
auto ds = ImageFolder(folder_path, false, RandomSampler(), {}, class_index);
|
||||
auto ds = ImageFolder(folder_path, false, std::make_shared<RandomSampler>(), {}, class_index);
|
||||
EXPECT_EQ(ds->GetNumClasses(), 2);
|
||||
}
|
||||
|
||||
|
|
@ -320,6 +320,6 @@ TEST_F(MindDataTestPipeline, TestImageFolderClassIndexDatasetSizeFail) {
|
|||
std::map<std::string, int32_t> class_index;
|
||||
class_index["class1"] = 111;
|
||||
class_index["wrong class"] = 333;
|
||||
auto ds = ImageFolder(folder_path, false, RandomSampler(), {}, class_index);
|
||||
auto ds = ImageFolder(folder_path, false, std::make_shared<RandomSampler>(), {}, class_index);
|
||||
EXPECT_EQ(ds->GetNumClasses(), -1);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -34,7 +34,7 @@ TEST_F(MindDataTestEpochCtrl, TestAutoInjectEpoch) {
|
|||
|
||||
// Create an ImageFolder Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data/";
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, SequentialSampler(0, sampler_size));
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<SequentialSampler>(0, sampler_size));
|
||||
ds = ds->SetNumWorkers(2);
|
||||
|
||||
// Create an iterator over the result of the above dataset
|
||||
|
|
@ -79,7 +79,7 @@ TEST_F(MindDataTestEpochCtrl, TestEpoch) {
|
|||
|
||||
// Create an ImageFolder Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data/";
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(0, sampler_size));
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(0, sampler_size));
|
||||
ds = ds->SetNumWorkers(3);
|
||||
|
||||
// Create an iterator over the result of the above dataset
|
||||
|
|
@ -125,7 +125,7 @@ TEST_F(MindDataTestEpochCtrl, TestRepeatEpoch) {
|
|||
|
||||
// Create an ImageFolder Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data/";
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(0, sampler_size));
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(0, sampler_size));
|
||||
ds = ds->SetNumWorkers(3);
|
||||
ds = ds->Repeat(num_repeats);
|
||||
|
||||
|
|
@ -172,7 +172,7 @@ TEST_F(MindDataTestEpochCtrl, TestRepeatRepeatEpoch) {
|
|||
|
||||
// Create an ImageFolder Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data/";
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, SequentialSampler(5, sampler_size));
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<SequentialSampler>(5, sampler_size));
|
||||
ds = ds->Repeat(num_repeats[0]);
|
||||
ds = ds->Repeat(num_repeats[1]);
|
||||
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@
|
|||
#include "common/common.h"
|
||||
#include "minddata/dataset/engine/datasetops/source/sampler/sampler.h"
|
||||
#include "minddata/dataset/include/datasets.h"
|
||||
#include <functional>
|
||||
|
||||
using namespace mindspore::dataset;
|
||||
using mindspore::dataset::Tensor;
|
||||
|
|
@ -25,27 +26,27 @@ class MindDataTestPipeline : public UT::DatasetOpTesting {
|
|||
};
|
||||
|
||||
TEST_F(MindDataTestPipeline, TestImageFolderWithSamplers) {
|
||||
std::shared_ptr<SamplerObj> sampl = DistributedSampler(2, 1);
|
||||
std::shared_ptr<Sampler> sampl = std::make_shared<DistributedSampler>(2, 1);
|
||||
EXPECT_NE(sampl, nullptr);
|
||||
|
||||
sampl = PKSampler(3);
|
||||
sampl = std::make_shared<PKSampler>(3);
|
||||
EXPECT_NE(sampl, nullptr);
|
||||
|
||||
sampl = RandomSampler(false, 12);
|
||||
sampl = std::make_shared<RandomSampler>(false, 12);
|
||||
EXPECT_NE(sampl, nullptr);
|
||||
|
||||
sampl = SequentialSampler(0, 12);
|
||||
sampl = std::make_shared<SequentialSampler>(0, 12);
|
||||
EXPECT_NE(sampl, nullptr);
|
||||
|
||||
std::vector<double> weights = {0.9, 0.8, 0.68, 0.7, 0.71, 0.6, 0.5, 0.4, 0.3, 0.5, 0.2, 0.1};
|
||||
sampl = WeightedRandomSampler(weights, 12);
|
||||
sampl = std::make_shared<WeightedRandomSampler>(weights, 12);
|
||||
EXPECT_NE(sampl, nullptr);
|
||||
|
||||
std::vector<int64_t> indices = {1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23};
|
||||
sampl = SubsetSampler(indices);
|
||||
sampl = std::make_shared<SubsetSampler>(indices);
|
||||
EXPECT_NE(sampl, nullptr);
|
||||
|
||||
sampl = SubsetRandomSampler(indices);
|
||||
sampl = std::make_shared<SubsetRandomSampler>(indices);
|
||||
EXPECT_NE(sampl, nullptr);
|
||||
|
||||
// Create an ImageFolder Dataset
|
||||
|
|
@ -88,47 +89,47 @@ TEST_F(MindDataTestPipeline, TestImageFolderWithSamplers) {
|
|||
|
||||
TEST_F(MindDataTestPipeline, TestCalculateNumSamples) {
|
||||
int64_t num_rows = 30; // dummy variable for number of rows in the dataset
|
||||
std::shared_ptr<SamplerObj> sampl = DistributedSampler(2, 1, false, 6);
|
||||
std::shared_ptr<SamplerObj> sampl = std::make_shared<DistributedSamplerObj>(2, 1, false, 6, 1, -1, true);
|
||||
EXPECT_NE(sampl, nullptr);
|
||||
std::shared_ptr<SamplerRT> sampler_rt;
|
||||
sampl->SamplerBuild(&sampler_rt);
|
||||
EXPECT_EQ(sampler_rt->CalculateNumSamples(num_rows), 6);
|
||||
|
||||
sampl = PKSampler(3, false);
|
||||
sampl = std::make_shared<PKSamplerObj>(3, false, 0);
|
||||
EXPECT_NE(sampl, nullptr);
|
||||
sampl->SamplerBuild(&sampler_rt);
|
||||
EXPECT_EQ(sampler_rt->CalculateNumSamples(num_rows), 30);
|
||||
|
||||
sampl = RandomSampler(false, 12);
|
||||
sampl = std::make_shared<RandomSamplerObj>(false, 12);
|
||||
EXPECT_NE(sampl, nullptr);
|
||||
sampl->SamplerBuild(&sampler_rt);
|
||||
EXPECT_EQ(sampler_rt->CalculateNumSamples(num_rows), 12);
|
||||
|
||||
sampl = SequentialSampler(0, 10);
|
||||
sampl = std::make_shared<SequentialSamplerObj>(0, 10);
|
||||
EXPECT_NE(sampl, nullptr);
|
||||
sampl->SamplerBuild(&sampler_rt);
|
||||
EXPECT_EQ(sampler_rt->CalculateNumSamples(num_rows), 10);
|
||||
|
||||
std::vector<double> weights = {0.9, 0.8, 0.68, 0.7, 0.71, 0.6, 0.5, 0.4, 0.3, 0.5, 0.2, 0.1};
|
||||
sampl = WeightedRandomSampler(weights, 12);
|
||||
sampl = std::make_shared<WeightedRandomSamplerObj>(weights, 12);
|
||||
EXPECT_NE(sampl, nullptr);
|
||||
sampl->SamplerBuild(&sampler_rt);
|
||||
EXPECT_EQ(sampler_rt->CalculateNumSamples(num_rows), 12);
|
||||
|
||||
std::vector<int64_t> indices = {1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21};
|
||||
sampl = SubsetRandomSampler(indices, 11);
|
||||
sampl = std::make_shared<SubsetRandomSamplerObj>(indices, 11);
|
||||
EXPECT_NE(sampl, nullptr);
|
||||
sampl->SamplerBuild(&sampler_rt);
|
||||
EXPECT_EQ(sampler_rt->CalculateNumSamples(num_rows), 11);
|
||||
|
||||
// Testing chains
|
||||
// Parent and child have num_samples
|
||||
std::shared_ptr<SamplerObj> sampl1 = WeightedRandomSampler(weights, 12);
|
||||
std::shared_ptr<SamplerObj> sampl1 = std::make_shared<WeightedRandomSamplerObj>(weights, 12);
|
||||
EXPECT_NE(sampl1, nullptr);
|
||||
std::shared_ptr<SamplerRT> sampler_rt1;
|
||||
sampl1->SamplerBuild(&sampler_rt1);
|
||||
|
||||
std::shared_ptr<SamplerObj> sampl2 = SequentialSampler(0, 10);
|
||||
std::shared_ptr<SamplerObj> sampl2 = std::make_shared<SequentialSamplerObj>(0, 10);
|
||||
EXPECT_NE(sampl2, nullptr);
|
||||
std::shared_ptr<SamplerRT> sampler_rt2;
|
||||
sampl2->SamplerBuild(&sampler_rt2);
|
||||
|
|
@ -136,12 +137,12 @@ TEST_F(MindDataTestPipeline, TestCalculateNumSamples) {
|
|||
EXPECT_EQ(sampler_rt2->CalculateNumSamples(num_rows), 10);
|
||||
|
||||
// Parent doesn't have num_samples
|
||||
std::shared_ptr<SamplerObj> sampl3 = WeightedRandomSampler(weights, 12);
|
||||
std::shared_ptr<SamplerObj> sampl3 = std::make_shared<WeightedRandomSamplerObj>(weights, 12);
|
||||
EXPECT_NE(sampl3, nullptr);
|
||||
std::shared_ptr<SamplerRT> sampler_rt3;
|
||||
sampl3->SamplerBuild(&sampler_rt3);
|
||||
|
||||
std::shared_ptr<SamplerObj> sampl4 = SubsetRandomSampler(indices);
|
||||
std::shared_ptr<SamplerObj> sampl4 = std::make_shared<SubsetRandomSamplerObj>(indices, 0);
|
||||
EXPECT_NE(sampl4, nullptr);
|
||||
std::shared_ptr<SamplerRT> sampler_rt4;
|
||||
sampl4->SamplerBuild(&sampler_rt4);
|
||||
|
|
@ -149,12 +150,12 @@ TEST_F(MindDataTestPipeline, TestCalculateNumSamples) {
|
|||
EXPECT_EQ(sampler_rt4->CalculateNumSamples(num_rows), 11);
|
||||
|
||||
// Child doesn't have num_samples
|
||||
std::shared_ptr<SamplerObj> sampl5 = RandomSampler(false);
|
||||
std::shared_ptr<SamplerObj> sampl5 = std::make_shared<RandomSamplerObj>(false, 0);
|
||||
EXPECT_NE(sampl5, nullptr);
|
||||
std::shared_ptr<SamplerRT> sampler_rt5;
|
||||
sampl5->SamplerBuild(&sampler_rt5);
|
||||
|
||||
std::shared_ptr<SamplerObj> sampl6 = PKSampler(3, false, 7);
|
||||
std::shared_ptr<SamplerObj> sampl6 = std::make_shared<PKSamplerObj>(3, false, 7);
|
||||
EXPECT_NE(sampl6, nullptr);
|
||||
std::shared_ptr<SamplerRT> sampler_rt6;
|
||||
sampl6->SamplerBuild(&sampler_rt6);
|
||||
|
|
@ -164,41 +165,50 @@ TEST_F(MindDataTestPipeline, TestCalculateNumSamples) {
|
|||
|
||||
TEST_F(MindDataTestPipeline, TestSamplersMoveParameters) {
|
||||
std::vector<int64_t> indices = {1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23};
|
||||
std::shared_ptr<SamplerObj> sampl1 = SubsetRandomSampler(indices);
|
||||
std::shared_ptr<SamplerObj> sampl1 = std::make_shared<SubsetRandomSamplerObj>(indices, 0);
|
||||
EXPECT_FALSE(indices.empty());
|
||||
std::shared_ptr<SamplerRT> sampler_rt = nullptr;
|
||||
sampl1->SamplerBuild(&sampler_rt);
|
||||
EXPECT_NE(sampler_rt, nullptr);
|
||||
std::shared_ptr<SamplerObj> sampl2 = SubsetRandomSampler(std::move(indices));
|
||||
std::shared_ptr<SamplerObj> sampl2 = std::make_shared<SubsetRandomSamplerObj>(std::move(indices), 0);
|
||||
EXPECT_TRUE(indices.empty());
|
||||
std::shared_ptr<SamplerRT> sampler_rt2 = nullptr;
|
||||
sampl2->SamplerBuild(&sampler_rt2);
|
||||
EXPECT_NE(sampler_rt, nullptr);
|
||||
}
|
||||
|
||||
TEST_F(MindDataTestPipeline, TestWeightedRandomSamplerFail) {
|
||||
// weights is empty
|
||||
std::vector<double> weights1 = {};
|
||||
std::shared_ptr<SamplerObj> sampl1 = WeightedRandomSampler(weights1);
|
||||
EXPECT_EQ(sampl1, nullptr);
|
||||
TEST_F(MindDataTestPipeline, TestNoSamplerSuccess1) {
|
||||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestNoSamplerSuccess1.";
|
||||
// Test building a dataset with no sampler provided (defaults to random sampler
|
||||
|
||||
// weights has negative number
|
||||
std::vector<double> weights2 = {0.5, 0.2, -0.4};
|
||||
std::shared_ptr<SamplerObj> sampl2 = WeightedRandomSampler(weights2);
|
||||
EXPECT_EQ(sampl2, nullptr);
|
||||
// Create an ImageFolder Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data/";
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, false);
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// weights elements are all zero
|
||||
std::vector<double> weights3 = {0, 0, 0};
|
||||
std::shared_ptr<SamplerObj> sampl3 = WeightedRandomSampler(weights3);
|
||||
EXPECT_EQ(sampl3, nullptr);
|
||||
// Iterate the dataset and get each row
|
||||
std::shared_ptr<Iterator> iter = ds->CreateIterator();
|
||||
EXPECT_NE(iter, nullptr);
|
||||
std::unordered_map<std::string, mindspore::MSTensor> row;
|
||||
iter->GetNextRow(&row);
|
||||
|
||||
uint64_t i = 0;
|
||||
while (row.size() != 0) {
|
||||
i++;
|
||||
auto label = row["label"];
|
||||
iter->GetNextRow(&row);
|
||||
}
|
||||
|
||||
EXPECT_EQ(i, ds->GetDatasetSize());
|
||||
iter->Stop();
|
||||
}
|
||||
|
||||
TEST_F(MindDataTestPipeline, TestDistributedSamplerSuccess) {
|
||||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestDistributedSamplerSuccess.";
|
||||
TEST_F(MindDataTestPipeline, TestDistributedSamplerSuccess1) {
|
||||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestDistributedSamplerSuccess1.";
|
||||
// Test basic setting of distributed_sampler
|
||||
|
||||
// num_shards=4, shard_id=0, shuffle=false, num_samplers=0, seed=0, offset=-1, even_dist=true
|
||||
std::shared_ptr<SamplerObj> sampler = DistributedSampler(4, 0, false, 0, 0, -1, true);
|
||||
std::shared_ptr<Sampler> sampler = std::make_shared<DistributedSampler>(4, 0, false, 0, 0, -1, true);
|
||||
EXPECT_NE(sampler, nullptr);
|
||||
|
||||
// Create an ImageFolder Dataset
|
||||
|
|
@ -223,16 +233,132 @@ TEST_F(MindDataTestPipeline, TestDistributedSamplerSuccess) {
|
|||
iter->Stop();
|
||||
}
|
||||
|
||||
TEST_F(MindDataTestPipeline, TestDistributedSamplerSuccess2) {
|
||||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestDistributedSamplerSuccess2.";
|
||||
// Test basic setting of distributed_sampler
|
||||
|
||||
// num_shards=4, shard_id=0, shuffle=false, num_samplers=0, seed=0, offset=-1, even_dist=true
|
||||
Sampler *sampler = new DistributedSampler(4, 0, false, 0, 0, -1, true);
|
||||
EXPECT_NE(sampler, nullptr);
|
||||
|
||||
// Create an ImageFolder Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data/";
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, false, sampler);
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Iterate the dataset and get each row
|
||||
std::shared_ptr<Iterator> iter = ds->CreateIterator();
|
||||
EXPECT_NE(iter, nullptr);
|
||||
std::unordered_map<std::string, mindspore::MSTensor> row;
|
||||
iter->GetNextRow(&row);
|
||||
|
||||
uint64_t i = 0;
|
||||
while (row.size() != 0) {
|
||||
i++;
|
||||
auto label = row["label"];
|
||||
iter->GetNextRow(&row);
|
||||
}
|
||||
|
||||
EXPECT_EQ(i, 11);
|
||||
iter->Stop();
|
||||
}
|
||||
|
||||
TEST_F(MindDataTestPipeline, TestDistributedSamplerSuccess3) {
|
||||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestDistributedSamplerSuccess3.";
|
||||
// Test basic setting of distributed_sampler
|
||||
|
||||
// num_shards=4, shard_id=0, shuffle=false, num_samplers=0, seed=0, offset=-1, even_dist=true
|
||||
DistributedSampler sampler = DistributedSampler(4, 0, false, 0, 0, -1, true);
|
||||
|
||||
// Create an ImageFolder Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data/";
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, false, sampler);
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Iterate the dataset and get each row
|
||||
std::shared_ptr<Iterator> iter = ds->CreateIterator();
|
||||
EXPECT_NE(iter, nullptr);
|
||||
std::unordered_map<std::string, mindspore::MSTensor> row;
|
||||
iter->GetNextRow(&row);
|
||||
|
||||
uint64_t i = 0;
|
||||
while (row.size() != 0) {
|
||||
i++;
|
||||
auto label = row["label"];
|
||||
iter->GetNextRow(&row);
|
||||
}
|
||||
|
||||
EXPECT_EQ(i, 11);
|
||||
iter->Stop();
|
||||
}
|
||||
|
||||
TEST_F(MindDataTestPipeline, TestDistributedSamplerFail1) {
|
||||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestDistributedSamplerFail1.";
|
||||
// Test basic setting of distributed_sampler
|
||||
|
||||
// num_shards=4, shard_id=0, shuffle=false, num_samplers=0, seed=0, offset=5, even_dist=true
|
||||
// offset=5 which is greater than num_shards=4 --> will fail later
|
||||
std::shared_ptr<Sampler> sampler = std::make_shared<DistributedSampler>(4, 0, false, 0, 0, 5, false);
|
||||
EXPECT_NE(sampler, nullptr);
|
||||
|
||||
// Create an ImageFolder Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data/";
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, false, sampler);
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Iterate will fail because sampler is not initiated successfully.
|
||||
std::shared_ptr<Iterator> iter = ds->CreateIterator();
|
||||
EXPECT_EQ(iter, nullptr);
|
||||
}
|
||||
|
||||
TEST_F(MindDataTestPipeline, TestDistributedSamplerFail2) {
|
||||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestDistributedSamplerFail2.";
|
||||
// Test basic setting of distributed_sampler
|
||||
|
||||
// num_shards=4, shard_id=0, shuffle=false, num_samplers=0, seed=0, offset=5, even_dist=true
|
||||
// offset=5 which is greater than num_shards=4 --> will fail later
|
||||
Sampler *sampler = new DistributedSampler(4, 0, false, 0, 0, 5, false);
|
||||
EXPECT_NE(sampler, nullptr);
|
||||
|
||||
// Create an ImageFolder Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data/";
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, false, sampler);
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Iterate will fail because sampler is not initiated successfully.
|
||||
std::shared_ptr<Iterator> iter = ds->CreateIterator();
|
||||
EXPECT_EQ(iter, nullptr);
|
||||
}
|
||||
|
||||
TEST_F(MindDataTestPipeline, TestDistributedSamplerFail3) {
|
||||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestDistributedSamplerFail3.";
|
||||
// Test basic setting of distributed_sampler
|
||||
|
||||
// num_shards=4, shard_id=0, shuffle=false, num_samplers=0, seed=0, offset=5, even_dist=true
|
||||
// offset=5 which is greater than num_shards=4 --> will fail later
|
||||
DistributedSampler sampler = DistributedSampler(4, 0, false, 0, 0, 5, false);
|
||||
|
||||
// Create an ImageFolder Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data/";
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, false, sampler);
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Iterate will fail because sampler is not initiated successfully.
|
||||
std::shared_ptr<Iterator> iter = ds->CreateIterator();
|
||||
EXPECT_EQ(iter, nullptr);
|
||||
}
|
||||
|
||||
TEST_F(MindDataTestPipeline, TestSamplerAddChild) {
|
||||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSamplerAddChild.";
|
||||
|
||||
auto sampler = DistributedSampler(1, 0, false, 5, 0, -1, true);
|
||||
auto sampler = std::make_shared<DistributedSampler>(1, 0, false, 5, 0, -1, true);
|
||||
EXPECT_NE(sampler, nullptr);
|
||||
|
||||
auto child_sampler = SequentialSampler();
|
||||
sampler->AddChildSampler(child_sampler);
|
||||
auto child_sampler = std::make_shared<SequentialSampler>();
|
||||
EXPECT_NE(child_sampler, nullptr);
|
||||
|
||||
sampler->AddChild(child_sampler);
|
||||
|
||||
// Create an ImageFolder Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data/";
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, false, sampler);
|
||||
|
|
@ -253,12 +379,3 @@ TEST_F(MindDataTestPipeline, TestSamplerAddChild) {
|
|||
EXPECT_EQ(ds->GetDatasetSize(), 5);
|
||||
iter->Stop();
|
||||
}
|
||||
|
||||
TEST_F(MindDataTestPipeline, TestDistributedSamplerFail) {
|
||||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestDistributedSamplerFail.";
|
||||
// Test invalid offset setting of distributed_sampler
|
||||
|
||||
// offset=5 which is greater than num_shards=4
|
||||
std::shared_ptr<SamplerObj> sampler = DistributedSampler(4, 0, false, 0, 0, 5, false);
|
||||
EXPECT_EQ(sampler, nullptr);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -33,7 +33,7 @@ TEST_F(MindDataTestPipeline, TestComposeSuccess) {
|
|||
|
||||
// Create an ImageFolder Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data/";
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, false, RandomSampler(false, 3));
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, false, std::make_shared<RandomSampler>(false, 3));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create objects for the tensor ops
|
||||
|
|
@ -68,6 +68,7 @@ TEST_F(MindDataTestPipeline, TestComposeSuccess) {
|
|||
|
||||
EXPECT_EQ(i, 3);
|
||||
|
||||
|
||||
// Manually terminate the pipeline
|
||||
iter->Stop();
|
||||
}
|
||||
|
|
@ -77,7 +78,7 @@ TEST_F(MindDataTestPipeline, TestComposeFail1) {
|
|||
|
||||
// Create a Cifar10 Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
|
||||
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10));
|
||||
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Resize: Non-positive size value: -1 at element: 0
|
||||
|
|
@ -100,7 +101,7 @@ TEST_F(MindDataTestPipeline, TestComposeFail2) {
|
|||
|
||||
// Create a Cifar10 Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
|
||||
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10));
|
||||
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Compose: transform ops must not be null
|
||||
|
|
@ -121,7 +122,7 @@ TEST_F(MindDataTestPipeline, TestComposeFail3) {
|
|||
|
||||
// Create a Cifar10 Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
|
||||
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10));
|
||||
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Compose: transform list must not be empty
|
||||
|
|
@ -142,7 +143,7 @@ TEST_F(MindDataTestPipeline, TestDuplicateSuccess) {
|
|||
|
||||
// Create a Cifar10 Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
|
||||
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10));
|
||||
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create objects for the tensor ops
|
||||
|
|
@ -182,7 +183,7 @@ TEST_F(MindDataTestPipeline, TestOneHotSuccess1) {
|
|||
// Create a Cifar10 Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
|
||||
int number_of_classes = 10;
|
||||
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10));
|
||||
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create objects for the tensor ops
|
||||
|
|
@ -248,7 +249,7 @@ TEST_F(MindDataTestPipeline, TestOneHotSuccess1) {
|
|||
TEST_F(MindDataTestPipeline, TestOneHotSuccess2) {
|
||||
// Create a Cifar10 Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
|
||||
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10));
|
||||
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create a Batch operation on ds
|
||||
|
|
@ -299,7 +300,7 @@ TEST_F(MindDataTestPipeline, TestOneHotFail1) {
|
|||
|
||||
// Create a Cifar10 Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
|
||||
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10));
|
||||
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// incorrect num_class
|
||||
|
|
@ -320,7 +321,7 @@ TEST_F(MindDataTestPipeline, TestOneHotFail2) {
|
|||
|
||||
// Create a Cifar10 Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
|
||||
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10));
|
||||
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// incorrect num_class
|
||||
|
|
@ -341,7 +342,7 @@ TEST_F(MindDataTestPipeline, TestRandomApplySuccess) {
|
|||
|
||||
// Create an ImageFolder Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data/";
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 5));
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 5));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create objects for the tensor ops
|
||||
|
|
@ -382,7 +383,7 @@ TEST_F(MindDataTestPipeline, TestRandomApplyFail1) {
|
|||
|
||||
// Create a Cifar10 Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
|
||||
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10));
|
||||
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Resize: Non-positive size value: -1 at element: 0
|
||||
|
|
@ -405,7 +406,7 @@ TEST_F(MindDataTestPipeline, TestRandomApplyFail2) {
|
|||
|
||||
// Create a Cifar10 Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
|
||||
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10));
|
||||
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// RandomApply: transform ops must not be null
|
||||
|
|
@ -426,7 +427,7 @@ TEST_F(MindDataTestPipeline, TestRandomApplyFail3) {
|
|||
|
||||
// Create a Cifar10 Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
|
||||
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10));
|
||||
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// RandomApply: Probability has to be between 0 and 1
|
||||
|
|
@ -447,7 +448,7 @@ TEST_F(MindDataTestPipeline, TestRandomApplyFail4) {
|
|||
|
||||
// Create a Cifar10 Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
|
||||
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10));
|
||||
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// RandomApply: transform list must not be empty
|
||||
|
|
@ -468,7 +469,7 @@ TEST_F(MindDataTestPipeline, TestRandomChoiceSuccess) {
|
|||
|
||||
// Create an ImageFolder Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data/";
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 3));
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 3));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create objects for the tensor ops
|
||||
|
|
@ -510,7 +511,8 @@ TEST_F(MindDataTestPipeline, TestRandomChoiceFail1) {
|
|||
|
||||
// Create a Cifar10 Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
|
||||
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10));
|
||||
RandomSampler sampler = RandomSampler(false, 10);
|
||||
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", sampler);
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Resize: Non-positive size value: -1 at element: 0
|
||||
|
|
@ -533,7 +535,7 @@ TEST_F(MindDataTestPipeline, TestRandomChoiceFail2) {
|
|||
|
||||
// Create a Cifar10 Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
|
||||
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10));
|
||||
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// RandomChoice: transform ops must not be null
|
||||
|
|
@ -554,7 +556,7 @@ TEST_F(MindDataTestPipeline, TestRandomChoiceFail3) {
|
|||
|
||||
// Create a Cifar10 Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
|
||||
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10));
|
||||
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// RandomChoice: transform list must not be empty
|
||||
|
|
@ -575,7 +577,7 @@ TEST_F(MindDataTestPipeline, TestTypeCastSuccess) {
|
|||
|
||||
// Create a Cifar10 Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
|
||||
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 1));
|
||||
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 1));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create an iterator over the result of the above dataset
|
||||
|
|
@ -625,7 +627,7 @@ TEST_F(MindDataTestPipeline, TestTypeCastFail) {
|
|||
|
||||
// Create a Cifar10 Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
|
||||
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10));
|
||||
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// incorrect data type
|
||||
|
|
|
|||
|
|
@ -32,7 +32,7 @@ TEST_F(MindDataTestPipeline, TestAutoContrastSuccess1) {
|
|||
|
||||
// Create an ImageFolder Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data/";
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 5));
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 5));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create a Repeat operation on ds
|
||||
|
|
@ -81,7 +81,7 @@ TEST_F(MindDataTestPipeline, TestAutoContrastSuccess2) {
|
|||
|
||||
// Create an ImageFolder Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data/";
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 5));
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 5));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create a Repeat operation on ds
|
||||
|
|
@ -142,7 +142,7 @@ TEST_F(MindDataTestPipeline, TestCenterCrop) {
|
|||
|
||||
// Create an ImageFolder Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data/";
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 5));
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 5));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create a Repeat operation on ds
|
||||
|
|
@ -228,7 +228,7 @@ TEST_F(MindDataTestPipeline, TestCutMixBatchSuccess1) {
|
|||
// Create a Cifar10 Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
|
||||
int number_of_classes = 10;
|
||||
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10));
|
||||
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create objects for the tensor ops
|
||||
|
|
@ -299,7 +299,7 @@ TEST_F(MindDataTestPipeline, TestCutMixBatchSuccess2) {
|
|||
// Create a Cifar10 Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
|
||||
int number_of_classes = 10;
|
||||
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10));
|
||||
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create a Batch operation on ds
|
||||
|
|
@ -358,7 +358,7 @@ TEST_F(MindDataTestPipeline, TestCutMixBatchFail1) {
|
|||
|
||||
// Create a Cifar10 Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
|
||||
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10));
|
||||
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create a Batch operation on ds
|
||||
|
|
@ -393,7 +393,7 @@ TEST_F(MindDataTestPipeline, TestCutMixBatchFail2) {
|
|||
|
||||
// Create a Cifar10 Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
|
||||
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10));
|
||||
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create a Batch operation on ds
|
||||
|
|
@ -428,7 +428,7 @@ TEST_F(MindDataTestPipeline, TestCutMixBatchFail3) {
|
|||
|
||||
// Create a Cifar10 Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
|
||||
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10));
|
||||
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create a Batch operation on ds
|
||||
|
|
@ -462,7 +462,7 @@ TEST_F(MindDataTestPipeline, TestCutMixBatchFail4) {
|
|||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestCutMixBatchFail4 with invalid greater than 1 prob parameter.";
|
||||
|
||||
std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
|
||||
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10));
|
||||
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create a Batch operation on ds
|
||||
|
|
@ -521,7 +521,7 @@ TEST_F(MindDataTestPipeline, TestCutOut) {
|
|||
|
||||
// Create an ImageFolder Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data/";
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create a Repeat operation on ds
|
||||
|
|
@ -573,7 +573,7 @@ TEST_F(MindDataTestPipeline, TestDecode) {
|
|||
|
||||
// Create an ImageFolder Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data/";
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, false, RandomSampler(false, 10));
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, false, std::make_shared<RandomSampler>(false, 10));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create a Repeat operation on ds
|
||||
|
|
@ -620,7 +620,7 @@ TEST_F(MindDataTestPipeline, TestHwcToChw) {
|
|||
|
||||
// Create an ImageFolder Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data/";
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create a Repeat operation on ds
|
||||
|
|
@ -672,7 +672,7 @@ TEST_F(MindDataTestPipeline, TestInvert) {
|
|||
|
||||
// Create an ImageFolder Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data/";
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 20));
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 20));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create objects for the tensor ops
|
||||
|
|
@ -710,7 +710,7 @@ TEST_F(MindDataTestPipeline, TestMixUpBatchFail1) {
|
|||
// FIXME: For error tests, need to check for failure from CreateIterator execution
|
||||
// Create a Cifar10 Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
|
||||
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10));
|
||||
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create a Batch operation on ds
|
||||
|
|
@ -744,7 +744,7 @@ TEST_F(MindDataTestPipeline, TestMixUpBatchFail2) {
|
|||
// FIXME: For error tests, need to check for failure from CreateIterator execution
|
||||
// Create a Cifar10 Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
|
||||
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10));
|
||||
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create a Batch operation on ds
|
||||
|
|
@ -778,7 +778,7 @@ TEST_F(MindDataTestPipeline, TestMixUpBatchSuccess1) {
|
|||
|
||||
// Create a Cifar10 Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
|
||||
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10));
|
||||
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create a Batch operation on ds
|
||||
|
|
@ -829,7 +829,7 @@ TEST_F(MindDataTestPipeline, TestMixUpBatchSuccess2) {
|
|||
|
||||
// Create a Cifar10 Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
|
||||
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10));
|
||||
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create a Batch operation on ds
|
||||
|
|
@ -880,7 +880,7 @@ TEST_F(MindDataTestPipeline, TestNormalize) {
|
|||
|
||||
// Create an ImageFolder Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data/";
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create a Repeat operation on ds
|
||||
|
|
@ -958,7 +958,7 @@ TEST_F(MindDataTestPipeline, TestNormalizePad) {
|
|||
|
||||
// Create an ImageFolder Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data/";
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create a Repeat operation on ds
|
||||
|
|
@ -1025,7 +1025,7 @@ TEST_F(MindDataTestPipeline, TestPad) {
|
|||
|
||||
// Create an ImageFolder Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data/";
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create a Repeat operation on ds
|
||||
|
|
|
|||
|
|
@ -30,7 +30,7 @@ TEST_F(MindDataTestPipeline, TestBoundingBoxAugmentSuccess) {
|
|||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestBoundingBoxAugmentSuccess.";
|
||||
// Create an VOC Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testVOC2012_2";
|
||||
std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", {}, true, SequentialSampler(0, 3));
|
||||
std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", {}, true, std::make_shared<SequentialSampler>(0, 3));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
/* FIXME - Resolve BoundingBoxAugment to properly handle TensorTransform input
|
||||
|
|
|
|||
|
|
@ -30,7 +30,7 @@ TEST_F(MindDataTestPipeline, TestRescaleSucess1) {
|
|||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRescaleSucess1.";
|
||||
// Create an ImageFolder Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data/";
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, SequentialSampler(0, 1));
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<SequentialSampler>(0, 1));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create an iterator over the result of the above dataset
|
||||
|
|
@ -76,7 +76,7 @@ TEST_F(MindDataTestPipeline, TestRescaleSucess2) {
|
|||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRescaleSucess2 with different params.";
|
||||
// Create an ImageFolder Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data/";
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 1));
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 1));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create objects for the tensor ops
|
||||
|
|
@ -121,7 +121,7 @@ TEST_F(MindDataTestPipeline, TestResize1) {
|
|||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestResize1 with single integer input.";
|
||||
// Create an ImageFolder Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data/";
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 6));
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 6));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create a Repeat operation on ds
|
||||
|
|
@ -183,7 +183,7 @@ TEST_F(MindDataTestPipeline, TestResizeWithBBoxSuccess) {
|
|||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestResizeWithBBoxSuccess.";
|
||||
// Create an VOC Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testVOC2012_2";
|
||||
std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", {}, true, SequentialSampler(0, 3));
|
||||
std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", {}, true, std::make_shared<SequentialSampler>(0, 3));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create objects for the tensor ops
|
||||
|
|
|
|||
|
|
@ -31,7 +31,7 @@ TEST_F(MindDataTestPipeline, TestRandomSelectSubpolicySuccess) {
|
|||
|
||||
// Create an ImageFolder Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data/";
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 7));
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 7));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
/* FIXME - Resolve RandomSelectSubpolicy to properly handle TensorTransform input
|
||||
|
|
|
|||
|
|
@ -46,7 +46,7 @@ TEST_F(MindDataTestPipeline, TestRandomAffineSuccess1) {
|
|||
|
||||
// Create an ImageFolder Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data/";
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create a Repeat operation on ds
|
||||
|
|
@ -96,7 +96,7 @@ TEST_F(MindDataTestPipeline, TestRandomAffineSuccess2) {
|
|||
|
||||
// Create an ImageFolder Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data/";
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create a Repeat operation on ds
|
||||
|
|
@ -145,7 +145,7 @@ TEST_F(MindDataTestPipeline, TestRandomColor) {
|
|||
|
||||
// Create an ImageFolder Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data/";
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create a Repeat operation on ds
|
||||
|
|
@ -209,7 +209,7 @@ TEST_F(MindDataTestPipeline, TestRandomColorAdjust) {
|
|||
|
||||
// Create an ImageFolder Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data/";
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create a Repeat operation on ds
|
||||
|
|
@ -303,7 +303,7 @@ TEST_F(MindDataTestPipeline, TestRandomCropSuccess) {
|
|||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRandomCropSuccess.";
|
||||
// Create an VOC Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testVOC2012_2";
|
||||
std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", {}, true, SequentialSampler(0, 10));
|
||||
std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", {}, true, std::make_shared<SequentialSampler>(0, 10));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create objects for the tensor ops
|
||||
|
|
@ -366,7 +366,7 @@ TEST_F(MindDataTestPipeline, TestRandomCropFail) {
|
|||
// FIXME: For error tests, need to check for failure from CreateIterator execution
|
||||
// Create an VOC Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testVOC2012_2";
|
||||
std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", {}, true, SequentialSampler(0, 3));
|
||||
std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", {}, true, std::make_shared<SequentialSampler>(0, 3));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create objects for the tensor ops
|
||||
|
|
@ -406,7 +406,7 @@ TEST_F(MindDataTestPipeline, TestRandomCropWithBboxSuccess) {
|
|||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRandomCropWithBboxSuccess.";
|
||||
// Create an VOC Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testVOC2012_2";
|
||||
std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", {}, true, SequentialSampler(0, 3));
|
||||
std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", {}, true, std::make_shared<SequentialSampler>(0, 3));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create objects for the tensor ops
|
||||
|
|
@ -446,7 +446,7 @@ TEST_F(MindDataTestPipeline, TestRandomCropWithBboxFail) {
|
|||
// FIXME: For error tests, need to check for failure from CreateIterator execution
|
||||
// Create an VOC Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testVOC2012_2";
|
||||
std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", {}, true, SequentialSampler(0, 3));
|
||||
std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", {}, true, std::make_shared<SequentialSampler>(0, 3));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create objects for the tensor ops
|
||||
|
|
@ -493,7 +493,7 @@ TEST_F(MindDataTestPipeline, TestRandomHorizontalFlipWithBBoxSuccess) {
|
|||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRandomHorizontalFlipWithBBoxSuccess.";
|
||||
// Create an VOC Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testVOC2012_2";
|
||||
std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", {}, true, SequentialSampler(0, 3));
|
||||
std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", {}, true, std::make_shared<SequentialSampler>(0, 3));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create objects for the tensor ops
|
||||
|
|
@ -531,7 +531,7 @@ TEST_F(MindDataTestPipeline, TestRandomHorizontalFlipWithBBoxFail) {
|
|||
// FIXME: For error tests, need to check for failure from CreateIterator execution
|
||||
// Create an VOC Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testVOC2012_2";
|
||||
std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", {}, true, SequentialSampler(0, 3));
|
||||
std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", {}, true, std::make_shared<SequentialSampler>(0, 3));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create objects for the tensor ops
|
||||
|
|
@ -548,7 +548,7 @@ TEST_F(MindDataTestPipeline, TestRandomHorizontalAndVerticalFlip) {
|
|||
|
||||
// Create an ImageFolder Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data/";
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create a Repeat operation on ds
|
||||
|
|
@ -618,7 +618,7 @@ TEST_F(MindDataTestPipeline, TestRandomPosterizeSuccess1) {
|
|||
|
||||
// Create an ImageFolder Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data/";
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create a Repeat operation on ds
|
||||
|
|
@ -667,7 +667,7 @@ TEST_F(MindDataTestPipeline, TestRandomPosterizeSuccess2) {
|
|||
|
||||
// Create an ImageFolder Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data/";
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create a Repeat operation on ds
|
||||
|
|
@ -716,7 +716,7 @@ TEST_F(MindDataTestPipeline, TestRandomResizeSuccess1) {
|
|||
|
||||
// Create an ImageFolder Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data/";
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 5));
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 5));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create objects for the tensor ops
|
||||
|
|
@ -756,7 +756,7 @@ TEST_F(MindDataTestPipeline, TestRandomResizeSuccess2) {
|
|||
|
||||
// Create an ImageFolder Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data/";
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 3));
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 3));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create a Repeat operation on ds
|
||||
|
|
@ -821,7 +821,7 @@ TEST_F(MindDataTestPipeline, TestRandomResizeWithBBoxSuccess1) {
|
|||
|
||||
// Create an VOC Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testVOC2012_2";
|
||||
std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", {}, true, SequentialSampler(0, 3));
|
||||
std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", {}, true, std::make_shared<SequentialSampler>(0, 3));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create objects for the tensor ops
|
||||
|
|
@ -861,7 +861,7 @@ TEST_F(MindDataTestPipeline, TestRandomResizeWithBBoxSuccess2) {
|
|||
|
||||
// Create an VOC Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testVOC2012_2";
|
||||
std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", {}, true, SequentialSampler(0, 4));
|
||||
std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", {}, true, std::make_shared<SequentialSampler>(0, 4));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create a Repeat operation on ds
|
||||
|
|
@ -921,7 +921,7 @@ TEST_F(MindDataTestPipeline, TestRandomResizedCropSuccess1) {
|
|||
// Testing RandomResizedCrop with default values
|
||||
// Create a Cifar10 Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
|
||||
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10));
|
||||
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create objects for the tensor ops
|
||||
|
|
@ -960,7 +960,7 @@ TEST_F(MindDataTestPipeline, TestRandomResizedCropSuccess2) {
|
|||
// Testing RandomResizedCrop with non-default values
|
||||
// Create a Cifar10 Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
|
||||
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10));
|
||||
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create objects for the tensor ops
|
||||
|
|
@ -1000,7 +1000,7 @@ TEST_F(MindDataTestPipeline, TestRandomResizedCropFail1) {
|
|||
// This should fail because size has negative value
|
||||
// Create a Cifar10 Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
|
||||
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10));
|
||||
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create objects for the tensor ops
|
||||
|
|
@ -1020,7 +1020,7 @@ TEST_F(MindDataTestPipeline, TestRandomResizedCropFail2) {
|
|||
// This should fail because scale isn't in {min, max} format
|
||||
// Create a Cifar10 Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
|
||||
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10));
|
||||
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create objects for the tensor ops
|
||||
|
|
@ -1040,7 +1040,7 @@ TEST_F(MindDataTestPipeline, TestRandomResizedCropFail3) {
|
|||
// This should fail because ratio isn't in {min, max} format
|
||||
// Create a Cifar10 Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
|
||||
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10));
|
||||
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create objects for the tensor ops
|
||||
|
|
@ -1060,7 +1060,7 @@ TEST_F(MindDataTestPipeline, TestRandomResizedCropFail4) {
|
|||
// This should fail because scale has a size of more than 2
|
||||
// Create a Cifar10 Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
|
||||
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10));
|
||||
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create objects for the tensor ops
|
||||
|
|
@ -1080,7 +1080,7 @@ TEST_F(MindDataTestPipeline, TestRandomResizedCropWithBBoxSuccess1) {
|
|||
// Testing RandomResizedCropWithBBox with default values
|
||||
// Create an VOC Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testVOC2012_2";
|
||||
std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", {}, true, SequentialSampler(0, 4));
|
||||
std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", {}, true, std::make_shared<SequentialSampler>(0, 4));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create objects for the tensor ops
|
||||
|
|
@ -1119,7 +1119,7 @@ TEST_F(MindDataTestPipeline, TestRandomResizedCropWithBBoxSuccess2) {
|
|||
// Testing RandomResizedCropWithBBox with non-default values
|
||||
// Create an VOC Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testVOC2012_2";
|
||||
std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", {}, true, SequentialSampler(0, 4));
|
||||
std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", {}, true, std::make_shared<SequentialSampler>(0, 4));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create objects for the tensor ops
|
||||
|
|
@ -1160,7 +1160,7 @@ TEST_F(MindDataTestPipeline, TestRandomResizedCropWithBBoxFail1) {
|
|||
// This should fail because size has negative value
|
||||
// Create a Cifar10 Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
|
||||
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10));
|
||||
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create objects for the tensor ops
|
||||
|
|
@ -1173,7 +1173,7 @@ TEST_F(MindDataTestPipeline, TestRandomResizedCropWithBBoxFail2) {
|
|||
// This should fail because scale isn't in {min, max} format
|
||||
// Create a Cifar10 Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
|
||||
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10));
|
||||
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create objects for the tensor ops
|
||||
|
|
@ -1186,7 +1186,7 @@ TEST_F(MindDataTestPipeline, TestRandomResizedCropWithBBoxFail3) {
|
|||
// This should fail because ratio isn't in {min, max} format
|
||||
// Create a Cifar10 Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
|
||||
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10));
|
||||
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create objects for the tensor ops
|
||||
|
|
@ -1199,7 +1199,7 @@ TEST_F(MindDataTestPipeline, TestRandomResizedCropWithBBoxFail4) {
|
|||
// This should fail because scale has a size of more than 2
|
||||
// Create a Cifar10 Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
|
||||
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10));
|
||||
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create objects for the tensor ops
|
||||
|
|
@ -1212,7 +1212,7 @@ TEST_F(MindDataTestPipeline, TestRandomRotation) {
|
|||
|
||||
// Create an ImageFolder Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data/";
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create a Repeat operation on ds
|
||||
|
|
@ -1273,7 +1273,7 @@ TEST_F(MindDataTestPipeline, TestRandomRotationFail) {
|
|||
// FIXME: For error tests, need to check for failure from CreateIterator execution
|
||||
// Create an ImageFolder Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data/";
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create objects for the tensor ops
|
||||
|
|
@ -1309,7 +1309,7 @@ TEST_F(MindDataTestPipeline, TestRandomSharpness) {
|
|||
|
||||
// Create an ImageFolder Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data/";
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create a Repeat operation on ds
|
||||
|
|
@ -1374,7 +1374,7 @@ TEST_F(MindDataTestPipeline, TestRandomSolarizeSucess1) {
|
|||
|
||||
// Create an ImageFolder Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data/";
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create objects for the tensor ops
|
||||
|
|
@ -1414,7 +1414,7 @@ TEST_F(MindDataTestPipeline, TestRandomSolarizeSucess2) {
|
|||
|
||||
// Create an ImageFolder Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data/";
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create objects for the tensor ops
|
||||
|
|
@ -1484,7 +1484,7 @@ TEST_F(MindDataTestPipeline, TestRandomVerticalFlipWithBBoxSuccess) {
|
|||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRandomVerticalFlipWithBBoxSuccess.";
|
||||
// Create an VOC Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testVOC2012_2";
|
||||
std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", {}, true, SequentialSampler(0, 3));
|
||||
std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", {}, true, std::make_shared<SequentialSampler>(0, 3));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create objects for the tensor ops
|
||||
|
|
@ -1522,7 +1522,7 @@ TEST_F(MindDataTestPipeline, TestRandomVerticalFlipWithBBoxFail) {
|
|||
// FIXME: For error tests, need to check for failure from CreateIterator execution
|
||||
// Create an VOC Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testVOC2012_2";
|
||||
std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", {}, true, SequentialSampler(0, 3));
|
||||
std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", {}, true, std::make_shared<SequentialSampler>(0, 3));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create objects for the tensor ops
|
||||
|
|
|
|||
|
|
@ -32,7 +32,7 @@ TEST_F(MindDataTestPipeline, TestSoftDvppDecodeRandomCropResizeJpegSuccess1) {
|
|||
|
||||
// Create an ImageFolder Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data/";
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, false, RandomSampler(false, 4));
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, false, std::make_shared<RandomSampler>(false, 4));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create objects for the tensor ops
|
||||
|
|
@ -74,7 +74,7 @@ TEST_F(MindDataTestPipeline, TestSoftDvppDecodeRandomCropResizeJpegSuccess2) {
|
|||
|
||||
// Create an ImageFolder Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data/";
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, false, RandomSampler(false, 6));
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, false, std::make_shared<RandomSampler>(false, 6));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create objects for the tensor ops
|
||||
|
|
@ -158,7 +158,7 @@ TEST_F(MindDataTestPipeline, TestSoftDvppDecodeResizeJpegSuccess1) {
|
|||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSoftDvppDecodeResizeJpegSuccess1 with single integer input.";
|
||||
// Create an ImageFolder Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data/";
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, false, RandomSampler(false, 4));
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, false, std::make_shared<RandomSampler>(false, 4));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create a Repeat operation on ds
|
||||
|
|
@ -201,7 +201,7 @@ TEST_F(MindDataTestPipeline, TestSoftDvppDecodeResizeJpegSuccess2) {
|
|||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSoftDvppDecodeResizeJpegSuccess2 with (height, width) input.";
|
||||
// Create an ImageFolder Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data/";
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, false, RandomSampler(false, 2));
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, false, std::make_shared<RandomSampler>(false, 2));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create SoftDvppDecodeResizeJpeg object with single integer input
|
||||
|
|
|
|||
|
|
@ -78,7 +78,7 @@ TEST_F(MindDataTestPipeline, TestUniformAugWithOps) {
|
|||
|
||||
// Create a Mnist Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testMnistData/";
|
||||
std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", RandomSampler(false, 20));
|
||||
std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", std::make_shared<RandomSampler>(false, 20));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create a Repeat operation on ds
|
||||
|
|
|
|||
|
|
@ -36,7 +36,7 @@ TEST_F(MindDataTestTensorOpFusionPass, RandomCropDecodeResizeDisabled) {
|
|||
MS_LOG(INFO) << "Doing MindDataTestTensorOpFusionPass-RandomCropDecodeResizeDisabled";
|
||||
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data/";
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, false, SequentialSampler(0, 11));
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11));
|
||||
|
||||
// Create objects for the tensor ops
|
||||
std::shared_ptr<TensorTransform> decode(new vision::Decode());
|
||||
|
|
@ -67,7 +67,7 @@ TEST_F(MindDataTestTensorOpFusionPass, RandomCropDecodeResizeEnabled) {
|
|||
MS_LOG(INFO) << "Doing MindDataTestTensorOpFusionPass-RandomCropDecodeResizeEnabled";
|
||||
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data/";
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, false, SequentialSampler(0, 11));
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11));
|
||||
|
||||
// Create objects for the tensor ops
|
||||
std::shared_ptr<TensorTransform> decode(new vision::Decode());
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
/**
|
||||
* Copyright 2020 Huawei Technologies Co., Ltd
|
||||
* Copyright 2020-2021 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
|
|
@ -48,7 +48,7 @@ TEST_F(MindDataTestTreeAdapter, TestSimpleTreeAdapter) {
|
|||
|
||||
// Create a Mnist Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testMnistData/";
|
||||
std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", SequentialSampler(0, 4));
|
||||
std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", std::make_shared<SequentialSampler>(0, 4));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
ds = ds->Batch(2);
|
||||
|
|
@ -83,7 +83,7 @@ TEST_F(MindDataTestTreeAdapter, TestTreeAdapterWithRepeat) {
|
|||
|
||||
// Create a Mnist Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testMnistData/";
|
||||
std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", SequentialSampler(0, 3));
|
||||
std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", std::make_shared<SequentialSampler>(0, 3));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
ds = ds->Batch(2, false);
|
||||
|
|
@ -115,7 +115,7 @@ TEST_F(MindDataTestTreeAdapter, TestProjectMapTreeAdapter) {
|
|||
|
||||
// Create an ImageFolder Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data/";
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, SequentialSampler(0, 2));
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<SequentialSampler>(0, 2));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create objects for the tensor ops
|
||||
|
|
|
|||
|
|
@ -49,9 +49,9 @@ TEST_F(MindDataTestTreeModifying, AppendChild) {
|
|||
* ds1
|
||||
*/
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data/";
|
||||
std::shared_ptr<Dataset> ds1 = ImageFolder(folder_path, false, SequentialSampler(0, 11));
|
||||
std::shared_ptr<Dataset> ds2 = ImageFolder(folder_path, false, SequentialSampler(0, 11));
|
||||
std::shared_ptr<Dataset> ds6 = ImageFolder(folder_path, false, SequentialSampler(0, 11));
|
||||
std::shared_ptr<Dataset> ds1 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11));
|
||||
std::shared_ptr<Dataset> ds2 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11));
|
||||
std::shared_ptr<Dataset> ds6 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11));
|
||||
std::shared_ptr<Dataset> ds3 = ds1->Take(10);
|
||||
std::shared_ptr<Dataset> ds4 = ds3->Concat({ds2});
|
||||
Status rc;
|
||||
|
|
@ -105,9 +105,9 @@ TEST_F(MindDataTestTreeModifying, InsertChildAt01) {
|
|||
*
|
||||
*/
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data/";
|
||||
std::shared_ptr<Dataset> ds1 = ImageFolder(folder_path, false, SequentialSampler(0, 11));
|
||||
std::shared_ptr<Dataset> ds1 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11));
|
||||
std::shared_ptr<Dataset> ds3 = ds1->Take(10);
|
||||
std::shared_ptr<Dataset> ds5 = ImageFolder(folder_path, false, SequentialSampler(0, 11));
|
||||
std::shared_ptr<Dataset> ds5 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11));
|
||||
std::shared_ptr<Dataset> ds2 = ds5->Repeat(4);
|
||||
std::shared_ptr<Dataset> ds4 = ds3->Concat({ds2});
|
||||
Status rc;
|
||||
|
|
@ -119,7 +119,7 @@ TEST_F(MindDataTestTreeModifying, InsertChildAt01) {
|
|||
EXPECT_EQ(rc, Status::OK());
|
||||
// Descend two levels as Compile adds the root node and the epochctrl node on top of ds4
|
||||
std::shared_ptr<DatasetNode> ds4_node = ir_tree->RootIRNode()->Children()[0]->Children()[0];
|
||||
std::shared_ptr<Dataset> ds6 = ImageFolder(folder_path, false, SequentialSampler(0, 11));
|
||||
std::shared_ptr<Dataset> ds6 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11));
|
||||
std::shared_ptr<DatasetNode> ds6_to_insert = ds6->IRNode();
|
||||
std::shared_ptr<DatasetNode> ds2_node = ds4_node->Children()[1];
|
||||
rc = ds4_node->InsertChildAt(1, ds6_to_insert);
|
||||
|
|
@ -132,7 +132,7 @@ TEST_F(MindDataTestTreeModifying, InsertChildAt01) {
|
|||
EXPECT_EQ(rc, Status::OK());
|
||||
// Descend two levels as Compile adds the root node and the epochctrl node on top of ds4
|
||||
ds4_node = ir_tree->RootIRNode()->Children()[0]->Children()[0];
|
||||
ds6 = ImageFolder(folder_path, false, SequentialSampler(0, 11));
|
||||
ds6 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11));
|
||||
ds6_to_insert = ds6->IRNode();
|
||||
std::shared_ptr<DatasetNode> ds3_node = ds4_node->Children()[0];
|
||||
rc = ds4_node->InsertChildAt(0, ds6_to_insert);
|
||||
|
|
@ -145,7 +145,7 @@ TEST_F(MindDataTestTreeModifying, InsertChildAt01) {
|
|||
EXPECT_EQ(rc, Status::OK());
|
||||
// Descend two levels as Compile adds the root node and the epochctrl node on top of ds4
|
||||
ds4_node = ir_tree->RootIRNode()->Children()[0]->Children()[0];
|
||||
ds6 = ImageFolder(folder_path, false, SequentialSampler(0, 11));
|
||||
ds6 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11));
|
||||
ds6_to_insert = ds6->IRNode();
|
||||
rc = ds4_node->InsertChildAt(2, ds6_to_insert);
|
||||
EXPECT_EQ(rc, Status::OK());
|
||||
|
|
@ -165,9 +165,9 @@ TEST_F(MindDataTestTreeModifying, InsertChildAt04) {
|
|||
*
|
||||
*/
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data/";
|
||||
std::shared_ptr<Dataset> ds1 = ImageFolder(folder_path, false, SequentialSampler(0, 11));
|
||||
std::shared_ptr<Dataset> ds1 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11));
|
||||
std::shared_ptr<Dataset> ds3 = ds1->Take(10);
|
||||
std::shared_ptr<Dataset> ds5 = ImageFolder(folder_path, false, SequentialSampler(0, 11));
|
||||
std::shared_ptr<Dataset> ds5 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11));
|
||||
std::shared_ptr<Dataset> ds2 = ds5->Repeat(4);
|
||||
std::shared_ptr<Dataset> ds4 = ds3->Concat({ds2});
|
||||
Status rc;
|
||||
|
|
@ -179,7 +179,7 @@ TEST_F(MindDataTestTreeModifying, InsertChildAt04) {
|
|||
EXPECT_EQ(rc, Status::OK());
|
||||
// Descend two levels as Compile adds the root node and the epochctrl node on top of ds4
|
||||
std::shared_ptr<DatasetNode> ds4_node = ir_tree->RootIRNode()->Children()[0]->Children()[0];
|
||||
std::shared_ptr<Dataset> ds6 = ImageFolder(folder_path, false, SequentialSampler(0, 11));
|
||||
std::shared_ptr<Dataset> ds6 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11));
|
||||
std::shared_ptr<DatasetNode> ds6_to_insert = ds6->IRNode();
|
||||
std::shared_ptr<DatasetNode> ds3_node = ds4_node->Children()[0];
|
||||
std::shared_ptr<DatasetNode> ds2_node = ds4_node->Children()[1];
|
||||
|
|
@ -193,7 +193,7 @@ TEST_F(MindDataTestTreeModifying, InsertChildAt04) {
|
|||
EXPECT_EQ(rc, Status::OK());
|
||||
// Descend two levels as Compile adds the root node and the epochctrl node on top of ds4
|
||||
ds4_node = ir_tree->RootIRNode()->Children()[0]->Children()[0];
|
||||
ds6 = ImageFolder(folder_path, false, SequentialSampler(0, 11));
|
||||
ds6 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11));
|
||||
ds6_to_insert = ds6->IRNode();
|
||||
ds3_node = ds4_node->Children()[0];
|
||||
ds2_node = ds4_node->Children()[1];
|
||||
|
|
@ -246,8 +246,8 @@ TEST_F(MindDataTestTreeModifying, InsertAbove01) {
|
|||
*/
|
||||
// Case 1
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data/";
|
||||
std::shared_ptr<Dataset> ds1 = ImageFolder(folder_path, false, SequentialSampler(0, 11));
|
||||
std::shared_ptr<Dataset> ds2 = ImageFolder(folder_path, false, SequentialSampler(0, 11));
|
||||
std::shared_ptr<Dataset> ds1 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11));
|
||||
std::shared_ptr<Dataset> ds2 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11));
|
||||
std::shared_ptr<Dataset> ds3 = ds1->Take(10);
|
||||
std::shared_ptr<Dataset> ds4 = ds3->Concat({ds2});
|
||||
Status rc;
|
||||
|
|
@ -271,8 +271,8 @@ TEST_F(MindDataTestTreeModifying, InsertAbove02) {
|
|||
|
||||
// Case 2
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data/";
|
||||
std::shared_ptr<Dataset> ds1 = ImageFolder(folder_path, false, SequentialSampler(0, 11));
|
||||
std::shared_ptr<Dataset> ds2 = ImageFolder(folder_path, false, SequentialSampler(0, 11));
|
||||
std::shared_ptr<Dataset> ds1 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11));
|
||||
std::shared_ptr<Dataset> ds2 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11));
|
||||
std::shared_ptr<Dataset> ds3 = ds1->Take(10);
|
||||
std::shared_ptr<Dataset> ds4 = ds3 + ds2;
|
||||
Status rc;
|
||||
|
|
@ -296,8 +296,8 @@ TEST_F(MindDataTestTreeModifying, InsertAbove03) {
|
|||
|
||||
// Case 3
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data/";
|
||||
std::shared_ptr<Dataset> ds1 = ImageFolder(folder_path, false, SequentialSampler(0, 11));
|
||||
std::shared_ptr<Dataset> ds2 = ImageFolder(folder_path, false, SequentialSampler(0, 11));
|
||||
std::shared_ptr<Dataset> ds1 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11));
|
||||
std::shared_ptr<Dataset> ds2 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11));
|
||||
std::shared_ptr<Dataset> ds3 = ds1->Take(10);
|
||||
std::shared_ptr<Dataset> ds4 = ds3->Concat({ds2});
|
||||
Status rc;
|
||||
|
|
@ -357,11 +357,11 @@ TEST_F(MindDataTestTreeModifying, Drop01) {
|
|||
*
|
||||
*/
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data/";
|
||||
std::shared_ptr<Dataset> ds7 = ImageFolder(folder_path, false, SequentialSampler(0, 11));
|
||||
std::shared_ptr<Dataset> ds7 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11));
|
||||
std::shared_ptr<Dataset> ds8 = ds7->Take(20);
|
||||
std::shared_ptr<Dataset> ds9 = ds8->Skip(1);
|
||||
std::shared_ptr<Dataset> ds3 = ImageFolder(folder_path, false, SequentialSampler(0, 11));
|
||||
std::shared_ptr<Dataset> ds2 = ImageFolder(folder_path, false, SequentialSampler(0, 11));
|
||||
std::shared_ptr<Dataset> ds3 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11));
|
||||
std::shared_ptr<Dataset> ds2 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11));
|
||||
std::shared_ptr<Dataset> ds4 = ds3->Concat({ds2});
|
||||
std::shared_ptr<Dataset> ds6 = ds4->Take(13);
|
||||
std::shared_ptr<Dataset> ds10 = ds9 + ds6;
|
||||
|
|
@ -419,11 +419,11 @@ TEST_F(MindDataTestTreeModifying, Drop03) {
|
|||
*
|
||||
*/
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data/";
|
||||
std::shared_ptr<Dataset> ds7 = ImageFolder(folder_path, false, SequentialSampler(0, 11));
|
||||
std::shared_ptr<Dataset> ds7 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11));
|
||||
std::shared_ptr<Dataset> ds8 = ds7->Take(20);
|
||||
std::shared_ptr<Dataset> ds9 = ds8->Skip(1);
|
||||
std::shared_ptr<Dataset> ds3 = ImageFolder(folder_path, false, SequentialSampler(0, 11));
|
||||
std::shared_ptr<Dataset> ds2 = ImageFolder(folder_path, false, SequentialSampler(0, 11));
|
||||
std::shared_ptr<Dataset> ds3 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11));
|
||||
std::shared_ptr<Dataset> ds2 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11));
|
||||
std::shared_ptr<Dataset> ds4 = ds3->Concat({ds2});
|
||||
std::shared_ptr<Dataset> ds6 = ds4->Take(13);
|
||||
std::shared_ptr<Dataset> ds10 = ds9 + ds6;
|
||||
|
|
@ -469,14 +469,14 @@ TEST_F(MindDataTestTreeModifying, Drop04) {
|
|||
*
|
||||
*/
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data/";
|
||||
std::shared_ptr<Dataset> ds7 = ImageFolder(folder_path, false, SequentialSampler(0, 11));
|
||||
std::shared_ptr<Dataset> ds7 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11));
|
||||
std::shared_ptr<Dataset> ds8 = ds7->Take(20);
|
||||
std::shared_ptr<Dataset> ds9 = ds8->Skip(1);
|
||||
std::shared_ptr<Dataset> ds3 = ImageFolder(folder_path, false, SequentialSampler(0, 11));
|
||||
std::shared_ptr<Dataset> ds2 = ImageFolder(folder_path, false, SequentialSampler(0, 11));
|
||||
std::shared_ptr<Dataset> ds3 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11));
|
||||
std::shared_ptr<Dataset> ds2 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11));
|
||||
std::shared_ptr<Dataset> ds4 = ds3->Concat({ds2});
|
||||
std::shared_ptr<Dataset> ds5 = ImageFolder(folder_path, false, SequentialSampler(0, 11));
|
||||
std::shared_ptr<Dataset> ds1 = ImageFolder(folder_path, false, SequentialSampler(0, 11));
|
||||
std::shared_ptr<Dataset> ds5 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11));
|
||||
std::shared_ptr<Dataset> ds1 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11));
|
||||
std::shared_ptr<Dataset> ds6 = ds5->Concat({ds4, ds1});
|
||||
std::shared_ptr<Dataset> ds10 = ds9 + ds6;
|
||||
Status rc;
|
||||
|
|
@ -526,13 +526,13 @@ TEST_F(MindDataTestTreeModifying, Drop05) {
|
|||
*
|
||||
*/
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data/";
|
||||
std::shared_ptr<Dataset> ds7 = ImageFolder(folder_path, false, SequentialSampler(0, 11));
|
||||
std::shared_ptr<Dataset> ds7 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11));
|
||||
std::shared_ptr<Dataset> ds8 = ds7->Take(20);
|
||||
std::shared_ptr<Dataset> ds9 = ds8->Skip(1);
|
||||
std::shared_ptr<Dataset> ds3 = ImageFolder(folder_path, false, SequentialSampler(0, 11));
|
||||
std::shared_ptr<Dataset> ds3 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11));
|
||||
std::shared_ptr<Dataset> ds4 = ds3->Skip(1);
|
||||
std::shared_ptr<Dataset> ds5 = ImageFolder(folder_path, false, SequentialSampler(0, 11));
|
||||
std::shared_ptr<Dataset> ds1 = ImageFolder(folder_path, false, SequentialSampler(0, 11));
|
||||
std::shared_ptr<Dataset> ds5 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11));
|
||||
std::shared_ptr<Dataset> ds1 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11));
|
||||
std::shared_ptr<Dataset> ds6 = ds5->Concat({ds4, ds1});
|
||||
std::shared_ptr<Dataset> ds10 = ds9 + ds6;
|
||||
Status rc;
|
||||
|
|
@ -581,14 +581,14 @@ TEST_F(MindDataTestTreeModifying, Drop06) {
|
|||
*
|
||||
*/
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data/";
|
||||
std::shared_ptr<Dataset> ds7 = ImageFolder(folder_path, false, SequentialSampler(0, 11));
|
||||
std::shared_ptr<Dataset> ds7 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11));
|
||||
std::shared_ptr<Dataset> ds8 = ds7->Take(20);
|
||||
std::shared_ptr<Dataset> ds9 = ds8->Skip(1);
|
||||
std::shared_ptr<Dataset> ds3 = ImageFolder(folder_path, false, SequentialSampler(0, 11));
|
||||
std::shared_ptr<Dataset> ds2 = ImageFolder(folder_path, false, SequentialSampler(0, 11));
|
||||
std::shared_ptr<Dataset> ds3 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11));
|
||||
std::shared_ptr<Dataset> ds2 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11));
|
||||
std::shared_ptr<Dataset> ds4 = ds3->Concat({ds2});
|
||||
std::shared_ptr<Dataset> ds5 = ImageFolder(folder_path, false, SequentialSampler(0, 11));
|
||||
std::shared_ptr<Dataset> ds1 = ImageFolder(folder_path, false, SequentialSampler(0, 11));
|
||||
std::shared_ptr<Dataset> ds5 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11));
|
||||
std::shared_ptr<Dataset> ds1 = ImageFolder(folder_path, false, std::make_shared<SequentialSampler>(0, 11));
|
||||
std::shared_ptr<Dataset> ds6 = ds5->Concat({ds4, ds1}); // ds1 is put after (ds5, ds4)!!!
|
||||
std::shared_ptr<Dataset> ds10 = ds9 + ds6;
|
||||
Status rc;
|
||||
|
|
|
|||
Loading…
Reference in New Issue