Col_lin
/
mindspore2022
forked from huawei/mindspore2022
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

!22552 [assistant][ops]New operator implementation, include STL10Dataset 

Merge pull request !22552 from 王睿/stl10_dataset
This commit is contained in:
i-robot 2021-12-07 06:37:04 +00:00 committed by Gitee
parent 438018d96b c4063f0f11
commit 3d0f8fcc99
21 changed files with 1858 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
mindspore/ccsrc/minddata/dataset/api/datasets.cc
View File

@ -113,6 +113,7 @@
#include "minddata/dataset/engine/ir/datasetops/source/random_node.h"
#include "minddata/dataset/engine/ir/datasetops/source/sbu_node.h"
#include "minddata/dataset/engine/ir/datasetops/source/speech_commands_node.h"
#include "minddata/dataset/engine/ir/datasetops/source/stl10_node.h"
#include "minddata/dataset/engine/ir/datasetops/source/tedlium_node.h"
#include "minddata/dataset/engine/ir/datasetops/source/text_file_node.h"
#include "minddata/dataset/engine/ir/datasetops/source/tf_record_node.h"
@ -1500,6 +1501,27 @@ TedliumDataset::TedliumDataset(const std::vector<char> &dataset_dir, const std::
ir_node_ = std::static_pointer_cast<DatasetNode>(ds);
}
STL10Dataset::STL10Dataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage,
const std::shared_ptr<Sampler> &sampler, const std::shared_ptr<DatasetCache> &cache) {
auto sampler_obj = sampler ? sampler->Parse() : nullptr;
auto ds = std::make_shared<STL10Node>(CharToString(dataset_dir), CharToString(usage), sampler_obj, cache);
ir_node_ = std::static_pointer_cast<DatasetNode>(ds);
}
STL10Dataset::STL10Dataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage, const Sampler *sampler,
const std::shared_ptr<DatasetCache> &cache) {
auto sampler_obj = sampler ? sampler->Parse() : nullptr;
auto ds = std::make_shared<STL10Node>(CharToString(dataset_dir), CharToString(usage), sampler_obj, cache);
ir_node_ = std::static_pointer_cast<DatasetNode>(ds);
}
STL10Dataset::STL10Dataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage,
const std::reference_wrapper<Sampler> sampler, const std::shared_ptr<DatasetCache> &cache) {
auto sampler_obj = sampler.get().Parse();
auto ds = std::make_shared<STL10Node>(CharToString(dataset_dir), CharToString(usage), sampler_obj, cache);
ir_node_ = std::static_pointer_cast<DatasetNode>(ds);
}
TextFileDataset::TextFileDataset(const std::vector<std::vector<char>> &dataset_files, int64_t num_samples,
ShuffleMode shuffle, int32_t num_shards, int32_t shard_id,
const std::shared_ptr<DatasetCache> &cache) {

11
mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/engine/ir/datasetops/source/bindings.cc
View File

@ -45,6 +45,7 @@
#include "minddata/dataset/engine/ir/datasetops/source/mnist_node.h"
#include "minddata/dataset/engine/ir/datasetops/source/random_node.h"
#include "minddata/dataset/engine/ir/datasetops/source/speech_commands_node.h"
#include "minddata/dataset/engine/ir/datasetops/source/stl10_node.h"
#include "minddata/dataset/engine/ir/datasetops/source/tedlium_node.h"
#include "minddata/dataset/engine/ir/datasetops/source/text_file_node.h"
#include "minddata/dataset/engine/ir/datasetops/source/yes_no_node.h"
@ -412,6 +413,16 @@ PYBIND_REGISTER(SpeechCommandsNode, 2, ([](const py::module *m) {
}));
}));
PYBIND_REGISTER(STL10Node, 2, ([](const py::module *m) {
(void)py::class_<STL10Node, DatasetNode, std::shared_ptr<STL10Node>>(*m, "STL10Node",
"to create a STL10Node")
.def(py::init([](std::string dataset_dir, std::string usage, py::handle sampler) {
auto stl10 = std::make_shared<STL10Node>(dataset_dir, usage, toSamplerObj(sampler), nullptr);
THROW_IF_ERROR(stl10->ValidateParams());
return stl10;
}));
}));
PYBIND_REGISTER(TedliumNode, 2, ([](const py::module *m) {
(void)py::class_<TedliumNode, DatasetNode, std::shared_ptr<TedliumNode>>(*m, "TedliumNode",
"to create a TedliumNode")

1
mindspore/ccsrc/minddata/dataset/engine/datasetops/source/CMakeLists.txt
View File

@ -30,6 +30,7 @@ set(DATASET_ENGINE_DATASETOPS_SOURCE_SRC_FILES
random_data_op.cc
sbu_op.cc
speech_commands_op.cc
stl10_op.cc
tedlium_op.cc
text_file_op.cc
usps_op.cc

395
mindspore/ccsrc/minddata/dataset/engine/datasetops/source/stl10_op.cc Normal file
View File

@ -0,0 +1,395 @@
/**
* Copyright 2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "minddata/dataset/engine/datasetops/source/stl10_op.h"
#include <algorithm>
#include <fstream>
#include <iomanip>
#include <set>
#include <utility>
#include "debug/common.h"
#include "minddata/dataset/core/config_manager.h"
#include "minddata/dataset/core/tensor_shape.h"
#include "minddata/dataset/engine/datasetops/source/sampler/sequential_sampler.h"
#include "minddata/dataset/engine/execution_tree.h"
#include "utils/file_utils.h"
#include "utils/ms_utils.h"
namespace mindspore {
namespace dataset {
constexpr uint32_t kSTLImageRows = 96;
constexpr uint32_t kSTLImageCols = 96;
constexpr uint32_t kSTLImageChannel = 3;
constexpr uint32_t kSTLImageSize = kSTLImageRows * kSTLImageCols * kSTLImageChannel;
STL10Op::STL10Op(const std::string &usage, int32_t num_workers, const std::string &folder_path, int32_t queue_size,
std::unique_ptr<DataSchema> data_schema, std::shared_ptr<SamplerRT> sampler)
: MappableLeafOp(num_workers, queue_size, std::move(sampler)),
folder_path_(folder_path),
usage_(usage),
data_schema_(std::move(data_schema)),
image_path_({}),
label_path_({}) {}
Status STL10Op::LoadTensorRow(row_id_type index, TensorRow *trow) {
RETURN_UNEXPECTED_IF_NULL(trow);
std::pair<std::shared_ptr<Tensor>, int32_t> stl10_pair = stl10_image_label_pairs_[index];
std::shared_ptr<Tensor> image, label;
// make a copy of cached tensor.
RETURN_IF_NOT_OK(Tensor::CreateFromTensor(stl10_pair.first, &image));
RETURN_IF_NOT_OK(Tensor::CreateScalar(stl10_pair.second, &label));
(*trow) = TensorRow(index, {std::move(image), std::move(label)});
trow->setPath({image_path_[index], label_path_[index]});
return Status::OK();
}
Status STL10Op::GetClassIds(std::map<int32_t, std::vector<int64_t>> *cls_ids) const {
if (cls_ids == nullptr || !cls_ids->empty() || stl10_image_label_pairs_.empty()) {
if (stl10_image_label_pairs_.empty()) {
RETURN_STATUS_UNEXPECTED("No image found in dataset. Check if image was generated successfully.");
} else {
RETURN_STATUS_UNEXPECTED(
"[Internal ERROR] Map for containing image-index pair is nullptr or has been set in other place, "
"it must be empty before using GetClassIds.");
}
}
for (size_t i = 0; i < stl10_image_label_pairs_.size(); ++i) {
(*cls_ids)[stl10_image_label_pairs_[i].second].push_back(i);
}
for (auto &pair : (*cls_ids)) {
pair.second.shrink_to_fit();
}
return Status::OK();
}
void STL10Op::Print(std::ostream &out, bool show_all) const {
if (!show_all) {
// Call the super class for displaying any common 1-liner info.
ParallelOp::Print(out, show_all);
// Then show any custom derived-internal 1-liner info for this op.
out << "\n";
} else {
// Call the super class for displaying any common detailed info.
ParallelOp::Print(out, show_all);
// Then show any custom derived-internal stuff.
out << "\nNumber of rows: " << num_rows_ << "\nSTL10 directory: " << folder_path_ << "\n\n";
}
}
Status STL10Op::WalkAllFiles() {
auto real_dataset_dir = FileUtils::GetRealPath(folder_path_.data());
CHECK_FAIL_RETURN_UNEXPECTED(real_dataset_dir.has_value(),
"Invalid file, get real path failed, path: " + folder_path_);
Path root_dir(real_dataset_dir.value());
const Path train_data_file("train_X.bin");
const Path train_label_file("train_y.bin");
const Path test_data_file("test_X.bin");
const Path test_label_file("test_y.bin");
const Path unlabeled_data_file("unlabeled_X.bin");
bool use_train = false;
bool use_test = false;
bool use_unlabeled = false;
if (usage_ == "train") {
use_train = true;
} else if (usage_ == "test") {
use_test = true;
} else if (usage_ == "unlabeled") {
use_unlabeled = true;
} else if (usage_ == "train+unlabeled") {
use_train = true;
use_unlabeled = true;
} else if (usage_ == "all") {
use_train = true;
use_test = true;
use_unlabeled = true;
}
if (use_train) {
Path train_data_path = root_dir / train_data_file;
Path train_label_path = root_dir / train_label_file;
CHECK_FAIL_RETURN_UNEXPECTED(
train_data_path.Exists() && !train_data_path.IsDirectory(),
"Invalid file, failed to find STL10 " + usage_ + " data file: " + train_data_path.ToString());
CHECK_FAIL_RETURN_UNEXPECTED(
train_label_path.Exists() && !train_label_path.IsDirectory(),
"Invalid file, failed to find STL10 " + usage_ + " label file: " + train_label_path.ToString());
image_names_.push_back(train_data_path.ToString());
label_names_.push_back(train_label_path.ToString());
MS_LOG(INFO) << "STL10 operator found train data file " << train_data_path.ToString() << ".";
MS_LOG(INFO) << "STL10 operator found train label file " << train_label_path.ToString() << ".";
}
if (use_test) {
Path test_data_path = root_dir / test_data_file;
Path test_label_path = root_dir / test_label_file;
CHECK_FAIL_RETURN_UNEXPECTED(
test_data_path.Exists() && !test_data_path.IsDirectory(),
"Invalid file, failed to find STL10 " + usage_ + " data file: " + test_data_path.ToString());
CHECK_FAIL_RETURN_UNEXPECTED(
test_label_path.Exists() && !test_label_path.IsDirectory(),
"Invalid file, failed to find STL10 " + usage_ + " label file: " + test_label_path.ToString());
image_names_.push_back(test_data_path.ToString());
label_names_.push_back(test_label_path.ToString());
MS_LOG(INFO) << "STL10 operator found test data file " << test_data_path.ToString() << ".";
MS_LOG(INFO) << "STL10 operator found test label file " << test_label_path.ToString() << ".";
}
if (use_unlabeled) {
Path unlabeled_data_path = root_dir / unlabeled_data_file;
CHECK_FAIL_RETURN_UNEXPECTED(
unlabeled_data_path.Exists() && !unlabeled_data_path.IsDirectory(),
"Invalid file, failed to find STL10 " + usage_ + " data file: " + unlabeled_data_path.ToString());
image_names_.push_back(unlabeled_data_path.ToString());
MS_LOG(INFO) << "STL10 operator found unlabeled data file " << unlabeled_data_path.ToString() << ".";
}
std::sort(image_names_.begin(), image_names_.end());
std::sort(label_names_.begin(), label_names_.end());
return Status::OK();
}
Status STL10Op::ParseSTLData() {
// STL10 contains 5 files, *_X.bin are image files, *_y.bin are labels.
// training files contain 5k images and testing files contain 8K examples.
// unlabeled file contain 10k images and they DO NOT have labels (i.e. no "unlabeled_y.bin" file).
for (size_t i = 0; i < image_names_.size(); ++i) {
std::ifstream image_reader, label_reader;
if (image_names_[i].find("unlabeled") == std::string::npos) {
image_reader.open(image_names_[i], std::ios::binary | std::ios::ate);
label_reader.open(label_names_[i], std::ios::binary | std::ios::ate);
Status s = ReadImageAndLabel(&image_reader, &label_reader, i);
// Close the readers.
image_reader.close();
label_reader.close();
RETURN_IF_NOT_OK(s);
} else { // unlabeled data -> no labels.
image_reader.open(image_names_[i], std::ios::binary | std::ios::ate);
Status s = ReadImageAndLabel(&image_reader, NULL, i);
// Close the readers.
image_reader.close();
RETURN_IF_NOT_OK(s);
}
}
stl10_image_label_pairs_.shrink_to_fit();
num_rows_ = stl10_image_label_pairs_.size();
if (num_rows_ == 0) {
RETURN_STATUS_UNEXPECTED(
"Invalid data, no valid data matching the dataset API STL10Dataset. Please check file path or dataset API.");
}
return Status::OK();
}
Status STL10Op::ReadImageAndLabel(std::ifstream *image_reader, std::ifstream *label_reader, size_t index) {
RETURN_UNEXPECTED_IF_NULL(image_reader);
Path image_path(image_names_[index]);
bool has_label_file = image_path.Basename().find("unlabeled") == std::string::npos;
std::streamsize image_size = image_reader->tellg();
image_reader->seekg(0, std::ios::beg);
auto images_buf = std::make_unique<char[]>(image_size);
auto labels_buf = std::make_unique<char[]>(0);
if (images_buf == nullptr) {
std::string err_msg = "Failed to allocate memory for STL10 buffer.";
MS_LOG(ERROR) << err_msg.c_str();
RETURN_STATUS_UNEXPECTED(err_msg);
}
uint64_t num_images = static_cast<uint64_t>(image_size / kSTLImageSize);
(void)image_reader->read(images_buf.get(), image_size);
if (image_reader->fail()) {
RETURN_STATUS_UNEXPECTED("Invalid file, failed to read image: " + image_names_[index] +
", size:" + std::to_string(kSTLImageSize * num_images));
}
if (has_label_file) {
RETURN_UNEXPECTED_IF_NULL(label_reader);
std::streamsize label_size = label_reader->tellg();
if (static_cast<uint64_t>(label_size) != num_images) {
RETURN_STATUS_UNEXPECTED("Invalid file, error in " + label_names_[index] +
": the number of labels is not equal to the number of images in " + image_names_[index] +
"! Please check the file integrity!");
}
label_reader->seekg(0, std::ios::beg);
labels_buf = std::make_unique<char[]>(label_size);
if (labels_buf == nullptr) {
std::string err_msg = "Failed to allocate memory for STL10 buffer.";
MS_LOG(ERROR) << err_msg.c_str();
RETURN_STATUS_UNEXPECTED(err_msg);
}
(void)label_reader->read(labels_buf.get(), label_size);
if (label_reader->fail()) {
RETURN_STATUS_UNEXPECTED("Invalid file, failed to read label:" + label_names_[index] +
", size: " + std::to_string(num_images));
}
}
for (int64_t j = 0; j < num_images; ++j) {
int32_t label = (has_label_file ? labels_buf[j] - 1 : -1);
std::shared_ptr<Tensor> image_tensor;
RETURN_IF_NOT_OK(Tensor::CreateEmpty(TensorShape({kSTLImageRows, kSTLImageCols, kSTLImageChannel}),
data_schema_->Column(0).Type(), &image_tensor));
auto iter = image_tensor->begin<uint8_t>();
uint64_t total_pix = kSTLImageRows * kSTLImageCols;
// stl10: Column major order.
for (uint64_t count = 0, pix = 0; count < total_pix; count++) {
if (count % kSTLImageRows == 0) {
pix = count / kSTLImageRows;
}
for (int ch = 0; ch < kSTLImageChannel; ch++) {
*iter = images_buf[j * kSTLImageSize + ch * total_pix + pix];
iter++;
}
pix += kSTLImageRows;
}
stl10_image_label_pairs_.emplace_back(std::make_pair(image_tensor, label));
image_path_.push_back(image_names_[index]);
label_path_.push_back(has_label_file ? label_names_[index] : "no label");
}
return Status::OK();
}
Status STL10Op::PrepareData() {
RETURN_IF_NOT_OK(this->WalkAllFiles());
RETURN_IF_NOT_OK(this->ParseSTLData()); // Parse stl10 data and get num rows, blocking.
return Status::OK();
}
Status STL10Op::CountTotalRows(const std::string &dir, const std::string &usage, int64_t *count) {
RETURN_UNEXPECTED_IF_NULL(count);
// the logic of counting the number of samples is copied from ParseSTLData().
const int64_t num_samples = 0;
const int64_t start_index = 0;
auto sampler = std::make_shared<SequentialSamplerRT>(start_index, num_samples);
auto schema = std::make_unique<DataSchema>();
RETURN_IF_NOT_OK(schema->AddColumn(ColDescriptor("image", DataType(DataType::DE_UINT8), TensorImpl::kCv, 1)));
TensorShape scalar = TensorShape::CreateScalar();
RETURN_IF_NOT_OK(
schema->AddColumn(ColDescriptor("label", DataType(DataType::DE_INT32), TensorImpl::kFlexible, 0, &scalar)));
std::shared_ptr<ConfigManager> cfg = GlobalContext::config_manager();
int32_t num_workers = cfg->num_parallel_workers();
int32_t op_connect_size = cfg->op_connector_size();
auto op = std::make_shared<STL10Op>(usage, num_workers, dir, op_connect_size, std::move(schema), std::move(sampler));
RETURN_IF_NOT_OK(op->WalkAllFiles());
bool use_train = false;
bool use_test = false;
bool use_unlabeled = false;
if (usage == "train") {
use_train = true;
} else if (usage == "test") {
use_test = true;
} else if (usage == "unlabeled") {
use_unlabeled = true;
} else if (usage == "train+unlabeled") {
use_train = true;
use_unlabeled = true;
} else if (usage == "all") {
use_train = true;
use_test = true;
use_unlabeled = true;
}
*count = 0;
uint64_t num_stl10_records = 0;
uint64_t total_image_size = 0;
if (use_train) {
uint32_t index = (usage == "all" ? 1 : 0);
Path train_image_path(op->image_names_[index]);
CHECK_FAIL_RETURN_UNEXPECTED(train_image_path.Exists() && !train_image_path.IsDirectory(),
"Invalid file, failed to open stl10 file: " + train_image_path.ToString());
std::ifstream train_image_file(train_image_path.ToString(), std::ios::binary | std::ios::ate);
CHECK_FAIL_RETURN_UNEXPECTED(train_image_file.is_open(),
"Invalid file, failed to open stl10 file: " + train_image_path.ToString());
total_image_size += static_cast<uint64_t>(train_image_file.tellg());
train_image_file.close();
}
if (use_test) {
uint32_t index = 0;
Path test_image_path(op->image_names_[index]);
CHECK_FAIL_RETURN_UNEXPECTED(test_image_path.Exists() && !test_image_path.IsDirectory(),
"Invalid file, failed to open stl10 file: " + test_image_path.ToString());
std::ifstream test_image_file(test_image_path.ToString(), std::ios::binary | std::ios::ate);
CHECK_FAIL_RETURN_UNEXPECTED(test_image_file.is_open(),
"Invalid file, failed to open stl10 file: " + test_image_path.ToString());
total_image_size += static_cast<uint64_t>(test_image_file.tellg());
test_image_file.close();
}
if (use_unlabeled) {
uint32_t index = (usage == "unlabeled" ? 0 : (usage == "train+unlabeled" ? 1 : 2));
Path unlabeled_image_path(op->image_names_[index]);
CHECK_FAIL_RETURN_UNEXPECTED(unlabeled_image_path.Exists() && !unlabeled_image_path.IsDirectory(),
"Invalid file, failed to open stl10 file: " + unlabeled_image_path.ToString());
std::ifstream unlabeled_image_file(unlabeled_image_path.ToString(), std::ios::binary | std::ios::ate);
CHECK_FAIL_RETURN_UNEXPECTED(unlabeled_image_file.is_open(),
"Invalid file, failed to open stl10 file: " + unlabeled_image_path.ToString());
total_image_size += static_cast<uint64_t>(unlabeled_image_file.tellg());
unlabeled_image_file.close();
}
num_stl10_records = static_cast<uint64_t>(total_image_size / kSTLImageSize);
*count = *count + num_stl10_records;
return Status::OK();
}
Status STL10Op::ComputeColMap() {
// set the column Name map (base class field).
if (column_name_id_map_.empty()) {
for (uint32_t i = 0; i < data_schema_->NumColumns(); ++i) {
column_name_id_map_[data_schema_->Column(i).Name()] = i;
}
} else {
MS_LOG(WARNING) << "Column Name map is already set!";
}
return Status::OK();
}
} // namespace dataset
} // namespace mindspore

118
mindspore/ccsrc/minddata/dataset/engine/datasetops/source/stl10_op.h Normal file
View File

@ -0,0 +1,118 @@
/**
* Copyright 2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef MINDSPORE_CCSRC_MINDDATA_DATASET_ENGINE_DATASETOPS_SOURCE_STL10_OP_H_
#define MINDSPORE_CCSRC_MINDDATA_DATASET_ENGINE_DATASETOPS_SOURCE_STL10_OP_H_
#include <algorithm>
#include <map>
#include <memory>
#include <string>
#include <utility>
#include <vector>
#include "minddata/dataset/core/tensor.h"
#include "minddata/dataset/engine/data_schema.h"
#include "minddata/dataset/engine/datasetops/parallel_op.h"
#include "minddata/dataset/engine/datasetops/source/mappable_leaf_op.h"
#include "minddata/dataset/engine/datasetops/source/sampler/sampler.h"
#include "minddata/dataset/util/path.h"
#include "minddata/dataset/util/queue.h"
#include "minddata/dataset/util/status.h"
#include "minddata/dataset/util/wait_post.h"
namespace mindspore {
namespace dataset {
class STL10Op : public MappableLeafOp {
public:
// Constructor
// @param const std::string &usage - Usage of this dataset, can be 'train', 'test', 'unlabeled', "train+unlabeled" or
// "all"
// @param int32_t num_workers - number of workers reading images in parallel.
// @param std::string folder_path - dir directory of stl10.
// @param int32_t queue_size - connector queue size.
// @param std::unique_ptr<DataSchema> data_schema - the schema of the stl10 dataset.
// @param td::unique_ptr<Sampler> sampler - sampler tells STL10Op what to read.
STL10Op(const std::string &usage, int32_t num_workers, const std::string &folder_path, int32_t queue_size,
std::unique_ptr<DataSchema> data_schema, std::shared_ptr<SamplerRT> sampler);
// Destructor
~STL10Op() = default;
// Method derived from RandomAccess Op, enable Sampler to get all ids for each class
// @param (std::map<int32_t, std::vector<int64_t>> * cls_ids - key label, val all ids for this class
// @return Status The status code returned
Status GetClassIds(std::map<int32_t, std::vector<int64_t>> *cls_ids) const override;
// A print method typically used for debugging.
// @param out - The output stream to write output to.
// @param show_all - A bool to control if you want to show all info or just a summary.
void Print(std::ostream &out, bool show_all) const override;
// Function to count the number of samples in the STL10 dataset.
// @param dir path to the STL10 directory.
// @param const std::string &usage - Usage of this dataset, can be 'train', 'test', 'unlabeled', "train+unlabeled" or
// "all"
// @param count output arg that will hold the minimum of the actual dataset size and numSamples.
// @return Status The status code returned.
static Status CountTotalRows(const std::string &dir, const std::string &usage, int64_t *count);
// Op name getter.
// @return Name of the current Op.
std::string Name() const override { return "STL10Op"; }
private:
// Read the needed files in the directory, save the file in image_names_ and label_names_.
// @return Status The status code returned.
Status WalkAllFiles();
// Read the specified number of images and labels from the file stream.
// @param std::ifstream *image_reader - image file stream.
// @param std::ifstream *label_reader - label file stream.
// @param int64_t index - number of image to read.
// @return Status The status code returned.
Status ReadImageAndLabel(std::ifstream *image_reader, std::ifstream *label_reader, size_t index);
// Parse all stl10 dataset files
// @return Status The status code returned
Status ParseSTLData();
// Read all stl10 data files in the directory
// @return Status The status code returned.
Status PrepareData() override;
// Private function for computing the assignment of the column name map.
// @return - Status.
Status ComputeColMap() override;
// Load a tensor row according to a pair.
// @param uint64_t index - index need to load.
// @param TensorRow trow - image & label read into this tensor row.
// @return Status The status code returned.
Status LoadTensorRow(row_id_type index, TensorRow *trow) override;
std::string folder_path_; // directory of image folder.
const std::string usage_; // can only be either "train" or "test" or "unlabeled" or "train+unlabeled" or "all".
std::unique_ptr<DataSchema> data_schema_;
std::vector<std::pair<std::shared_ptr<Tensor>, int32_t>> stl10_image_label_pairs_;
std::vector<std::string> image_names_;
std::vector<std::string> label_names_;
std::vector<std::string> image_path_;
std::vector<std::string> label_path_;
};
} // namespace dataset
} // namespace mindspore
#endif // MINDSPORE_CCSRC_MINDDATA_DATASET_ENGINE_DATASETOPS_SOURCE_STL10_OP_H_

1
mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/dataset_node.h
View File

@ -104,6 +104,7 @@ constexpr char kQMnistNode[] = "QMnistDataset";
constexpr char kRandomNode[] = "RandomDataset";
constexpr char kSBUNode[] = "SBUDataset";
constexpr char kSpeechCommandsNode[] = "SpeechCommandsDataset";
constexpr char kSTL10Node[] = "STL10Dataset";
constexpr char kTedliumNode[] = "TedliumDataset";
constexpr char kTextFileNode[] = "TextFileDataset";
constexpr char kTFRecordNode[] = "TFRecordDataset";

1
mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/CMakeLists.txt
View File

@ -30,6 +30,7 @@ set(DATASET_ENGINE_IR_DATASETOPS_SOURCE_SRC_FILES
random_node.cc
sbu_node.cc
speech_commands_node.cc
stl10_node.cc
tedlium_node.cc
text_file_node.cc
tf_record_node.cc

121
mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/stl10_node.cc Normal file
View File

@ -0,0 +1,121 @@
/**
* Copyright 2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "minddata/dataset/engine/ir/datasetops/source/stl10_node.h"
#include <memory>
#include <string>
#include <utility>
#include <vector>
#include "minddata/dataset/engine/datasetops/source/stl10_op.h"
#include "minddata/dataset/util/status.h"
namespace mindspore {
namespace dataset {
STL10Node::STL10Node(const std::string &dataset_dir, const std::string &usage, std::shared_ptr<SamplerObj> sampler,
std::shared_ptr<DatasetCache> cache)
: MappableSourceNode(std::move(cache)), dataset_dir_(dataset_dir), usage_(usage), sampler_(sampler) {}
std::shared_ptr<DatasetNode> STL10Node::Copy() {
std::shared_ptr<SamplerObj> sampler = (sampler_ == nullptr) ? nullptr : sampler_->SamplerCopy();
auto node = std::make_shared<STL10Node>(dataset_dir_, usage_, sampler, cache_);
return node;
}
void STL10Node::Print(std::ostream &out) const {
out << (Name() + "(cache:" + ((cache_ != nullptr) ? "true" : "false") + ")");
}
Status STL10Node::ValidateParams() {
RETURN_IF_NOT_OK(DatasetNode::ValidateParams());
RETURN_IF_NOT_OK(ValidateDatasetDirParam("STL10Node", dataset_dir_));
RETURN_IF_NOT_OK(ValidateDatasetSampler("STL10Node", sampler_));
RETURN_IF_NOT_OK(ValidateStringValue("STL10Node", usage_, {"train", "test", "unlabeled", "train+unlabeled", "all"}));
return Status::OK();
}
// Function to build STL10Op for STL10
Status STL10Node::Build(std::vector<std::shared_ptr<DatasetOp>> *const node_ops) {
// Do internal Schema generation.
auto schema = std::make_unique<DataSchema>();
RETURN_IF_NOT_OK(schema->AddColumn(ColDescriptor("image", DataType(DataType::DE_UINT8), TensorImpl::kCv, 1)));
TensorShape scalar = TensorShape::CreateScalar();
RETURN_IF_NOT_OK(
schema->AddColumn(ColDescriptor("label", DataType(DataType::DE_INT32), TensorImpl::kFlexible, 0, &scalar)));
std::shared_ptr<SamplerRT> sampler_rt = nullptr;
RETURN_IF_NOT_OK(sampler_->SamplerBuild(&sampler_rt));
auto stl10_op = std::make_shared<STL10Op>(usage_, num_workers_, dataset_dir_, connector_que_size_, std::move(schema),
std::move(sampler_rt));
stl10_op->SetTotalRepeats(GetTotalRepeats());
stl10_op->SetNumRepeatsPerEpoch(GetNumRepeatsPerEpoch());
node_ops->push_back(stl10_op);
return Status::OK();
}
// Get the shard id of node
Status STL10Node::GetShardId(int32_t *shard_id) {
*shard_id = sampler_->ShardId();
return Status::OK();
}
// Get Dataset size
Status STL10Node::GetDatasetSize(const std::shared_ptr<DatasetSizeGetter> &size_getter, bool estimate,
int64_t *dataset_size) {
if (dataset_size_ > 0) {
*dataset_size = dataset_size_;
return Status::OK();
}
int64_t num_rows, sample_size;
RETURN_IF_NOT_OK(STL10Op::CountTotalRows(dataset_dir_, usage_, &num_rows));
std::shared_ptr<SamplerRT> sampler_rt = nullptr;
RETURN_IF_NOT_OK(sampler_->SamplerBuild(&sampler_rt));
sample_size = sampler_rt->CalculateNumSamples(num_rows);
if (sample_size == -1) {
RETURN_IF_NOT_OK(size_getter->DryRun(shared_from_this(), &sample_size));
}
*dataset_size = sample_size;
dataset_size_ = *dataset_size;
return Status::OK();
}
Status STL10Node::to_json(nlohmann::json *out_json) {
nlohmann::json args, sampler_args;
RETURN_IF_NOT_OK(sampler_->to_json(&sampler_args));
args["sampler"] = sampler_args;
args["num_parallel_workers"] = num_workers_;
args["dataset_dir"] = dataset_dir_;
args["usage"] = usage_;
if (cache_ != nullptr) {
nlohmann::json cache_args;
RETURN_IF_NOT_OK(cache_->to_json(&cache_args));
args["cache"] = cache_args;
}
*out_json = args;
return Status::OK();
}
} // namespace dataset
} // namespace mindspore

96
mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/stl10_node.h Normal file
View File

@ -0,0 +1,96 @@
/**
* Copyright 2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef MINDSPORE_CCSRC_MINDDATA_DATASET_ENGINE_IR_DATASETOPS_SOURCE_STL10_NODE_H_
#define MINDSPORE_CCSRC_MINDDATA_DATASET_ENGINE_IR_DATASETOPS_SOURCE_STL10_NODE_H_
#include <memory>
#include <string>
#include <vector>
#include "minddata/dataset/engine/ir/datasetops/dataset_node.h"
namespace mindspore {
namespace dataset {
class STL10Node : public MappableSourceNode {
public:
/// \brief Constructor
STL10Node(const std::string &dataset_dir, const std::string &usage, std::shared_ptr<SamplerObj> sampler,
std::shared_ptr<DatasetCache> cache);
/// \brief Destructor
~STL10Node() = default;
/// \brief Node name getter
/// \return Name of the current node
std::string Name() const override { return kSTL10Node; }
/// \brief Print the description
/// \param out - The output stream to write output to
void Print(std::ostream &out) const override;
/// \brief Copy the node to a new object
/// \return A shared pointer to the new copy
std::shared_ptr<DatasetNode> Copy() override;
/// \brief a base class override function to create the required runtime dataset op objects for this class
/// \param node_ops - A vector containing shared pointer to the Dataset Ops that this object will create
/// \return Status Status::OK() if build successfully
Status Build(std::vector<std::shared_ptr<DatasetOp>> *const node_ops) override;
/// \brief Parameters validation
/// \return Status Status::OK() if all the parameters are valid
Status ValidateParams() override;
/// \brief Get the shard id of node
/// \param[in] shard_id - The shard id.
/// \return Status Status::OK() if get shard id successfully
Status GetShardId(int32_t *shard_id) override;
/// \brief Base-class override for GetDatasetSize
/// \param[in] size_getter Shared pointer to DatasetSizeGetter
/// \param[in] estimate This is only supported by some of the ops and it's used to speed up the process of getting
/// dataset size at the expense of accuracy.
/// \param[out] dataset_size the size of the dataset
/// \return Status of the function
Status GetDatasetSize(const std::shared_ptr<DatasetSizeGetter> &size_getter, bool estimate,
int64_t *dataset_size) override;
/// \brief Getter functions
const std::string &DatasetDir() const { return dataset_dir_; }
const std::string &Usage() const { return usage_; }
/// \brief Get the arguments of node
/// \param[out] out_json JSON string of all attributes
/// \return Status of the function
Status to_json(nlohmann::json *out_json) override;
/// \brief Sampler getter
/// \return SamplerObj of the current node
std::shared_ptr<SamplerObj> Sampler() override { return sampler_; }
/// \brief Sampler setter
/// \param[in] sampler The Sampler setter of the current node
void SetSampler(std::shared_ptr<SamplerObj> sampler) override { sampler_ = sampler; }
private:
std::string dataset_dir_;
std::string usage_;
std::shared_ptr<SamplerObj> sampler_;
};
} // namespace dataset
} // namespace mindspore
#endif // MINDSPORE_CCSRC_MINDDATA_DATASET_ENGINE_IR_DATASETOPS_SOURCE_STL10_NODE_H_

77
mindspore/ccsrc/minddata/dataset/include/dataset/datasets.h
View File

@ -3691,6 +3691,83 @@ SpeechCommands(const std::string &dataset_dir, const std::string &usage, const s
return std::make_shared<SpeechCommandsDataset>(StringToChar(dataset_dir), StringToChar(usage), sampler, cache);
}
/// \class STL10Dataset
/// \brief A source dataset that reads and parses STL10 dataset.
class MS_API STL10Dataset : public Dataset {
public:
/// \brief Constructor of STL10Dataset.
/// \param[in] dataset_dir Path to the root directory that contains the dataset.
/// \param[in] usage Part of dataset of STL10, can be "train", "test", "unlabeled", "train+unlabeled"
/// or "all".
/// \param[in] sampler Shared pointer to a sampler object used to choose samples from the dataset. If sampler is not
/// given, a `RandomSampler` will be used to randomly iterate the entire dataset.
/// \param[in] cache Tensor cache to use.
STL10Dataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage,
const std::shared_ptr<Sampler> &sampler, const std::shared_ptr<DatasetCache> &cache);
/// \brief Constructor of STL10Dataset.
/// \param[in] dataset_dir Path to the root directory that contains the dataset.
/// \param[in] usage Part of dataset of STL10, can be "train", "test", "unlabeled", "train+unlabeled"
/// or "all".
/// \param[in] sampler Raw pointer to a sampler object used to choose samples from the dataset.
/// \param[in] cache Tensor cache to use.
STL10Dataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage, const Sampler *sampler,
const std::shared_ptr<DatasetCache> &cache);
/// \brief Constructor of STL10Dataset.
/// \param[in] dataset_dir Path to the root directory that contains the dataset.
/// \param[in] usage Part of dataset of STL10, can be "train", "test", "unlabeled", "train+unlabeled"
/// or "all".
/// \param[in] sampler Sampler object used to choose samples from the dataset.
/// \param[in] cache Tensor cache to use.
STL10Dataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage,
const std::reference_wrapper<Sampler> sampler, const std::shared_ptr<DatasetCache> &cache);
/// \brief Destructor of STL10Dataset.
~STL10Dataset() = default;
};
/// \brief Function to create a STL10 Dataset.
/// \notes The generated dataset has two columns ["image", "label"].
/// \param[in] dataset_dir Path to the root directory that contains the dataset.
/// \param[in] usage Usage of STL10, can be "train", "test", "unlabeled", "train+unlabeled" or "all" (default = "all").
/// \param[in] sampler Shared pointer to a sampler object used to choose samples from the dataset. If sampler is not
/// given, a `RandomSampler` will be used to randomly iterate the entire dataset (default = RandomSampler()).
/// \param[in] cache Tensor cache to use. (default=nullptr, which means no cache is used).
/// \return Shared pointer to the current Dataset.
inline std::shared_ptr<STL10Dataset> MS_API
STL10(const std::string &dataset_dir, const std::string &usage = "all",
const std::shared_ptr<Sampler> &sampler = std::make_shared<RandomSampler>(),
const std::shared_ptr<DatasetCache> &cache = nullptr) {
return std::make_shared<STL10Dataset>(StringToChar(dataset_dir), StringToChar(usage), sampler, cache);
}
/// \brief Function to create a STL10 Dataset.
/// \notes The generated dataset has two columns ["image", "label"].
/// \param[in] dataset_dir Path to the root directory that contains the dataset.
/// \param[in] usage Usage of STL10, can be "train", "test", "unlabeled" or "train+unlabeled" or "all"
/// \param[in] sampler Raw pointer to a sampler object used to choose samples from the dataset.
/// \param[in] cache Tensor cache to use. (default=nullptr, which means no cache is used).
/// \return Shared pointer to the current Dataset.
inline std::shared_ptr<STL10Dataset> MS_API STL10(const std::string &dataset_dir, const std::string &usage,
const Sampler *sampler,
const std::shared_ptr<DatasetCache> &cache = nullptr) {
return std::make_shared<STL10Dataset>(StringToChar(dataset_dir), StringToChar(usage), sampler, cache);
}
/// \brief Function to create a STL10 Dataset.
/// \notes The generated dataset has two columns ["image", "label"].
/// \param[in] dataset_dir Path to the root directory that contains the dataset.
/// \param[in] usage Usage of STL10, can be "train", "test", "unlabeled", "train+unlabeled" or "all"
/// \param[in] sampler Sampler object used to choose samples from the dataset.
/// \param[in] cache Tensor cache to use. (default=nullptr, which means no cache is used).
/// \return Shared pointer to the current Dataset.
inline std::shared_ptr<STL10Dataset> MS_API STL10(const std::string &dataset_dir, const std::string &usage,
const std::reference_wrapper<Sampler> sampler,
const std::shared_ptr<DatasetCache> &cache = nullptr) {
return std::make_shared<STL10Dataset>(StringToChar(dataset_dir), StringToChar(usage), sampler, cache);
}
/// \class TedliumDataset
/// \brief A source dataset for reading and parsing tedlium dataset.
class MS_API TedliumDataset : public Dataset {

1
mindspore/ccsrc/minddata/dataset/include/dataset/samplers.h
View File

@ -57,6 +57,7 @@ class MS_API Sampler : std::enable_shared_from_this<Sampler> {
friend class RandomDataDataset;
friend class SBUDataset;
friend class SpeechCommandsDataset;
friend class STL10Dataset;
friend class TedliumDataset;
friend class TextFileDataset;
friend class TFRecordDataset;

138
mindspore/dataset/engine/datasets.py
View File

@ -70,7 +70,8 @@ from .validators import check_batch, check_shuffle, check_map, check_filter, che
check_sb_dataset, check_flowers102dataset, check_cityscapes_dataset, check_usps_dataset, check_div2k_dataset, \
check_sbu_dataset, check_qmnist_dataset, check_emnist_dataset, check_fake_image_dataset, check_places365_dataset, \
check_photo_tour_dataset, check_ag_news_dataset, check_dbpedia_dataset, check_lj_speech_dataset, \
check_yes_no_dataset, check_speech_commands_dataset, check_tedlium_dataset, check_svhn_dataset
check_yes_no_dataset, check_speech_commands_dataset, check_tedlium_dataset, check_svhn_dataset, \
check_stl10_dataset
from ..core.config import get_callback_timeout, _init_device_info, get_enable_shared_mem, get_num_parallel_workers, \
get_prefetch_size, get_auto_offload
from ..core.datatypes import mstype_to_detype, mstypelist_to_detypelist
@ -9108,3 +9109,138 @@ class SVHNDataset(GeneratorDataset):
super().__init__(dataset, column_names=dataset.column_names, num_samples=num_samples,
num_parallel_workers=num_parallel_workers, shuffle=shuffle, sampler=sampler,
num_shards=num_shards, shard_id=shard_id)
class STL10Dataset(MappableDataset):
"""
A source dataset for reading and parsing STL10 dataset.
The generated dataset has two columns: :py:obj:`[image, label]`.
The tensor of column :py:obj:`image` is of the uint8 type.
The tensor of column :py:obj:`label` is of a scalar of int32 type.
Args:
dataset_dir (str): Path to the root directory that contains the dataset.
usage (str, optional): Usage of this dataset, can be "train", "test",
"unlabeled", "train+unlabeled" or "all" . "train" will read from 5,000
train samples, "test" will read from 8,000 test samples,
"unlabeled" will read from all 100,000 samples, and "train+unlabeled"
will read from 105000 samples, "all" will read all the samples
(default=None, all samples).
num_samples (int, optional): The number of images to be included in the dataset.
(default=None, all images).
num_parallel_workers (int, optional): Number of workers to read the data
(default=None, number set in the config).
shuffle (bool, optional): Whether to perform shuffle on the dataset (default=None, expected
order behavior shown in the table).
sampler (Sampler, optional): Object used to choose samples from the
dataset (default=None, expected order behavior shown in the table).
num_shards (int, optional): Number of shards that the dataset will be divided
into (default=None). When this argument is specified, 'num_samples' reflects
the max sample number of per shard.
shard_id (int, optional): The shard ID within num_shards (default=None). This
argument can only be specified when num_shards is also specified.
cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing.
(default=None, which means no cache is used).
Raises:
RuntimeError: If dataset_dir is not valid or does not exist or does not contain data files.
RuntimeError: If num_parallel_workers exceeds the max thread numbers.
RuntimeError: If sampler and shuffle are specified at the same time.
RuntimeError: If sampler and sharding are specified at the same time.
RuntimeError: If num_shards is specified but shard_id is None.
RuntimeError: If shard_id is specified but num_shards is None.
ValueError: If usage is invalid.
ValueError: If shard_id is invalid (< 0 or >= num_shards).
Note:
- This dataset can take in a sampler. 'sampler' and 'shuffle' are mutually exclusive.
The table below shows what input arguments are allowed and their expected behavior.
.. list-table:: Expected Order Behavior of Using 'sampler' and 'shuffle'
:widths: 25 25 50
:header-rows: 1
* - Parameter 'sampler'
- Parameter 'shuffle'
- Expected Order Behavior
* - None
- None
- random order
* - None
- True
- random order
* - None
- False
- sequential order
* - Sampler object
- None
- order defined by sampler
* - Sampler object
- True
- not allowed
* - Sampler object
- False
- not allowed
Examples:
>>> stl10_dataset_dir = "/path/to/stl10_dataset_directory"
>>>
>>> # 1) Get all samples from STL10 dataset in sequence
>>> dataset = ds.STL10Dataset(dataset_dir=stl10_dataset_dir, shuffle=False)
>>>
>>> # 2) Randomly select 350 samples from STL10 dataset
>>> dataset = ds.STL10Dataset(dataset_dir=stl10_dataset_dir, num_samples=350, shuffle=True)
>>>
>>> # 3) Get samples from STL10 dataset for shard 0 in a 2-way distributed training
>>> dataset = ds.STL10Dataset(dataset_dir=stl10_dataset_dir, num_shards=2, shard_id=0)
About STL10 dataset:
STL10 dataset consists of 10 classes: airplane, bird, car, cat, deer, dog, horse, monkey, ship, truck.
STL10 is is inspired by the CIFAR-10 dataset.
Images are 96x96 pixels, color.
500 training images, 800 test images per class and 100000 unlabeled images.
Labels are 0-indexed, and unlabeled images have -1 as their labels.
Here is the original STL10 dataset structure.
You can unzip the dataset files into this directory structure and read by MindSpore's API.
.. code-block::
.
└── stl10_dataset_dir
├── train_X.bin
├── train_y.bin
├── test_X.bin
├── test_y.bin
└── unlabeled_X.bin
Citation of STL10 dataset.
.. code-block::
@techreport{Coates10,
author = {Adam Coates},
title = {Learning multiple layers of features from tiny images},
year = {20010},
howpublished = {https://cs.stanford.edu/~acoates/stl10/},
description = {The STL-10 dataset consists of 96x96 RGB images in 10 classes,
with 500 training images and 800 testing images per class.
There are 5000 training images and 8000 test images.
It also has 100000 unlabeled images for unsupervised learning.
These examples are extracted from a similar but broader distribution of images.
}
}
"""
@check_stl10_dataset
def __init__(self, dataset_dir, usage=None, num_samples=None, num_parallel_workers=None, shuffle=None,
sampler=None, num_shards=None, shard_id=None, cache=None):
super().__init__(num_parallel_workers=num_parallel_workers, sampler=sampler, num_samples=num_samples,
shuffle=shuffle, num_shards=num_shards, shard_id=shard_id, cache=cache)
self.dataset_dir = dataset_dir
self.usage = replace_none(usage, "all")
def parse(self, children=None):
return cde.STL10Node(self.dataset_dir, self.usage, self.sampler)

46
mindspore/dataset/engine/validators.py
View File

@ -1930,3 +1930,49 @@ def check_svhn_dataset(method):
return method(self, *args, **kwargs)
return new_method
def check_stl10_dataset(method):
"""A wrapper that wraps a parameter checker around the original Dataset(STL10Dataset)."""
@wraps(method)
def new_method(self, *args, **kwargs):
_, param_dict = parse_user_args(method, *args, **kwargs)
nreq_param_int = ['num_samples', 'num_parallel_workers', 'num_shards', 'shard_id']
nreq_param_bool = ['shuffle']
dataset_dir = param_dict.get('dataset_dir')
check_dir(dataset_dir)
usage = param_dict.get('usage')
if usage is not None:
check_valid_str(usage, ["train", "test", "unlabeled", "train+unlabeled", "all"], "usage")
if usage == "all":
for _usage in ["train", "test", "unlabeled"]:
check_file(os.path.join(dataset_dir, _usage + "_X.bin"))
if _usage == "unlabeled":
continue
else:
check_file(os.path.join(dataset_dir, _usage + "_y.bin"))
elif usage == "train+unlabeled":
check_file(os.path.join(dataset_dir, "train_X.bin"))
check_file(os.path.join(dataset_dir, "train_y.bin"))
check_file(os.path.join(dataset_dir, "unlabeled_X.bin"))
elif usage == "unlabeled":
check_file(os.path.join(dataset_dir, "unlabeled_X.bin"))
else:
check_file(os.path.join(dataset_dir, usage + "_X.bin"))
check_file(os.path.join(dataset_dir, usage + "_y.bin"))
validate_dataset_param_value(nreq_param_int, param_dict, int)
validate_dataset_param_value(nreq_param_bool, param_dict, bool)
check_sampler_shuffle_shard_options(param_dict)
cache = param_dict.get('cache')
check_cache_option(cache)
return method(self, *args, **kwargs)
return new_method

1
tests/ut/cpp/dataset/CMakeLists.txt
View File

@ -41,6 +41,7 @@ SET(DE_UT_SRCS
c_api_dataset_save.cc
c_api_dataset_sbu_test.cc
c_api_dataset_speech_commands_test.cc
c_api_dataset_stl10_test.cc
c_api_dataset_tedlium_test.cc
c_api_dataset_textfile_test.cc
c_api_dataset_tfrecord_test.cc

431
tests/ut/cpp/dataset/c_api_dataset_stl10_test.cc Normal file
View File

@ -0,0 +1,431 @@
/**
* Copyright 2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "common/common.h"
#include "minddata/dataset/include/dataset/datasets.h"
using namespace mindspore::dataset;
using mindspore::dataset::DataType;
using mindspore::dataset::Tensor;
using mindspore::dataset::TensorShape;
class MindDataTestPipeline : public UT::DatasetOpTesting {
protected:
};
/// Feature: STL10TrainDataset.
/// Description: test basic usage of STL10TrainDataset.
/// Expectation: get correct number of data.
TEST_F(MindDataTestPipeline, TestSTL10TrainDataset) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSTL10TrainDataset.";
// Create a STL10 Dataset
std::string folder_path = datasets_root_path_ + "/testSTL10Data/";
std::shared_ptr<Dataset> ds = STL10(folder_path, "train", std::make_shared<RandomSampler>(false, 1));
EXPECT_NE(ds, nullptr);
// Create an iterator over the result of the above dataset
// This will trigger the creation of the Execution Tree and launch it.
std::shared_ptr<Iterator> iter = ds->CreateIterator();
EXPECT_NE(iter, nullptr);
// Iterate the dataset and get each row
std::unordered_map<std::string, mindspore::MSTensor> row;
ASSERT_OK(iter->GetNextRow(&row));
EXPECT_NE(row.find("image"), row.end());
EXPECT_NE(row.find("label"), row.end());
uint64_t i = 0;
while (row.size() != 0) {
i++;
auto image = row["image"];
MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
ASSERT_OK(iter->GetNextRow(&row));
}
EXPECT_EQ(i, 1);
// Manually terminate the pipeline
iter->Stop();
}
/// Feature: STL10TestDataset.
/// Description: test basic usage of STL10TestDataset.
/// Expectation: get correct number of data.
TEST_F(MindDataTestPipeline, TestSTL10TestDataset) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSTL10TestDataset.";
// Create a STL10 Dataset
std::string folder_path = datasets_root_path_ + "/testSTL10Data/";
std::shared_ptr<Dataset> ds = STL10(folder_path, "test", std::make_shared<RandomSampler>(false, 1));
EXPECT_NE(ds, nullptr);
// Create an iterator over the result of the above dataset
// This will trigger the creation of the Execution Tree and launch it.
std::shared_ptr<Iterator> iter = ds->CreateIterator();
EXPECT_NE(iter, nullptr);
// Iterate the dataset and get each row
std::unordered_map<std::string, mindspore::MSTensor> row;
ASSERT_OK(iter->GetNextRow(&row));
EXPECT_NE(row.find("image"), row.end());
EXPECT_NE(row.find("label"), row.end());
uint64_t i = 0;
while (row.size() != 0) {
i++;
auto image = row["image"];
MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
ASSERT_OK(iter->GetNextRow(&row));
}
EXPECT_EQ(i, 1);
// Manually terminate the pipeline
iter->Stop();
}
/// Feature: STL10UnlabeledDataset.
/// Description: test basic usage of STL10UnlabeledDataset.
/// Expectation: get correct number of data.
TEST_F(MindDataTestPipeline, TestSTL10UnlabeledDataset) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSTL10UnlabeledDataset.";
// Create a STL10 Dataset
std::string folder_path = datasets_root_path_ + "/testSTL10Data/";
std::shared_ptr<Dataset> ds = STL10(folder_path, "unlabeled", std::make_shared<RandomSampler>(false, 1));
EXPECT_NE(ds, nullptr);
// Create an iterator over the result of the above dataset
// This will trigger the creation of the Execution Tree and launch it.
std::shared_ptr<Iterator> iter = ds->CreateIterator();
EXPECT_NE(iter, nullptr);
// Iterate the dataset and get each row
std::unordered_map<std::string, mindspore::MSTensor> row;
ASSERT_OK(iter->GetNextRow(&row));
EXPECT_NE(row.find("image"), row.end());
EXPECT_NE(row.find("label"), row.end());
uint64_t i = 0;
while (row.size() != 0) {
i++;
auto image = row["image"];
MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
ASSERT_OK(iter->GetNextRow(&row));
}
EXPECT_EQ(i, 1);
// Manually terminate the pipeline
iter->Stop();
}
/// Feature: STL10TrainUnlabeledDataset.
/// Description: test basic usage of STL10TrainUnlabeledDataset.
/// Expectation: get correct number of data.
TEST_F(MindDataTestPipeline, TestSTL10TrainUnlabeledDataset) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSTL10TrainUnlabeledDataset.";
// Create a STL10 Dataset
std::string folder_path = datasets_root_path_ + "/testSTL10Data/";
std::shared_ptr<Dataset> ds = STL10(folder_path, "train+unlabeled", std::make_shared<RandomSampler>(false, 2));
EXPECT_NE(ds, nullptr);
// Create an iterator over the result of the above dataset
// This will trigger the creation of the Execution Tree and launch it.
std::shared_ptr<Iterator> iter = ds->CreateIterator();
EXPECT_NE(iter, nullptr);
// Iterate the dataset and get each row
std::unordered_map<std::string, mindspore::MSTensor> row;
ASSERT_OK(iter->GetNextRow(&row));
EXPECT_NE(row.find("image"), row.end());
EXPECT_NE(row.find("label"), row.end());
uint64_t i = 0;
while (row.size() != 0) {
i++;
auto image = row["image"];
MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
ASSERT_OK(iter->GetNextRow(&row));
}
EXPECT_EQ(i, 2);
// Manually terminate the pipeline
iter->Stop();
}
/// Feature: STL10AllDataset.
/// Description: test basic usage of STL10AllDataset.
/// Expectation: get correct number of data.
TEST_F(MindDataTestPipeline, TestSTL10AllDataset) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSTL10AllDataset.";
// Create a STL10 Dataset
std::string folder_path = datasets_root_path_ + "/testSTL10Data/";
std::shared_ptr<Dataset> ds = STL10(folder_path, "all", std::make_shared<RandomSampler>(false, 2));
EXPECT_NE(ds, nullptr);
// Create an iterator over the result of the above dataset
// This will trigger the creation of the Execution Tree and launch it.
std::shared_ptr<Iterator> iter = ds->CreateIterator();
EXPECT_NE(iter, nullptr);
// Iterate the dataset and get each row
std::unordered_map<std::string, mindspore::MSTensor> row;
ASSERT_OK(iter->GetNextRow(&row));
EXPECT_NE(row.find("image"), row.end());
EXPECT_NE(row.find("label"), row.end());
uint64_t i = 0;
while (row.size() != 0) {
i++;
auto image = row["image"];
MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
ASSERT_OK(iter->GetNextRow(&row));
}
EXPECT_EQ(i, 2);
// Manually terminate the pipeline
iter->Stop();
}
/// Feature: STL10TrainDatasetWithPipeline.
/// Description: test usage of STL10TrainDataset with pipeline.
/// Expectation: get correct number of data.
TEST_F(MindDataTestPipeline, TestSTL10TrainDatasetWithPipeline) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSTL10TrainDatasetWithPipeline.";
// Create two STL10 Dataset
std::string folder_path = datasets_root_path_ + "/testSTL10Data/";
std::shared_ptr<Dataset> ds1 = STL10(folder_path, "train", std::make_shared<RandomSampler>(false, 2));
std::shared_ptr<Dataset> ds2 = STL10(folder_path, "train", std::make_shared<RandomSampler>(false, 2));
EXPECT_NE(ds1, nullptr);
EXPECT_NE(ds2, nullptr);
// Create two Repeat operation on ds
int32_t repeat_num = 1;
ds1 = ds1->Repeat(repeat_num);
EXPECT_NE(ds1, nullptr);
repeat_num = 1;
ds2 = ds2->Repeat(repeat_num);
EXPECT_NE(ds2, nullptr);
// Create two Project operation on ds
std::vector<std::string> column_project = {"image", "label"};
ds1 = ds1->Project(column_project);
EXPECT_NE(ds1, nullptr);
ds2 = ds2->Project(column_project);
EXPECT_NE(ds2, nullptr);
// Create a Concat operation on the ds
ds1 = ds1->Concat({ds2});
EXPECT_NE(ds1, nullptr);
// Create an iterator over the result of the above dataset
// This will trigger the creation of the Execution Tree and launch it.
std::shared_ptr<Iterator> iter = ds1->CreateIterator();
EXPECT_NE(iter, nullptr);
// Iterate the dataset and get each row
std::unordered_map<std::string, mindspore::MSTensor> row;
ASSERT_OK(iter->GetNextRow(&row));
EXPECT_NE(row.find("image"), row.end());
EXPECT_NE(row.find("label"), row.end());
uint64_t i = 0;
while (row.size() != 0) {
i++;
auto image = row["image"];
MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
ASSERT_OK(iter->GetNextRow(&row));
}
EXPECT_EQ(i, 2);
// Manually terminate the pipeline
iter->Stop();
}
/// Feature: STL10GetTrainDatasetSize.
/// Description: test usage of STL10GetTrainDatasetSize with pipeline.
/// Expectation: get correct number of data.
TEST_F(MindDataTestPipeline, TestSTL10GetTrainDatasetSize) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSTL10GetTrainDatasetSize.";
// Create a STL10 Dataset
std::string folder_path = datasets_root_path_ + "/testSTL10Data/";
std::shared_ptr<Dataset> ds = STL10(folder_path, "train");
EXPECT_NE(ds, nullptr);
EXPECT_EQ(ds->GetDatasetSize(), 1);
}
/// Feature: STL10GetTestDatasetSize.
/// Description: test usage of STL10GetTestDatasetSize with pipeline.
/// Expectation: get correct number of data.
TEST_F(MindDataTestPipeline, TestSTL10GetTestDatasetSize) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSTL10GetTestDatasetSize.";
// Create a STL10 Dataset
std::string folder_path = datasets_root_path_ + "/testSTL10Data/";
std::shared_ptr<Dataset> ds = STL10(folder_path, "test");
EXPECT_NE(ds, nullptr);
EXPECT_EQ(ds->GetDatasetSize(), 1);
}
/// Feature: STL10GetUnlabeledDatasetSize.
/// Description: test usage of STL10GetUnlabeledDatasetSize with pipeline.
/// Expectation: get correct number of data.
TEST_F(MindDataTestPipeline, TestSTL10GetUnlabeledDatasetSize) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSTL10GetUnlabeledDatasetSize.";
// Create a STL10 Dataset
std::string folder_path = datasets_root_path_ + "/testSTL10Data/";
std::shared_ptr<Dataset> ds = STL10(folder_path, "unlabeled");
EXPECT_NE(ds, nullptr);
EXPECT_EQ(ds->GetDatasetSize(), 1);
}
/// Feature: STL10GetTrainUnlabeledDatasetSize.
/// Description: test usage of STL10GetTrainUnlabeledDatasetSize with pipeline.
/// Expectation: get correct number of data.
TEST_F(MindDataTestPipeline, TestSTL10GetTrainUnlabeledDatasetSize) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSTL10GetTrainUnlabeledDatasetSize.";
// Create a STL10 Dataset
std::string folder_path = datasets_root_path_ + "/testSTL10Data/";
std::shared_ptr<Dataset> ds = STL10(folder_path, "train+unlabeled");
EXPECT_NE(ds, nullptr);
EXPECT_EQ(ds->GetDatasetSize(), 2);
}
/// Feature: STL10GetAllDatasetSize.
/// Description: test usage of STL10GetAllDatasetSize with pipeline.
/// Expectation: get correct number of data.
TEST_F(MindDataTestPipeline, TestSTL10GetAllDatasetSize) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSTL10GetAllDatasetSize.";
// Create a STL10 Dataset
std::string folder_path = datasets_root_path_ + "/testSTL10Data/";
std::shared_ptr<Dataset> ds = STL10(folder_path, "all");
EXPECT_NE(ds, nullptr);
EXPECT_EQ(ds->GetDatasetSize(), 3);
}
/// Feature: STL10TrainDatasetGetters.
/// Description: test usage of getters STL10TrainDataset.
/// Expectation: get correct number of data and correct tensor shape.
TEST_F(MindDataTestPipeline, TestSTL10TrainGetters) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSTL10TrainGetter.";
// Create a STL10 Dataset
std::string folder_path = datasets_root_path_ + "/testSTL10Data/";
std::shared_ptr<Dataset> ds = STL10(folder_path, "train");
EXPECT_NE(ds, nullptr);
EXPECT_EQ(ds->GetDatasetSize(), 1);
std::vector<DataType> types = ToDETypes(ds->GetOutputTypes());
std::vector<TensorShape> shapes = ToTensorShapeVec(ds->GetOutputShapes());
std::vector<std::string> column_names = {"image", "label"};
int64_t num_classes = ds->GetNumClasses();
EXPECT_EQ(types.size(), 2);
EXPECT_EQ(types[0].ToString(), "uint8");
EXPECT_EQ(types[1].ToString(), "int32");
EXPECT_EQ(shapes.size(), 2);
EXPECT_EQ(shapes[0].ToString(), "<96,96,3>");
EXPECT_EQ(shapes[1].ToString(), "<>");
EXPECT_EQ(num_classes, -1);
EXPECT_EQ(ds->GetBatchSize(), 1);
EXPECT_EQ(ds->GetRepeatCount(), 1);
EXPECT_EQ(ds->GetDatasetSize(), 1);
EXPECT_EQ(ToDETypes(ds->GetOutputTypes()), types);
EXPECT_EQ(ToTensorShapeVec(ds->GetOutputShapes()), shapes);
EXPECT_EQ(ds->GetNumClasses(), -1);
EXPECT_EQ(ds->GetColumnNames(), column_names);
EXPECT_EQ(ds->GetDatasetSize(), 1);
EXPECT_EQ(ToDETypes(ds->GetOutputTypes()), types);
EXPECT_EQ(ToTensorShapeVec(ds->GetOutputShapes()), shapes);
EXPECT_EQ(ds->GetBatchSize(), 1);
EXPECT_EQ(ds->GetRepeatCount(), 1);
EXPECT_EQ(ds->GetNumClasses(), -1);
EXPECT_EQ(ds->GetDatasetSize(), 1);
}
/// Feature: STL10DatasetFail.
/// Description: test failure of STL10Dataset.
/// Expectation: get none piece of data.
TEST_F(MindDataTestPipeline, testSTL10DataFail) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-testSTL10DataFail.";
// Create a STL10 Dataset
std::shared_ptr<Dataset> ds = STL10("", "train", std::make_shared<RandomSampler>(false, 2));
EXPECT_NE(ds, nullptr);
// Create an iterator over the result of the above dataset
std::shared_ptr<Iterator> iter = ds->CreateIterator();
// Expect failure: invalid STL10 input
EXPECT_EQ(iter, nullptr);
}
/// Feature: STL10DatasetWithInvalidUsageFail.
/// Description: test failure of STL10Dataset with invalid usage.
/// Expectation: get none piece of data.
TEST_F(MindDataTestPipeline, testSTL10DataWithInvalidUsageFail) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-testSTL10DataWithNullSamplerFail.";
// Create a STL10 Dataset
std::string folder_path = datasets_root_path_ + "/testSTL10Data/";
std::shared_ptr<Dataset> ds = STL10(folder_path, "validation");
EXPECT_NE(ds, nullptr);
// Create an iterator over the result of the above dataset
std::shared_ptr<Iterator> iter = ds->CreateIterator();
// Expect failure: invalid STL10 input, validation is not a valid usage
EXPECT_EQ(iter, nullptr);
}
/// Feature: STL10DatasetWithNullSamplerFail.
/// Description: test failure of STL10Dataset with null sampler.
/// Expectation: get none piece of data.
TEST_F(MindDataTestPipeline, testSTL10DataWithNullSamplerFail) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-testSTL10DataWithNullSamplerFail.";
// Create a STL10 Dataset
std::string folder_path = datasets_root_path_ + "/testSTL10Data/";
std::shared_ptr<Dataset> ds = STL10(folder_path, "train", nullptr);
EXPECT_NE(ds, nullptr);
// Create an iterator over the result of the above dataset
std::shared_ptr<Iterator> iter = ds->CreateIterator();
// Expect failure: invalid STL10 input, sampler cannot be nullptr
EXPECT_EQ(iter, nullptr);
}

BIN
tests/ut/data/dataset/testSTL10Data/test_X.bin Normal file
View File

Binary file not shown.

1
tests/ut/data/dataset/testSTL10Data/test_y.bin Normal file
View File

@ -0,0 +1 @@

BIN
tests/ut/data/dataset/testSTL10Data/train_X.bin Normal file
View File

Binary file not shown.

1
tests/ut/data/dataset/testSTL10Data/train_y.bin Normal file
View File

@ -0,0 +1 @@


BIN
tests/ut/data/dataset/testSTL10Data/unlabeled_X.bin Normal file
View File

Binary file not shown.

397
tests/ut/python/dataset/test_datasets_stl10.py Normal file
View File

@ -0,0 +1,397 @@
# Copyright 2021 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""
Test STL10 dataset operators
"""
import os
import matplotlib.pyplot as plt
import numpy as np
import pytest
import mindspore.dataset as ds
import mindspore.dataset.vision.c_transforms as vision
from mindspore import log as logger
DATA_DIR = "../data/dataset/testSTL10Data"
WRONG_DIR = "../data/dataset/testMnistData"
def loadfile(path_to_data, path_to_labels=None):
"""
Feature: loadfile.
Description: parse stl10 file.
Expectation: get image and label of stl10 dataset.
"""
labels = None
if path_to_labels:
with open(os.path.realpath(path_to_labels), 'rb') as f:
labels = np.fromfile(f, dtype=np.uint8) - 1 # 0-based
with open(path_to_data, 'rb') as f:
# read whole file in uint8 chunks
everything = np.fromfile(f, dtype=np.uint8)
images = np.reshape(everything, (-1, 3, 96, 96))
images = np.transpose(images, (0, 1, 3, 2))
return images, labels
def load_stl10(path, usage):
"""
Feature: load_stl10.
Description: load stl10.
Expectation: get data of stl10 dataset.
"""
assert usage in ["train", "test", "unlabeled", "train+unlabeled", "all"]
if usage == "train":
image_path = os.path.join(path, "train_X.bin")
label_path = os.path.join(path, "train_y.bin")
images, labels = loadfile(image_path, label_path)
elif usage == "train+unlabeled":
image_path = os.path.join(path, "train_X.bin")
label_path = os.path.join(path, "train_y.bin")
images, labels = loadfile(image_path, label_path)
image_path = os.path.join(path, "unlabeled_X.bin")
unlabeled_image, _ = loadfile(image_path)
images = np.concatenate((images, unlabeled_image))
labels = np.concatenate((labels, np.asarray([-1] * unlabeled_image.shape[0])))
elif usage == "unlabeled":
image_path = os.path.join(path, "unlabeled_X.bin")
images, _ = loadfile(image_path)
labels = np.asarray([-1] * images.shape[0])
elif usage == "test":
image_path = os.path.join(path, "test_X.bin")
label_path = os.path.join(path, "test_y.bin")
images, labels = loadfile(image_path, label_path)
elif usage == "all":
image_path = os.path.join(path, "test_X.bin")
label_path = os.path.join(path, "test_y.bin")
images, labels = loadfile(image_path, label_path)
image_path = os.path.join(path, "train_X.bin")
label_path = os.path.join(path, "train_y.bin")
train_image, train_label = loadfile(image_path, label_path)
images = np.concatenate((images, train_image))
labels = np.concatenate((labels, train_label))
image_path = os.path.join(path, "unlabeled_X.bin")
unlabeled_image, _ = loadfile(image_path)
images = np.concatenate((images, unlabeled_image))
labels = np.concatenate((labels, np.asarray([-1] * unlabeled_image.shape[0])))
return images, labels
def visualize_dataset(images, labels):
"""
Feature: visualize_dataset.
Description: visualize stl10 dataset.
Expectation: plot images.
"""
num_samples = len(images)
for i in range(num_samples):
plt.subplot(1, num_samples, i + 1)
plt.imshow(np.transpose(images[i], (1, 2, 0)))
plt.title(labels[i])
plt.show()
def test_stl10_content_check():
"""
Feature: test_stl10_content_check.
Description: validate STL10ataset image readings.
Expectation: get correct number of data and correct content.
"""
logger.info("Test STL10Dataset Op with content check")
# 1. train data.
data1 = ds.STL10Dataset(DATA_DIR, usage="train", num_samples=1, shuffle=False)
images, labels = load_stl10(DATA_DIR, "train")
num_iter = 0
# in this example, each dictionary has keys "image" and "label".
for i, d in enumerate(data1.create_dict_iterator(num_epochs=1, output_numpy=True)):
np.testing.assert_array_equal(d["image"], np.transpose(images[i], (1, 2, 0)))
np.testing.assert_array_equal(d["label"], labels[i])
num_iter += 1
assert num_iter == 1
# 2. test data.
data1 = ds.STL10Dataset(DATA_DIR, usage="test", num_samples=1, shuffle=False)
images, labels = load_stl10(DATA_DIR, "test")
num_iter = 0
# in this example, each dictionary has keys "image" and "label".
for i, d in enumerate(data1.create_dict_iterator(num_epochs=1, output_numpy=True)):
np.testing.assert_array_equal(d["image"], np.transpose(images[i], (1, 2, 0)))
np.testing.assert_array_equal(d["label"], labels[i])
num_iter += 1
assert num_iter == 1
# 3. unlabeled data.
data1 = ds.STL10Dataset(DATA_DIR, usage="unlabeled", num_samples=1, shuffle=False)
images, labels = load_stl10(DATA_DIR, "unlabeled")
num_iter = 0
# in this example, each dictionary has keys "image" and "label".
for i, d in enumerate(data1.create_dict_iterator(num_epochs=1, output_numpy=True)):
np.testing.assert_array_equal(d["image"], np.transpose(images[i], (1, 2, 0)))
np.testing.assert_array_equal(d["label"], labels[i])
num_iter += 1
assert num_iter == 1
# 4. train+unlabeled data.
data1 = ds.STL10Dataset(DATA_DIR, usage="train+unlabeled", num_samples=2, shuffle=False)
images, labels = load_stl10(DATA_DIR, "train+unlabeled")
num_iter = 0
# in this example, each dictionary has keys "image" and "label".
for i, d in enumerate(data1.create_dict_iterator(num_epochs=1, output_numpy=True)):
np.testing.assert_array_equal(d["image"], np.transpose(images[i], (1, 2, 0)))
np.testing.assert_array_equal(d["label"], labels[i])
num_iter += 1
assert num_iter == 2
# 4. all data.
data1 = ds.STL10Dataset(DATA_DIR, usage="all", num_samples=3, shuffle=False)
images, labels = load_stl10(DATA_DIR, "all")
num_iter = 0
# in this example, each dictionary has keys "image" and "label".
for i, d in enumerate(data1.create_dict_iterator(num_epochs=1, output_numpy=True)):
np.testing.assert_array_equal(d["image"], np.transpose(images[i], (1, 2, 0)))
np.testing.assert_array_equal(d["label"], labels[i])
num_iter += 1
assert num_iter == 3
def test_stl10_basic():
"""
Feature: test_stl10_basic.
Description: test basic usage of STL10Dataset.
Expectation: get correct number of data.
"""
logger.info("Test STL10Dataset Op")
# case 1: test loading whole dataset.
all_data = ds.STL10Dataset(DATA_DIR, "all")
num_iter = 0
for _ in all_data.create_dict_iterator(num_epochs=1):
num_iter += 1
assert num_iter == 3
# case 2: test num_samples.
all_data = ds.STL10Dataset(DATA_DIR, "all", num_samples=1)
num_iter = 0
for _ in all_data.create_dict_iterator(num_epochs=1):
num_iter += 1
assert num_iter == 1
# case 3: test repeat.
all_data = ds.STL10Dataset(DATA_DIR, "all", num_samples=2)
all_data = all_data.repeat(5)
num_iter = 0
for _ in all_data.create_dict_iterator(num_epochs=1):
num_iter += 1
assert num_iter == 10
# case 4: test batch with drop_remainder=False.
all_data = ds.STL10Dataset(DATA_DIR, "all", num_samples=2)
assert all_data.get_dataset_size() == 2
assert all_data.get_batch_size() == 1
all_data = all_data.batch(batch_size=2) # drop_remainder is default to be False.
assert all_data.get_batch_size() == 2
assert all_data.get_dataset_size() == 1
num_iter = 0
for _ in all_data.create_dict_iterator(num_epochs=1):
num_iter += 1
assert num_iter == 1
# case 5: test batch with drop_remainder=True.
all_data = ds.STL10Dataset(DATA_DIR, "all", num_samples=2)
assert all_data.get_dataset_size() == 2
assert all_data.get_batch_size() == 1
all_data = all_data.batch(batch_size=2, drop_remainder=True) # the rest of incomplete batch will be dropped.
assert all_data.get_dataset_size() == 1
assert all_data.get_batch_size() == 2
num_iter = 0
for _ in all_data.create_dict_iterator(num_epochs=1):
num_iter += 1
assert num_iter == 1
def test_stl10_sequential_sampler():
"""
Feature: test_stl10_sequential_sampler.
Description: test usage of STL10Dataset with SequentialSampler.
Expectation: get correct number of data.
"""
logger.info("Test STL10Dataset Op with SequentialSampler")
num_samples = 2
sampler = ds.SequentialSampler(num_samples=num_samples)
all_data_1 = ds.STL10Dataset(DATA_DIR, "all", sampler=sampler)
all_data_2 = ds.STL10Dataset(DATA_DIR, "all", shuffle=False, num_samples=num_samples)
label_list_1, label_list_2 = [], []
num_iter = 0
for item1, item2 in zip(all_data_1.create_dict_iterator(num_epochs=1),
all_data_2.create_dict_iterator(num_epochs=1)):
label_list_1.append(item1["label"].asnumpy())
label_list_2.append(item2["label"].asnumpy())
num_iter += 1
np.testing.assert_array_equal(label_list_1, label_list_2)
assert num_iter == num_samples
def test_stl10_exception():
"""
Feature: test_stl10_exception.
Description: test error cases for STL10Dataset.
Expectation: raise exception.
"""
logger.info("Test error cases for STL10Dataset")
error_msg_1 = "sampler and shuffle cannot be specified at the same time"
with pytest.raises(RuntimeError, match=error_msg_1):
ds.STL10Dataset(DATA_DIR, "all", shuffle=False, sampler=ds.PKSampler(3))
error_msg_2 = "sampler and sharding cannot be specified at the same time"
with pytest.raises(RuntimeError, match=error_msg_2):
ds.STL10Dataset(DATA_DIR, "all", sampler=ds.PKSampler(3), num_shards=2, shard_id=0)
error_msg_3 = "num_shards is specified and currently requires shard_id as well"
with pytest.raises(RuntimeError, match=error_msg_3):
ds.STL10Dataset(DATA_DIR, "all", num_shards=10)
error_msg_4 = "shard_id is specified but num_shards is not"
with pytest.raises(RuntimeError, match=error_msg_4):
ds.STL10Dataset(DATA_DIR, "all", shard_id=0)
error_msg_5 = "Input shard_id is not within the required interval"
with pytest.raises(ValueError, match=error_msg_5):
ds.STL10Dataset(DATA_DIR, "all", num_shards=5, shard_id=-1)
with pytest.raises(ValueError, match=error_msg_5):
ds.STL10Dataset(DATA_DIR, "all", num_shards=5, shard_id=5)
with pytest.raises(ValueError, match=error_msg_5):
ds.STL10Dataset(DATA_DIR, "all", num_shards=2, shard_id=5)
error_msg_6 = "num_parallel_workers exceeds"
with pytest.raises(ValueError, match=error_msg_6):
ds.STL10Dataset(DATA_DIR, "all", shuffle=False, num_parallel_workers=0)
with pytest.raises(ValueError, match=error_msg_6):
ds.STL10Dataset(DATA_DIR, "all", shuffle=False, num_parallel_workers=256)
with pytest.raises(ValueError, match=error_msg_6):
ds.STL10Dataset(DATA_DIR, "all", shuffle=False, num_parallel_workers=-2)
error_msg_7 = "Argument shard_id"
with pytest.raises(TypeError, match=error_msg_7):
ds.STL10Dataset(DATA_DIR, "all", num_shards=2, shard_id="0")
def exception_func(item):
raise Exception("Error occur!")
error_msg_8 = "The corresponding data files"
with pytest.raises(RuntimeError, match=error_msg_8):
all_data = ds.STL10Dataset(DATA_DIR, "all")
all_data = all_data.map(operations=exception_func, input_columns=["image"], num_parallel_workers=1)
for _ in all_data.__iter__():
pass
with pytest.raises(RuntimeError, match=error_msg_8):
all_data = ds.STL10Dataset(DATA_DIR, "all")
all_data = all_data.map(operations=vision.Decode(), input_columns=["image"], num_parallel_workers=1)
for _ in all_data.__iter__():
pass
error_msg_9 = "does not exist or permission denied!"
with pytest.raises(ValueError, match=error_msg_9):
all_data = ds.STL10Dataset(WRONG_DIR, "all")
for _ in all_data.__iter__():
pass
def test_stl10_visualize(plot=False):
"""
Feature: test_stl10_visualize.
Description: visualize STL10Dataset results.
Expectation: get correct number of data and plot them.
"""
logger.info("Test STL10Dataset visualization")
all_data = ds.STL10Dataset(DATA_DIR, "all", num_samples=2, shuffle=False)
num_iter = 0
image_list, label_list = [], []
for item in all_data.create_dict_iterator(num_epochs=1, output_numpy=True):
image = item["image"]
label = item["label"]
image_list.append(image)
label_list.append("label {}".format(label))
assert isinstance(image, np.ndarray)
assert image.shape == (96, 96, 3)
assert image.dtype == np.uint8
assert label.dtype == np.int32
num_iter += 1
assert num_iter == 2
if plot:
visualize_dataset(image_list, label_list)
def test_stl10_usage():
"""
Feature: test_stl10_usage.
Description: validate STL10Dataset image readings.
Expectation: get correct number of data.
"""
logger.info("Test STL10Dataset usage flag")
def test_config(usage, path=None):
path = DATA_DIR if path is None else path
try:
data = ds.STL10Dataset(path, usage=usage, shuffle=False)
num_rows = 0
for _ in data.create_dict_iterator(num_epochs=1, output_numpy=True):
num_rows += 1
except (ValueError, TypeError, RuntimeError) as e:
return str(e)
return num_rows
assert test_config("train") == 1
assert test_config("test") == 1
assert test_config("unlabeled") == 1
assert test_config("train+unlabeled") == 2
assert test_config("all") == 3
assert "Input usage is not within the valid set of ['train', 'test', 'unlabeled', 'train+unlabeled', 'all']."\
in test_config("invalid")
assert "Argument usage with value ['list'] is not of type [<class 'str'>]" in test_config(["list"])
# change this directory to the folder that contains all STL10 files.
all_files_path = None
# the following tests on the entire datasets.
if all_files_path is not None:
assert test_config("train", all_files_path) == 1
assert ds.STL10Dataset(all_files_path, usage="train").get_dataset_size() == 1
if __name__ == '__main__':
test_stl10_content_check()
test_stl10_basic()
test_stl10_sequential_sampler()
test_stl10_exception()
test_stl10_visualize(plot=True)
test_stl10_usage()