diff --git a/mindspore/ccsrc/minddata/dataset/api/datasets.cc b/mindspore/ccsrc/minddata/dataset/api/datasets.cc index 3ad7b53bf07..c02e543c42d 100644 --- a/mindspore/ccsrc/minddata/dataset/api/datasets.cc +++ b/mindspore/ccsrc/minddata/dataset/api/datasets.cc @@ -113,6 +113,7 @@ #include "minddata/dataset/engine/ir/datasetops/source/random_node.h" #include "minddata/dataset/engine/ir/datasetops/source/sbu_node.h" #include "minddata/dataset/engine/ir/datasetops/source/speech_commands_node.h" +#include "minddata/dataset/engine/ir/datasetops/source/stl10_node.h" #include "minddata/dataset/engine/ir/datasetops/source/tedlium_node.h" #include "minddata/dataset/engine/ir/datasetops/source/text_file_node.h" #include "minddata/dataset/engine/ir/datasetops/source/tf_record_node.h" @@ -1500,6 +1501,27 @@ TedliumDataset::TedliumDataset(const std::vector &dataset_dir, const std:: ir_node_ = std::static_pointer_cast(ds); } +STL10Dataset::STL10Dataset(const std::vector &dataset_dir, const std::vector &usage, + const std::shared_ptr &sampler, const std::shared_ptr &cache) { + auto sampler_obj = sampler ? sampler->Parse() : nullptr; + auto ds = std::make_shared(CharToString(dataset_dir), CharToString(usage), sampler_obj, cache); + ir_node_ = std::static_pointer_cast(ds); +} + +STL10Dataset::STL10Dataset(const std::vector &dataset_dir, const std::vector &usage, const Sampler *sampler, + const std::shared_ptr &cache) { + auto sampler_obj = sampler ? sampler->Parse() : nullptr; + auto ds = std::make_shared(CharToString(dataset_dir), CharToString(usage), sampler_obj, cache); + ir_node_ = std::static_pointer_cast(ds); +} + +STL10Dataset::STL10Dataset(const std::vector &dataset_dir, const std::vector &usage, + const std::reference_wrapper sampler, const std::shared_ptr &cache) { + auto sampler_obj = sampler.get().Parse(); + auto ds = std::make_shared(CharToString(dataset_dir), CharToString(usage), sampler_obj, cache); + ir_node_ = std::static_pointer_cast(ds); +} + TextFileDataset::TextFileDataset(const std::vector> &dataset_files, int64_t num_samples, ShuffleMode shuffle, int32_t num_shards, int32_t shard_id, const std::shared_ptr &cache) { diff --git a/mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/engine/ir/datasetops/source/bindings.cc b/mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/engine/ir/datasetops/source/bindings.cc index 29ad4ecca73..cde5102e96f 100644 --- a/mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/engine/ir/datasetops/source/bindings.cc +++ b/mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/engine/ir/datasetops/source/bindings.cc @@ -45,6 +45,7 @@ #include "minddata/dataset/engine/ir/datasetops/source/mnist_node.h" #include "minddata/dataset/engine/ir/datasetops/source/random_node.h" #include "minddata/dataset/engine/ir/datasetops/source/speech_commands_node.h" +#include "minddata/dataset/engine/ir/datasetops/source/stl10_node.h" #include "minddata/dataset/engine/ir/datasetops/source/tedlium_node.h" #include "minddata/dataset/engine/ir/datasetops/source/text_file_node.h" #include "minddata/dataset/engine/ir/datasetops/source/yes_no_node.h" @@ -412,6 +413,16 @@ PYBIND_REGISTER(SpeechCommandsNode, 2, ([](const py::module *m) { })); })); +PYBIND_REGISTER(STL10Node, 2, ([](const py::module *m) { + (void)py::class_>(*m, "STL10Node", + "to create a STL10Node") + .def(py::init([](std::string dataset_dir, std::string usage, py::handle sampler) { + auto stl10 = std::make_shared(dataset_dir, usage, toSamplerObj(sampler), nullptr); + THROW_IF_ERROR(stl10->ValidateParams()); + return stl10; + })); + })); + PYBIND_REGISTER(TedliumNode, 2, ([](const py::module *m) { (void)py::class_>(*m, "TedliumNode", "to create a TedliumNode") diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/CMakeLists.txt b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/CMakeLists.txt index f253c5c5a28..06942d3d2a0 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/CMakeLists.txt +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/CMakeLists.txt @@ -30,6 +30,7 @@ set(DATASET_ENGINE_DATASETOPS_SOURCE_SRC_FILES random_data_op.cc sbu_op.cc speech_commands_op.cc + stl10_op.cc tedlium_op.cc text_file_op.cc usps_op.cc diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/stl10_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/stl10_op.cc new file mode 100644 index 00000000000..30e916253f6 --- /dev/null +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/stl10_op.cc @@ -0,0 +1,395 @@ +/** + * Copyright 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "minddata/dataset/engine/datasetops/source/stl10_op.h" + +#include +#include +#include +#include +#include + +#include "debug/common.h" +#include "minddata/dataset/core/config_manager.h" +#include "minddata/dataset/core/tensor_shape.h" +#include "minddata/dataset/engine/datasetops/source/sampler/sequential_sampler.h" +#include "minddata/dataset/engine/execution_tree.h" +#include "utils/file_utils.h" +#include "utils/ms_utils.h" + +namespace mindspore { +namespace dataset { +constexpr uint32_t kSTLImageRows = 96; +constexpr uint32_t kSTLImageCols = 96; +constexpr uint32_t kSTLImageChannel = 3; +constexpr uint32_t kSTLImageSize = kSTLImageRows * kSTLImageCols * kSTLImageChannel; + +STL10Op::STL10Op(const std::string &usage, int32_t num_workers, const std::string &folder_path, int32_t queue_size, + std::unique_ptr data_schema, std::shared_ptr sampler) + : MappableLeafOp(num_workers, queue_size, std::move(sampler)), + folder_path_(folder_path), + usage_(usage), + data_schema_(std::move(data_schema)), + image_path_({}), + label_path_({}) {} + +Status STL10Op::LoadTensorRow(row_id_type index, TensorRow *trow) { + RETURN_UNEXPECTED_IF_NULL(trow); + std::pair, int32_t> stl10_pair = stl10_image_label_pairs_[index]; + std::shared_ptr image, label; + // make a copy of cached tensor. + RETURN_IF_NOT_OK(Tensor::CreateFromTensor(stl10_pair.first, &image)); + RETURN_IF_NOT_OK(Tensor::CreateScalar(stl10_pair.second, &label)); + + (*trow) = TensorRow(index, {std::move(image), std::move(label)}); + trow->setPath({image_path_[index], label_path_[index]}); + + return Status::OK(); +} + +Status STL10Op::GetClassIds(std::map> *cls_ids) const { + if (cls_ids == nullptr || !cls_ids->empty() || stl10_image_label_pairs_.empty()) { + if (stl10_image_label_pairs_.empty()) { + RETURN_STATUS_UNEXPECTED("No image found in dataset. Check if image was generated successfully."); + } else { + RETURN_STATUS_UNEXPECTED( + "[Internal ERROR] Map for containing image-index pair is nullptr or has been set in other place, " + "it must be empty before using GetClassIds."); + } + } + for (size_t i = 0; i < stl10_image_label_pairs_.size(); ++i) { + (*cls_ids)[stl10_image_label_pairs_[i].second].push_back(i); + } + for (auto &pair : (*cls_ids)) { + pair.second.shrink_to_fit(); + } + return Status::OK(); +} + +void STL10Op::Print(std::ostream &out, bool show_all) const { + if (!show_all) { + // Call the super class for displaying any common 1-liner info. + ParallelOp::Print(out, show_all); + // Then show any custom derived-internal 1-liner info for this op. + out << "\n"; + } else { + // Call the super class for displaying any common detailed info. + ParallelOp::Print(out, show_all); + // Then show any custom derived-internal stuff. + out << "\nNumber of rows: " << num_rows_ << "\nSTL10 directory: " << folder_path_ << "\n\n"; + } +} + +Status STL10Op::WalkAllFiles() { + auto real_dataset_dir = FileUtils::GetRealPath(folder_path_.data()); + CHECK_FAIL_RETURN_UNEXPECTED(real_dataset_dir.has_value(), + "Invalid file, get real path failed, path: " + folder_path_); + Path root_dir(real_dataset_dir.value()); + + const Path train_data_file("train_X.bin"); + const Path train_label_file("train_y.bin"); + const Path test_data_file("test_X.bin"); + const Path test_label_file("test_y.bin"); + const Path unlabeled_data_file("unlabeled_X.bin"); + + bool use_train = false; + bool use_test = false; + bool use_unlabeled = false; + + if (usage_ == "train") { + use_train = true; + } else if (usage_ == "test") { + use_test = true; + } else if (usage_ == "unlabeled") { + use_unlabeled = true; + } else if (usage_ == "train+unlabeled") { + use_train = true; + use_unlabeled = true; + } else if (usage_ == "all") { + use_train = true; + use_test = true; + use_unlabeled = true; + } + + if (use_train) { + Path train_data_path = root_dir / train_data_file; + Path train_label_path = root_dir / train_label_file; + CHECK_FAIL_RETURN_UNEXPECTED( + train_data_path.Exists() && !train_data_path.IsDirectory(), + "Invalid file, failed to find STL10 " + usage_ + " data file: " + train_data_path.ToString()); + CHECK_FAIL_RETURN_UNEXPECTED( + train_label_path.Exists() && !train_label_path.IsDirectory(), + "Invalid file, failed to find STL10 " + usage_ + " label file: " + train_label_path.ToString()); + image_names_.push_back(train_data_path.ToString()); + label_names_.push_back(train_label_path.ToString()); + MS_LOG(INFO) << "STL10 operator found train data file " << train_data_path.ToString() << "."; + MS_LOG(INFO) << "STL10 operator found train label file " << train_label_path.ToString() << "."; + } + + if (use_test) { + Path test_data_path = root_dir / test_data_file; + Path test_label_path = root_dir / test_label_file; + CHECK_FAIL_RETURN_UNEXPECTED( + test_data_path.Exists() && !test_data_path.IsDirectory(), + "Invalid file, failed to find STL10 " + usage_ + " data file: " + test_data_path.ToString()); + CHECK_FAIL_RETURN_UNEXPECTED( + test_label_path.Exists() && !test_label_path.IsDirectory(), + "Invalid file, failed to find STL10 " + usage_ + " label file: " + test_label_path.ToString()); + image_names_.push_back(test_data_path.ToString()); + label_names_.push_back(test_label_path.ToString()); + MS_LOG(INFO) << "STL10 operator found test data file " << test_data_path.ToString() << "."; + MS_LOG(INFO) << "STL10 operator found test label file " << test_label_path.ToString() << "."; + } + + if (use_unlabeled) { + Path unlabeled_data_path = root_dir / unlabeled_data_file; + CHECK_FAIL_RETURN_UNEXPECTED( + unlabeled_data_path.Exists() && !unlabeled_data_path.IsDirectory(), + "Invalid file, failed to find STL10 " + usage_ + " data file: " + unlabeled_data_path.ToString()); + image_names_.push_back(unlabeled_data_path.ToString()); + MS_LOG(INFO) << "STL10 operator found unlabeled data file " << unlabeled_data_path.ToString() << "."; + } + + std::sort(image_names_.begin(), image_names_.end()); + std::sort(label_names_.begin(), label_names_.end()); + + return Status::OK(); +} + +Status STL10Op::ParseSTLData() { + // STL10 contains 5 files, *_X.bin are image files, *_y.bin are labels. + // training files contain 5k images and testing files contain 8K examples. + // unlabeled file contain 10k images and they DO NOT have labels (i.e. no "unlabeled_y.bin" file). + for (size_t i = 0; i < image_names_.size(); ++i) { + std::ifstream image_reader, label_reader; + if (image_names_[i].find("unlabeled") == std::string::npos) { + image_reader.open(image_names_[i], std::ios::binary | std::ios::ate); + label_reader.open(label_names_[i], std::ios::binary | std::ios::ate); + + Status s = ReadImageAndLabel(&image_reader, &label_reader, i); + // Close the readers. + image_reader.close(); + label_reader.close(); + + RETURN_IF_NOT_OK(s); + } else { // unlabeled data -> no labels. + image_reader.open(image_names_[i], std::ios::binary | std::ios::ate); + + Status s = ReadImageAndLabel(&image_reader, NULL, i); + // Close the readers. + image_reader.close(); + + RETURN_IF_NOT_OK(s); + } + } + stl10_image_label_pairs_.shrink_to_fit(); + num_rows_ = stl10_image_label_pairs_.size(); + if (num_rows_ == 0) { + RETURN_STATUS_UNEXPECTED( + "Invalid data, no valid data matching the dataset API STL10Dataset. Please check file path or dataset API."); + } + + return Status::OK(); +} + +Status STL10Op::ReadImageAndLabel(std::ifstream *image_reader, std::ifstream *label_reader, size_t index) { + RETURN_UNEXPECTED_IF_NULL(image_reader); + + Path image_path(image_names_[index]); + bool has_label_file = image_path.Basename().find("unlabeled") == std::string::npos; + + std::streamsize image_size = image_reader->tellg(); + + image_reader->seekg(0, std::ios::beg); + auto images_buf = std::make_unique(image_size); + auto labels_buf = std::make_unique(0); + + if (images_buf == nullptr) { + std::string err_msg = "Failed to allocate memory for STL10 buffer."; + MS_LOG(ERROR) << err_msg.c_str(); + RETURN_STATUS_UNEXPECTED(err_msg); + } + + uint64_t num_images = static_cast(image_size / kSTLImageSize); + (void)image_reader->read(images_buf.get(), image_size); + if (image_reader->fail()) { + RETURN_STATUS_UNEXPECTED("Invalid file, failed to read image: " + image_names_[index] + + ", size:" + std::to_string(kSTLImageSize * num_images)); + } + + if (has_label_file) { + RETURN_UNEXPECTED_IF_NULL(label_reader); + std::streamsize label_size = label_reader->tellg(); + if (static_cast(label_size) != num_images) { + RETURN_STATUS_UNEXPECTED("Invalid file, error in " + label_names_[index] + + ": the number of labels is not equal to the number of images in " + image_names_[index] + + "! Please check the file integrity!"); + } + + label_reader->seekg(0, std::ios::beg); + labels_buf = std::make_unique(label_size); + if (labels_buf == nullptr) { + std::string err_msg = "Failed to allocate memory for STL10 buffer."; + MS_LOG(ERROR) << err_msg.c_str(); + RETURN_STATUS_UNEXPECTED(err_msg); + } + + (void)label_reader->read(labels_buf.get(), label_size); + if (label_reader->fail()) { + RETURN_STATUS_UNEXPECTED("Invalid file, failed to read label:" + label_names_[index] + + ", size: " + std::to_string(num_images)); + } + } + + for (int64_t j = 0; j < num_images; ++j) { + int32_t label = (has_label_file ? labels_buf[j] - 1 : -1); + + std::shared_ptr image_tensor; + RETURN_IF_NOT_OK(Tensor::CreateEmpty(TensorShape({kSTLImageRows, kSTLImageCols, kSTLImageChannel}), + data_schema_->Column(0).Type(), &image_tensor)); + + auto iter = image_tensor->begin(); + uint64_t total_pix = kSTLImageRows * kSTLImageCols; + // stl10: Column major order. + for (uint64_t count = 0, pix = 0; count < total_pix; count++) { + if (count % kSTLImageRows == 0) { + pix = count / kSTLImageRows; + } + + for (int ch = 0; ch < kSTLImageChannel; ch++) { + *iter = images_buf[j * kSTLImageSize + ch * total_pix + pix]; + iter++; + } + pix += kSTLImageRows; + } + + stl10_image_label_pairs_.emplace_back(std::make_pair(image_tensor, label)); + image_path_.push_back(image_names_[index]); + label_path_.push_back(has_label_file ? label_names_[index] : "no label"); + } + + return Status::OK(); +} + +Status STL10Op::PrepareData() { + RETURN_IF_NOT_OK(this->WalkAllFiles()); + RETURN_IF_NOT_OK(this->ParseSTLData()); // Parse stl10 data and get num rows, blocking. + + return Status::OK(); +} + +Status STL10Op::CountTotalRows(const std::string &dir, const std::string &usage, int64_t *count) { + RETURN_UNEXPECTED_IF_NULL(count); + // the logic of counting the number of samples is copied from ParseSTLData(). + const int64_t num_samples = 0; + const int64_t start_index = 0; + auto sampler = std::make_shared(start_index, num_samples); + auto schema = std::make_unique(); + RETURN_IF_NOT_OK(schema->AddColumn(ColDescriptor("image", DataType(DataType::DE_UINT8), TensorImpl::kCv, 1))); + TensorShape scalar = TensorShape::CreateScalar(); + RETURN_IF_NOT_OK( + schema->AddColumn(ColDescriptor("label", DataType(DataType::DE_INT32), TensorImpl::kFlexible, 0, &scalar))); + std::shared_ptr cfg = GlobalContext::config_manager(); + int32_t num_workers = cfg->num_parallel_workers(); + int32_t op_connect_size = cfg->op_connector_size(); + auto op = std::make_shared(usage, num_workers, dir, op_connect_size, std::move(schema), std::move(sampler)); + + RETURN_IF_NOT_OK(op->WalkAllFiles()); + + bool use_train = false; + bool use_test = false; + bool use_unlabeled = false; + + if (usage == "train") { + use_train = true; + } else if (usage == "test") { + use_test = true; + } else if (usage == "unlabeled") { + use_unlabeled = true; + } else if (usage == "train+unlabeled") { + use_train = true; + use_unlabeled = true; + } else if (usage == "all") { + use_train = true; + use_test = true; + use_unlabeled = true; + } + + *count = 0; + uint64_t num_stl10_records = 0; + uint64_t total_image_size = 0; + + if (use_train) { + uint32_t index = (usage == "all" ? 1 : 0); + Path train_image_path(op->image_names_[index]); + CHECK_FAIL_RETURN_UNEXPECTED(train_image_path.Exists() && !train_image_path.IsDirectory(), + "Invalid file, failed to open stl10 file: " + train_image_path.ToString()); + + std::ifstream train_image_file(train_image_path.ToString(), std::ios::binary | std::ios::ate); + CHECK_FAIL_RETURN_UNEXPECTED(train_image_file.is_open(), + "Invalid file, failed to open stl10 file: " + train_image_path.ToString()); + total_image_size += static_cast(train_image_file.tellg()); + + train_image_file.close(); + } + + if (use_test) { + uint32_t index = 0; + Path test_image_path(op->image_names_[index]); + CHECK_FAIL_RETURN_UNEXPECTED(test_image_path.Exists() && !test_image_path.IsDirectory(), + "Invalid file, failed to open stl10 file: " + test_image_path.ToString()); + + std::ifstream test_image_file(test_image_path.ToString(), std::ios::binary | std::ios::ate); + CHECK_FAIL_RETURN_UNEXPECTED(test_image_file.is_open(), + "Invalid file, failed to open stl10 file: " + test_image_path.ToString()); + total_image_size += static_cast(test_image_file.tellg()); + + test_image_file.close(); + } + + if (use_unlabeled) { + uint32_t index = (usage == "unlabeled" ? 0 : (usage == "train+unlabeled" ? 1 : 2)); + Path unlabeled_image_path(op->image_names_[index]); + CHECK_FAIL_RETURN_UNEXPECTED(unlabeled_image_path.Exists() && !unlabeled_image_path.IsDirectory(), + "Invalid file, failed to open stl10 file: " + unlabeled_image_path.ToString()); + + std::ifstream unlabeled_image_file(unlabeled_image_path.ToString(), std::ios::binary | std::ios::ate); + CHECK_FAIL_RETURN_UNEXPECTED(unlabeled_image_file.is_open(), + "Invalid file, failed to open stl10 file: " + unlabeled_image_path.ToString()); + total_image_size += static_cast(unlabeled_image_file.tellg()); + + unlabeled_image_file.close(); + } + + num_stl10_records = static_cast(total_image_size / kSTLImageSize); + + *count = *count + num_stl10_records; + + return Status::OK(); +} + +Status STL10Op::ComputeColMap() { + // set the column Name map (base class field). + if (column_name_id_map_.empty()) { + for (uint32_t i = 0; i < data_schema_->NumColumns(); ++i) { + column_name_id_map_[data_schema_->Column(i).Name()] = i; + } + } else { + MS_LOG(WARNING) << "Column Name map is already set!"; + } + return Status::OK(); +} +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/stl10_op.h b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/stl10_op.h new file mode 100644 index 00000000000..6fc316b9fe1 --- /dev/null +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/stl10_op.h @@ -0,0 +1,118 @@ +/** + * Copyright 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_MINDDATA_DATASET_ENGINE_DATASETOPS_SOURCE_STL10_OP_H_ +#define MINDSPORE_CCSRC_MINDDATA_DATASET_ENGINE_DATASETOPS_SOURCE_STL10_OP_H_ + +#include +#include +#include +#include +#include +#include + +#include "minddata/dataset/core/tensor.h" +#include "minddata/dataset/engine/data_schema.h" +#include "minddata/dataset/engine/datasetops/parallel_op.h" +#include "minddata/dataset/engine/datasetops/source/mappable_leaf_op.h" +#include "minddata/dataset/engine/datasetops/source/sampler/sampler.h" +#include "minddata/dataset/util/path.h" +#include "minddata/dataset/util/queue.h" +#include "minddata/dataset/util/status.h" +#include "minddata/dataset/util/wait_post.h" + +namespace mindspore { +namespace dataset { +class STL10Op : public MappableLeafOp { + public: + // Constructor + // @param const std::string &usage - Usage of this dataset, can be 'train', 'test', 'unlabeled', "train+unlabeled" or + // "all" + // @param int32_t num_workers - number of workers reading images in parallel. + // @param std::string folder_path - dir directory of stl10. + // @param int32_t queue_size - connector queue size. + // @param std::unique_ptr data_schema - the schema of the stl10 dataset. + // @param td::unique_ptr sampler - sampler tells STL10Op what to read. + STL10Op(const std::string &usage, int32_t num_workers, const std::string &folder_path, int32_t queue_size, + std::unique_ptr data_schema, std::shared_ptr sampler); + + // Destructor + ~STL10Op() = default; + + // Method derived from RandomAccess Op, enable Sampler to get all ids for each class + // @param (std::map> * cls_ids - key label, val all ids for this class + // @return Status The status code returned + Status GetClassIds(std::map> *cls_ids) const override; + + // A print method typically used for debugging. + // @param out - The output stream to write output to. + // @param show_all - A bool to control if you want to show all info or just a summary. + void Print(std::ostream &out, bool show_all) const override; + + // Function to count the number of samples in the STL10 dataset. + // @param dir path to the STL10 directory. + // @param const std::string &usage - Usage of this dataset, can be 'train', 'test', 'unlabeled', "train+unlabeled" or + // "all" + // @param count output arg that will hold the minimum of the actual dataset size and numSamples. + // @return Status The status code returned. + static Status CountTotalRows(const std::string &dir, const std::string &usage, int64_t *count); + + // Op name getter. + // @return Name of the current Op. + std::string Name() const override { return "STL10Op"; } + + private: + // Read the needed files in the directory, save the file in image_names_ and label_names_. + // @return Status The status code returned. + Status WalkAllFiles(); + + // Read the specified number of images and labels from the file stream. + // @param std::ifstream *image_reader - image file stream. + // @param std::ifstream *label_reader - label file stream. + // @param int64_t index - number of image to read. + // @return Status The status code returned. + Status ReadImageAndLabel(std::ifstream *image_reader, std::ifstream *label_reader, size_t index); + + // Parse all stl10 dataset files + // @return Status The status code returned + Status ParseSTLData(); + + // Read all stl10 data files in the directory + // @return Status The status code returned. + Status PrepareData() override; + + // Private function for computing the assignment of the column name map. + // @return - Status. + Status ComputeColMap() override; + + // Load a tensor row according to a pair. + // @param uint64_t index - index need to load. + // @param TensorRow trow - image & label read into this tensor row. + // @return Status The status code returned. + Status LoadTensorRow(row_id_type index, TensorRow *trow) override; + + std::string folder_path_; // directory of image folder. + const std::string usage_; // can only be either "train" or "test" or "unlabeled" or "train+unlabeled" or "all". + std::unique_ptr data_schema_; + + std::vector, int32_t>> stl10_image_label_pairs_; + std::vector image_names_; + std::vector label_names_; + std::vector image_path_; + std::vector label_path_; +}; +} // namespace dataset +} // namespace mindspore +#endif // MINDSPORE_CCSRC_MINDDATA_DATASET_ENGINE_DATASETOPS_SOURCE_STL10_OP_H_ diff --git a/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/dataset_node.h b/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/dataset_node.h index c5937f18eec..be68d27e5c3 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/dataset_node.h +++ b/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/dataset_node.h @@ -104,6 +104,7 @@ constexpr char kQMnistNode[] = "QMnistDataset"; constexpr char kRandomNode[] = "RandomDataset"; constexpr char kSBUNode[] = "SBUDataset"; constexpr char kSpeechCommandsNode[] = "SpeechCommandsDataset"; +constexpr char kSTL10Node[] = "STL10Dataset"; constexpr char kTedliumNode[] = "TedliumDataset"; constexpr char kTextFileNode[] = "TextFileDataset"; constexpr char kTFRecordNode[] = "TFRecordDataset"; diff --git a/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/CMakeLists.txt b/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/CMakeLists.txt index 8e49296ba1c..1d321f9b115 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/CMakeLists.txt +++ b/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/CMakeLists.txt @@ -30,6 +30,7 @@ set(DATASET_ENGINE_IR_DATASETOPS_SOURCE_SRC_FILES random_node.cc sbu_node.cc speech_commands_node.cc + stl10_node.cc tedlium_node.cc text_file_node.cc tf_record_node.cc diff --git a/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/stl10_node.cc b/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/stl10_node.cc new file mode 100644 index 00000000000..9eaa374d693 --- /dev/null +++ b/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/stl10_node.cc @@ -0,0 +1,121 @@ +/** + * Copyright 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "minddata/dataset/engine/ir/datasetops/source/stl10_node.h" + +#include +#include +#include +#include + +#include "minddata/dataset/engine/datasetops/source/stl10_op.h" +#include "minddata/dataset/util/status.h" + +namespace mindspore { +namespace dataset { +STL10Node::STL10Node(const std::string &dataset_dir, const std::string &usage, std::shared_ptr sampler, + std::shared_ptr cache) + : MappableSourceNode(std::move(cache)), dataset_dir_(dataset_dir), usage_(usage), sampler_(sampler) {} + +std::shared_ptr STL10Node::Copy() { + std::shared_ptr sampler = (sampler_ == nullptr) ? nullptr : sampler_->SamplerCopy(); + auto node = std::make_shared(dataset_dir_, usage_, sampler, cache_); + return node; +} + +void STL10Node::Print(std::ostream &out) const { + out << (Name() + "(cache:" + ((cache_ != nullptr) ? "true" : "false") + ")"); +} + +Status STL10Node::ValidateParams() { + RETURN_IF_NOT_OK(DatasetNode::ValidateParams()); + RETURN_IF_NOT_OK(ValidateDatasetDirParam("STL10Node", dataset_dir_)); + + RETURN_IF_NOT_OK(ValidateDatasetSampler("STL10Node", sampler_)); + + RETURN_IF_NOT_OK(ValidateStringValue("STL10Node", usage_, {"train", "test", "unlabeled", "train+unlabeled", "all"})); + + return Status::OK(); +} + +// Function to build STL10Op for STL10 +Status STL10Node::Build(std::vector> *const node_ops) { + // Do internal Schema generation. + auto schema = std::make_unique(); + RETURN_IF_NOT_OK(schema->AddColumn(ColDescriptor("image", DataType(DataType::DE_UINT8), TensorImpl::kCv, 1))); + TensorShape scalar = TensorShape::CreateScalar(); + RETURN_IF_NOT_OK( + schema->AddColumn(ColDescriptor("label", DataType(DataType::DE_INT32), TensorImpl::kFlexible, 0, &scalar))); + std::shared_ptr sampler_rt = nullptr; + RETURN_IF_NOT_OK(sampler_->SamplerBuild(&sampler_rt)); + + auto stl10_op = std::make_shared(usage_, num_workers_, dataset_dir_, connector_que_size_, std::move(schema), + std::move(sampler_rt)); + stl10_op->SetTotalRepeats(GetTotalRepeats()); + stl10_op->SetNumRepeatsPerEpoch(GetNumRepeatsPerEpoch()); + node_ops->push_back(stl10_op); + + return Status::OK(); +} + +// Get the shard id of node +Status STL10Node::GetShardId(int32_t *shard_id) { + *shard_id = sampler_->ShardId(); + + return Status::OK(); +} + +// Get Dataset size +Status STL10Node::GetDatasetSize(const std::shared_ptr &size_getter, bool estimate, + int64_t *dataset_size) { + if (dataset_size_ > 0) { + *dataset_size = dataset_size_; + return Status::OK(); + } + + int64_t num_rows, sample_size; + RETURN_IF_NOT_OK(STL10Op::CountTotalRows(dataset_dir_, usage_, &num_rows)); + std::shared_ptr sampler_rt = nullptr; + RETURN_IF_NOT_OK(sampler_->SamplerBuild(&sampler_rt)); + + sample_size = sampler_rt->CalculateNumSamples(num_rows); + if (sample_size == -1) { + RETURN_IF_NOT_OK(size_getter->DryRun(shared_from_this(), &sample_size)); + } + + *dataset_size = sample_size; + dataset_size_ = *dataset_size; + + return Status::OK(); +} + +Status STL10Node::to_json(nlohmann::json *out_json) { + nlohmann::json args, sampler_args; + RETURN_IF_NOT_OK(sampler_->to_json(&sampler_args)); + args["sampler"] = sampler_args; + args["num_parallel_workers"] = num_workers_; + args["dataset_dir"] = dataset_dir_; + args["usage"] = usage_; + if (cache_ != nullptr) { + nlohmann::json cache_args; + RETURN_IF_NOT_OK(cache_->to_json(&cache_args)); + args["cache"] = cache_args; + } + *out_json = args; + return Status::OK(); +} +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/stl10_node.h b/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/stl10_node.h new file mode 100644 index 00000000000..76fde409d20 --- /dev/null +++ b/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/stl10_node.h @@ -0,0 +1,96 @@ +/** + * Copyright 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_MINDDATA_DATASET_ENGINE_IR_DATASETOPS_SOURCE_STL10_NODE_H_ +#define MINDSPORE_CCSRC_MINDDATA_DATASET_ENGINE_IR_DATASETOPS_SOURCE_STL10_NODE_H_ + +#include +#include +#include + +#include "minddata/dataset/engine/ir/datasetops/dataset_node.h" + +namespace mindspore { +namespace dataset { +class STL10Node : public MappableSourceNode { + public: + /// \brief Constructor + STL10Node(const std::string &dataset_dir, const std::string &usage, std::shared_ptr sampler, + std::shared_ptr cache); + + /// \brief Destructor + ~STL10Node() = default; + + /// \brief Node name getter + /// \return Name of the current node + std::string Name() const override { return kSTL10Node; } + + /// \brief Print the description + /// \param out - The output stream to write output to + void Print(std::ostream &out) const override; + + /// \brief Copy the node to a new object + /// \return A shared pointer to the new copy + std::shared_ptr Copy() override; + + /// \brief a base class override function to create the required runtime dataset op objects for this class + /// \param node_ops - A vector containing shared pointer to the Dataset Ops that this object will create + /// \return Status Status::OK() if build successfully + Status Build(std::vector> *const node_ops) override; + + /// \brief Parameters validation + /// \return Status Status::OK() if all the parameters are valid + Status ValidateParams() override; + + /// \brief Get the shard id of node + /// \param[in] shard_id - The shard id. + /// \return Status Status::OK() if get shard id successfully + Status GetShardId(int32_t *shard_id) override; + + /// \brief Base-class override for GetDatasetSize + /// \param[in] size_getter Shared pointer to DatasetSizeGetter + /// \param[in] estimate This is only supported by some of the ops and it's used to speed up the process of getting + /// dataset size at the expense of accuracy. + /// \param[out] dataset_size the size of the dataset + /// \return Status of the function + Status GetDatasetSize(const std::shared_ptr &size_getter, bool estimate, + int64_t *dataset_size) override; + + /// \brief Getter functions + const std::string &DatasetDir() const { return dataset_dir_; } + const std::string &Usage() const { return usage_; } + + /// \brief Get the arguments of node + /// \param[out] out_json JSON string of all attributes + /// \return Status of the function + Status to_json(nlohmann::json *out_json) override; + + /// \brief Sampler getter + /// \return SamplerObj of the current node + std::shared_ptr Sampler() override { return sampler_; } + + /// \brief Sampler setter + /// \param[in] sampler The Sampler setter of the current node + void SetSampler(std::shared_ptr sampler) override { sampler_ = sampler; } + + private: + std::string dataset_dir_; + std::string usage_; + std::shared_ptr sampler_; +}; +} // namespace dataset +} // namespace mindspore +#endif // MINDSPORE_CCSRC_MINDDATA_DATASET_ENGINE_IR_DATASETOPS_SOURCE_STL10_NODE_H_ diff --git a/mindspore/ccsrc/minddata/dataset/include/dataset/datasets.h b/mindspore/ccsrc/minddata/dataset/include/dataset/datasets.h index deeee42a872..05a19b88635 100644 --- a/mindspore/ccsrc/minddata/dataset/include/dataset/datasets.h +++ b/mindspore/ccsrc/minddata/dataset/include/dataset/datasets.h @@ -3691,6 +3691,83 @@ SpeechCommands(const std::string &dataset_dir, const std::string &usage, const s return std::make_shared(StringToChar(dataset_dir), StringToChar(usage), sampler, cache); } +/// \class STL10Dataset +/// \brief A source dataset that reads and parses STL10 dataset. +class MS_API STL10Dataset : public Dataset { + public: + /// \brief Constructor of STL10Dataset. + /// \param[in] dataset_dir Path to the root directory that contains the dataset. + /// \param[in] usage Part of dataset of STL10, can be "train", "test", "unlabeled", "train+unlabeled" + /// or "all". + /// \param[in] sampler Shared pointer to a sampler object used to choose samples from the dataset. If sampler is not + /// given, a `RandomSampler` will be used to randomly iterate the entire dataset. + /// \param[in] cache Tensor cache to use. + STL10Dataset(const std::vector &dataset_dir, const std::vector &usage, + const std::shared_ptr &sampler, const std::shared_ptr &cache); + + /// \brief Constructor of STL10Dataset. + /// \param[in] dataset_dir Path to the root directory that contains the dataset. + /// \param[in] usage Part of dataset of STL10, can be "train", "test", "unlabeled", "train+unlabeled" + /// or "all". + /// \param[in] sampler Raw pointer to a sampler object used to choose samples from the dataset. + /// \param[in] cache Tensor cache to use. + STL10Dataset(const std::vector &dataset_dir, const std::vector &usage, const Sampler *sampler, + const std::shared_ptr &cache); + + /// \brief Constructor of STL10Dataset. + /// \param[in] dataset_dir Path to the root directory that contains the dataset. + /// \param[in] usage Part of dataset of STL10, can be "train", "test", "unlabeled", "train+unlabeled" + /// or "all". + /// \param[in] sampler Sampler object used to choose samples from the dataset. + /// \param[in] cache Tensor cache to use. + STL10Dataset(const std::vector &dataset_dir, const std::vector &usage, + const std::reference_wrapper sampler, const std::shared_ptr &cache); + + /// \brief Destructor of STL10Dataset. + ~STL10Dataset() = default; +}; + +/// \brief Function to create a STL10 Dataset. +/// \notes The generated dataset has two columns ["image", "label"]. +/// \param[in] dataset_dir Path to the root directory that contains the dataset. +/// \param[in] usage Usage of STL10, can be "train", "test", "unlabeled", "train+unlabeled" or "all" (default = "all"). +/// \param[in] sampler Shared pointer to a sampler object used to choose samples from the dataset. If sampler is not +/// given, a `RandomSampler` will be used to randomly iterate the entire dataset (default = RandomSampler()). +/// \param[in] cache Tensor cache to use. (default=nullptr, which means no cache is used). +/// \return Shared pointer to the current Dataset. +inline std::shared_ptr MS_API +STL10(const std::string &dataset_dir, const std::string &usage = "all", + const std::shared_ptr &sampler = std::make_shared(), + const std::shared_ptr &cache = nullptr) { + return std::make_shared(StringToChar(dataset_dir), StringToChar(usage), sampler, cache); +} + +/// \brief Function to create a STL10 Dataset. +/// \notes The generated dataset has two columns ["image", "label"]. +/// \param[in] dataset_dir Path to the root directory that contains the dataset. +/// \param[in] usage Usage of STL10, can be "train", "test", "unlabeled" or "train+unlabeled" or "all" +/// \param[in] sampler Raw pointer to a sampler object used to choose samples from the dataset. +/// \param[in] cache Tensor cache to use. (default=nullptr, which means no cache is used). +/// \return Shared pointer to the current Dataset. +inline std::shared_ptr MS_API STL10(const std::string &dataset_dir, const std::string &usage, + const Sampler *sampler, + const std::shared_ptr &cache = nullptr) { + return std::make_shared(StringToChar(dataset_dir), StringToChar(usage), sampler, cache); +} + +/// \brief Function to create a STL10 Dataset. +/// \notes The generated dataset has two columns ["image", "label"]. +/// \param[in] dataset_dir Path to the root directory that contains the dataset. +/// \param[in] usage Usage of STL10, can be "train", "test", "unlabeled", "train+unlabeled" or "all" +/// \param[in] sampler Sampler object used to choose samples from the dataset. +/// \param[in] cache Tensor cache to use. (default=nullptr, which means no cache is used). +/// \return Shared pointer to the current Dataset. +inline std::shared_ptr MS_API STL10(const std::string &dataset_dir, const std::string &usage, + const std::reference_wrapper sampler, + const std::shared_ptr &cache = nullptr) { + return std::make_shared(StringToChar(dataset_dir), StringToChar(usage), sampler, cache); +} + /// \class TedliumDataset /// \brief A source dataset for reading and parsing tedlium dataset. class MS_API TedliumDataset : public Dataset { diff --git a/mindspore/ccsrc/minddata/dataset/include/dataset/samplers.h b/mindspore/ccsrc/minddata/dataset/include/dataset/samplers.h index 82de1ecb0e8..e529febcc89 100644 --- a/mindspore/ccsrc/minddata/dataset/include/dataset/samplers.h +++ b/mindspore/ccsrc/minddata/dataset/include/dataset/samplers.h @@ -57,6 +57,7 @@ class MS_API Sampler : std::enable_shared_from_this { friend class RandomDataDataset; friend class SBUDataset; friend class SpeechCommandsDataset; + friend class STL10Dataset; friend class TedliumDataset; friend class TextFileDataset; friend class TFRecordDataset; diff --git a/mindspore/dataset/engine/datasets.py b/mindspore/dataset/engine/datasets.py index e9f2efbaa4c..e4a7a3b18a1 100644 --- a/mindspore/dataset/engine/datasets.py +++ b/mindspore/dataset/engine/datasets.py @@ -70,7 +70,8 @@ from .validators import check_batch, check_shuffle, check_map, check_filter, che check_sb_dataset, check_flowers102dataset, check_cityscapes_dataset, check_usps_dataset, check_div2k_dataset, \ check_sbu_dataset, check_qmnist_dataset, check_emnist_dataset, check_fake_image_dataset, check_places365_dataset, \ check_photo_tour_dataset, check_ag_news_dataset, check_dbpedia_dataset, check_lj_speech_dataset, \ - check_yes_no_dataset, check_speech_commands_dataset, check_tedlium_dataset, check_svhn_dataset + check_yes_no_dataset, check_speech_commands_dataset, check_tedlium_dataset, check_svhn_dataset, \ + check_stl10_dataset from ..core.config import get_callback_timeout, _init_device_info, get_enable_shared_mem, get_num_parallel_workers, \ get_prefetch_size, get_auto_offload from ..core.datatypes import mstype_to_detype, mstypelist_to_detypelist @@ -9108,3 +9109,138 @@ class SVHNDataset(GeneratorDataset): super().__init__(dataset, column_names=dataset.column_names, num_samples=num_samples, num_parallel_workers=num_parallel_workers, shuffle=shuffle, sampler=sampler, num_shards=num_shards, shard_id=shard_id) + + +class STL10Dataset(MappableDataset): + """ + A source dataset for reading and parsing STL10 dataset. + + The generated dataset has two columns: :py:obj:`[image, label]`. + The tensor of column :py:obj:`image` is of the uint8 type. + The tensor of column :py:obj:`label` is of a scalar of int32 type. + + Args: + dataset_dir (str): Path to the root directory that contains the dataset. + usage (str, optional): Usage of this dataset, can be "train", "test", + "unlabeled", "train+unlabeled" or "all" . "train" will read from 5,000 + train samples, "test" will read from 8,000 test samples, + "unlabeled" will read from all 100,000 samples, and "train+unlabeled" + will read from 105000 samples, "all" will read all the samples + (default=None, all samples). + num_samples (int, optional): The number of images to be included in the dataset. + (default=None, all images). + num_parallel_workers (int, optional): Number of workers to read the data + (default=None, number set in the config). + shuffle (bool, optional): Whether to perform shuffle on the dataset (default=None, expected + order behavior shown in the table). + sampler (Sampler, optional): Object used to choose samples from the + dataset (default=None, expected order behavior shown in the table). + num_shards (int, optional): Number of shards that the dataset will be divided + into (default=None). When this argument is specified, 'num_samples' reflects + the max sample number of per shard. + shard_id (int, optional): The shard ID within num_shards (default=None). This + argument can only be specified when num_shards is also specified. + cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. + (default=None, which means no cache is used). + + Raises: + RuntimeError: If dataset_dir is not valid or does not exist or does not contain data files. + RuntimeError: If num_parallel_workers exceeds the max thread numbers. + RuntimeError: If sampler and shuffle are specified at the same time. + RuntimeError: If sampler and sharding are specified at the same time. + RuntimeError: If num_shards is specified but shard_id is None. + RuntimeError: If shard_id is specified but num_shards is None. + ValueError: If usage is invalid. + ValueError: If shard_id is invalid (< 0 or >= num_shards). + + Note: + - This dataset can take in a sampler. 'sampler' and 'shuffle' are mutually exclusive. + The table below shows what input arguments are allowed and their expected behavior. + + .. list-table:: Expected Order Behavior of Using 'sampler' and 'shuffle' + :widths: 25 25 50 + :header-rows: 1 + + * - Parameter 'sampler' + - Parameter 'shuffle' + - Expected Order Behavior + * - None + - None + - random order + * - None + - True + - random order + * - None + - False + - sequential order + * - Sampler object + - None + - order defined by sampler + * - Sampler object + - True + - not allowed + * - Sampler object + - False + - not allowed + + Examples: + >>> stl10_dataset_dir = "/path/to/stl10_dataset_directory" + >>> + >>> # 1) Get all samples from STL10 dataset in sequence + >>> dataset = ds.STL10Dataset(dataset_dir=stl10_dataset_dir, shuffle=False) + >>> + >>> # 2) Randomly select 350 samples from STL10 dataset + >>> dataset = ds.STL10Dataset(dataset_dir=stl10_dataset_dir, num_samples=350, shuffle=True) + >>> + >>> # 3) Get samples from STL10 dataset for shard 0 in a 2-way distributed training + >>> dataset = ds.STL10Dataset(dataset_dir=stl10_dataset_dir, num_shards=2, shard_id=0) + + About STL10 dataset: + + STL10 dataset consists of 10 classes: airplane, bird, car, cat, deer, dog, horse, monkey, ship, truck. + STL10 is is inspired by the CIFAR-10 dataset. + Images are 96x96 pixels, color. + 500 training images, 800 test images per class and 100000 unlabeled images. + Labels are 0-indexed, and unlabeled images have -1 as their labels. + + Here is the original STL10 dataset structure. + You can unzip the dataset files into this directory structure and read by MindSpore's API. + + .. code-block:: + . + └── stl10_dataset_dir + ├── train_X.bin + ├── train_y.bin + ├── test_X.bin + ├── test_y.bin + └── unlabeled_X.bin + + Citation of STL10 dataset. + + .. code-block:: + + @techreport{Coates10, + author = {Adam Coates}, + title = {Learning multiple layers of features from tiny images}, + year = {20010}, + howpublished = {https://cs.stanford.edu/~acoates/stl10/}, + description = {The STL-10 dataset consists of 96x96 RGB images in 10 classes, + with 500 training images and 800 testing images per class. + There are 5000 training images and 8000 test images. + It also has 100000 unlabeled images for unsupervised learning. + These examples are extracted from a similar but broader distribution of images. + } + } + """ + + @check_stl10_dataset + def __init__(self, dataset_dir, usage=None, num_samples=None, num_parallel_workers=None, shuffle=None, + sampler=None, num_shards=None, shard_id=None, cache=None): + super().__init__(num_parallel_workers=num_parallel_workers, sampler=sampler, num_samples=num_samples, + shuffle=shuffle, num_shards=num_shards, shard_id=shard_id, cache=cache) + + self.dataset_dir = dataset_dir + self.usage = replace_none(usage, "all") + + def parse(self, children=None): + return cde.STL10Node(self.dataset_dir, self.usage, self.sampler) diff --git a/mindspore/dataset/engine/validators.py b/mindspore/dataset/engine/validators.py index 99ac5a2bb64..e256a876fa7 100644 --- a/mindspore/dataset/engine/validators.py +++ b/mindspore/dataset/engine/validators.py @@ -1930,3 +1930,49 @@ def check_svhn_dataset(method): return method(self, *args, **kwargs) return new_method + + +def check_stl10_dataset(method): + """A wrapper that wraps a parameter checker around the original Dataset(STL10Dataset).""" + + @wraps(method) + def new_method(self, *args, **kwargs): + _, param_dict = parse_user_args(method, *args, **kwargs) + + nreq_param_int = ['num_samples', 'num_parallel_workers', 'num_shards', 'shard_id'] + nreq_param_bool = ['shuffle'] + + dataset_dir = param_dict.get('dataset_dir') + check_dir(dataset_dir) + + usage = param_dict.get('usage') + if usage is not None: + check_valid_str(usage, ["train", "test", "unlabeled", "train+unlabeled", "all"], "usage") + if usage == "all": + for _usage in ["train", "test", "unlabeled"]: + check_file(os.path.join(dataset_dir, _usage + "_X.bin")) + if _usage == "unlabeled": + continue + else: + check_file(os.path.join(dataset_dir, _usage + "_y.bin")) + elif usage == "train+unlabeled": + check_file(os.path.join(dataset_dir, "train_X.bin")) + check_file(os.path.join(dataset_dir, "train_y.bin")) + check_file(os.path.join(dataset_dir, "unlabeled_X.bin")) + elif usage == "unlabeled": + check_file(os.path.join(dataset_dir, "unlabeled_X.bin")) + else: + check_file(os.path.join(dataset_dir, usage + "_X.bin")) + check_file(os.path.join(dataset_dir, usage + "_y.bin")) + + validate_dataset_param_value(nreq_param_int, param_dict, int) + validate_dataset_param_value(nreq_param_bool, param_dict, bool) + + check_sampler_shuffle_shard_options(param_dict) + + cache = param_dict.get('cache') + check_cache_option(cache) + + return method(self, *args, **kwargs) + + return new_method diff --git a/tests/ut/cpp/dataset/CMakeLists.txt b/tests/ut/cpp/dataset/CMakeLists.txt index 296d81fb21a..341573ceba9 100644 --- a/tests/ut/cpp/dataset/CMakeLists.txt +++ b/tests/ut/cpp/dataset/CMakeLists.txt @@ -41,6 +41,7 @@ SET(DE_UT_SRCS c_api_dataset_save.cc c_api_dataset_sbu_test.cc c_api_dataset_speech_commands_test.cc + c_api_dataset_stl10_test.cc c_api_dataset_tedlium_test.cc c_api_dataset_textfile_test.cc c_api_dataset_tfrecord_test.cc diff --git a/tests/ut/cpp/dataset/c_api_dataset_stl10_test.cc b/tests/ut/cpp/dataset/c_api_dataset_stl10_test.cc new file mode 100644 index 00000000000..c8d7b11c2f7 --- /dev/null +++ b/tests/ut/cpp/dataset/c_api_dataset_stl10_test.cc @@ -0,0 +1,431 @@ +/** + * Copyright 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "common/common.h" + +#include "minddata/dataset/include/dataset/datasets.h" + +using namespace mindspore::dataset; +using mindspore::dataset::DataType; +using mindspore::dataset::Tensor; +using mindspore::dataset::TensorShape; + +class MindDataTestPipeline : public UT::DatasetOpTesting { + protected: +}; + +/// Feature: STL10TrainDataset. +/// Description: test basic usage of STL10TrainDataset. +/// Expectation: get correct number of data. +TEST_F(MindDataTestPipeline, TestSTL10TrainDataset) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSTL10TrainDataset."; + + // Create a STL10 Dataset + std::string folder_path = datasets_root_path_ + "/testSTL10Data/"; + std::shared_ptr ds = STL10(folder_path, "train", std::make_shared(false, 1)); + EXPECT_NE(ds, nullptr); + + // Create an iterator over the result of the above dataset + // This will trigger the creation of the Execution Tree and launch it. + std::shared_ptr iter = ds->CreateIterator(); + EXPECT_NE(iter, nullptr); + + // Iterate the dataset and get each row + std::unordered_map row; + ASSERT_OK(iter->GetNextRow(&row)); + + EXPECT_NE(row.find("image"), row.end()); + EXPECT_NE(row.find("label"), row.end()); + + uint64_t i = 0; + while (row.size() != 0) { + i++; + auto image = row["image"]; + MS_LOG(INFO) << "Tensor image shape: " << image.Shape(); + ASSERT_OK(iter->GetNextRow(&row)); + } + + EXPECT_EQ(i, 1); + + // Manually terminate the pipeline + iter->Stop(); +} + +/// Feature: STL10TestDataset. +/// Description: test basic usage of STL10TestDataset. +/// Expectation: get correct number of data. +TEST_F(MindDataTestPipeline, TestSTL10TestDataset) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSTL10TestDataset."; + + // Create a STL10 Dataset + std::string folder_path = datasets_root_path_ + "/testSTL10Data/"; + std::shared_ptr ds = STL10(folder_path, "test", std::make_shared(false, 1)); + EXPECT_NE(ds, nullptr); + + // Create an iterator over the result of the above dataset + // This will trigger the creation of the Execution Tree and launch it. + std::shared_ptr iter = ds->CreateIterator(); + EXPECT_NE(iter, nullptr); + + // Iterate the dataset and get each row + std::unordered_map row; + ASSERT_OK(iter->GetNextRow(&row)); + + EXPECT_NE(row.find("image"), row.end()); + EXPECT_NE(row.find("label"), row.end()); + + uint64_t i = 0; + while (row.size() != 0) { + i++; + auto image = row["image"]; + MS_LOG(INFO) << "Tensor image shape: " << image.Shape(); + ASSERT_OK(iter->GetNextRow(&row)); + } + + EXPECT_EQ(i, 1); + + // Manually terminate the pipeline + iter->Stop(); +} + +/// Feature: STL10UnlabeledDataset. +/// Description: test basic usage of STL10UnlabeledDataset. +/// Expectation: get correct number of data. +TEST_F(MindDataTestPipeline, TestSTL10UnlabeledDataset) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSTL10UnlabeledDataset."; + + // Create a STL10 Dataset + std::string folder_path = datasets_root_path_ + "/testSTL10Data/"; + std::shared_ptr ds = STL10(folder_path, "unlabeled", std::make_shared(false, 1)); + EXPECT_NE(ds, nullptr); + + // Create an iterator over the result of the above dataset + // This will trigger the creation of the Execution Tree and launch it. + std::shared_ptr iter = ds->CreateIterator(); + EXPECT_NE(iter, nullptr); + + // Iterate the dataset and get each row + std::unordered_map row; + ASSERT_OK(iter->GetNextRow(&row)); + + EXPECT_NE(row.find("image"), row.end()); + EXPECT_NE(row.find("label"), row.end()); + + uint64_t i = 0; + while (row.size() != 0) { + i++; + auto image = row["image"]; + MS_LOG(INFO) << "Tensor image shape: " << image.Shape(); + ASSERT_OK(iter->GetNextRow(&row)); + } + + EXPECT_EQ(i, 1); + + // Manually terminate the pipeline + iter->Stop(); +} + +/// Feature: STL10TrainUnlabeledDataset. +/// Description: test basic usage of STL10TrainUnlabeledDataset. +/// Expectation: get correct number of data. +TEST_F(MindDataTestPipeline, TestSTL10TrainUnlabeledDataset) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSTL10TrainUnlabeledDataset."; + + // Create a STL10 Dataset + std::string folder_path = datasets_root_path_ + "/testSTL10Data/"; + std::shared_ptr ds = STL10(folder_path, "train+unlabeled", std::make_shared(false, 2)); + EXPECT_NE(ds, nullptr); + + // Create an iterator over the result of the above dataset + // This will trigger the creation of the Execution Tree and launch it. + std::shared_ptr iter = ds->CreateIterator(); + EXPECT_NE(iter, nullptr); + + // Iterate the dataset and get each row + std::unordered_map row; + ASSERT_OK(iter->GetNextRow(&row)); + + EXPECT_NE(row.find("image"), row.end()); + EXPECT_NE(row.find("label"), row.end()); + + uint64_t i = 0; + while (row.size() != 0) { + i++; + auto image = row["image"]; + MS_LOG(INFO) << "Tensor image shape: " << image.Shape(); + ASSERT_OK(iter->GetNextRow(&row)); + } + + EXPECT_EQ(i, 2); + + // Manually terminate the pipeline + iter->Stop(); +} + +/// Feature: STL10AllDataset. +/// Description: test basic usage of STL10AllDataset. +/// Expectation: get correct number of data. +TEST_F(MindDataTestPipeline, TestSTL10AllDataset) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSTL10AllDataset."; + + // Create a STL10 Dataset + std::string folder_path = datasets_root_path_ + "/testSTL10Data/"; + std::shared_ptr ds = STL10(folder_path, "all", std::make_shared(false, 2)); + EXPECT_NE(ds, nullptr); + + // Create an iterator over the result of the above dataset + // This will trigger the creation of the Execution Tree and launch it. + std::shared_ptr iter = ds->CreateIterator(); + EXPECT_NE(iter, nullptr); + + // Iterate the dataset and get each row + std::unordered_map row; + ASSERT_OK(iter->GetNextRow(&row)); + + EXPECT_NE(row.find("image"), row.end()); + EXPECT_NE(row.find("label"), row.end()); + + uint64_t i = 0; + while (row.size() != 0) { + i++; + auto image = row["image"]; + MS_LOG(INFO) << "Tensor image shape: " << image.Shape(); + ASSERT_OK(iter->GetNextRow(&row)); + } + + EXPECT_EQ(i, 2); + + // Manually terminate the pipeline + iter->Stop(); +} + +/// Feature: STL10TrainDatasetWithPipeline. +/// Description: test usage of STL10TrainDataset with pipeline. +/// Expectation: get correct number of data. +TEST_F(MindDataTestPipeline, TestSTL10TrainDatasetWithPipeline) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSTL10TrainDatasetWithPipeline."; + + // Create two STL10 Dataset + std::string folder_path = datasets_root_path_ + "/testSTL10Data/"; + std::shared_ptr ds1 = STL10(folder_path, "train", std::make_shared(false, 2)); + std::shared_ptr ds2 = STL10(folder_path, "train", std::make_shared(false, 2)); + EXPECT_NE(ds1, nullptr); + EXPECT_NE(ds2, nullptr); + + // Create two Repeat operation on ds + int32_t repeat_num = 1; + ds1 = ds1->Repeat(repeat_num); + EXPECT_NE(ds1, nullptr); + repeat_num = 1; + ds2 = ds2->Repeat(repeat_num); + EXPECT_NE(ds2, nullptr); + + // Create two Project operation on ds + std::vector column_project = {"image", "label"}; + ds1 = ds1->Project(column_project); + EXPECT_NE(ds1, nullptr); + ds2 = ds2->Project(column_project); + EXPECT_NE(ds2, nullptr); + + // Create a Concat operation on the ds + ds1 = ds1->Concat({ds2}); + EXPECT_NE(ds1, nullptr); + + // Create an iterator over the result of the above dataset + // This will trigger the creation of the Execution Tree and launch it. + std::shared_ptr iter = ds1->CreateIterator(); + EXPECT_NE(iter, nullptr); + + // Iterate the dataset and get each row + std::unordered_map row; + ASSERT_OK(iter->GetNextRow(&row)); + + EXPECT_NE(row.find("image"), row.end()); + EXPECT_NE(row.find("label"), row.end()); + + uint64_t i = 0; + while (row.size() != 0) { + i++; + auto image = row["image"]; + MS_LOG(INFO) << "Tensor image shape: " << image.Shape(); + ASSERT_OK(iter->GetNextRow(&row)); + } + + EXPECT_EQ(i, 2); + + // Manually terminate the pipeline + iter->Stop(); +} + +/// Feature: STL10GetTrainDatasetSize. +/// Description: test usage of STL10GetTrainDatasetSize with pipeline. +/// Expectation: get correct number of data. +TEST_F(MindDataTestPipeline, TestSTL10GetTrainDatasetSize) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSTL10GetTrainDatasetSize."; + + // Create a STL10 Dataset + std::string folder_path = datasets_root_path_ + "/testSTL10Data/"; + std::shared_ptr ds = STL10(folder_path, "train"); + EXPECT_NE(ds, nullptr); + + EXPECT_EQ(ds->GetDatasetSize(), 1); +} + +/// Feature: STL10GetTestDatasetSize. +/// Description: test usage of STL10GetTestDatasetSize with pipeline. +/// Expectation: get correct number of data. +TEST_F(MindDataTestPipeline, TestSTL10GetTestDatasetSize) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSTL10GetTestDatasetSize."; + + // Create a STL10 Dataset + std::string folder_path = datasets_root_path_ + "/testSTL10Data/"; + std::shared_ptr ds = STL10(folder_path, "test"); + EXPECT_NE(ds, nullptr); + + EXPECT_EQ(ds->GetDatasetSize(), 1); +} + +/// Feature: STL10GetUnlabeledDatasetSize. +/// Description: test usage of STL10GetUnlabeledDatasetSize with pipeline. +/// Expectation: get correct number of data. +TEST_F(MindDataTestPipeline, TestSTL10GetUnlabeledDatasetSize) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSTL10GetUnlabeledDatasetSize."; + + // Create a STL10 Dataset + std::string folder_path = datasets_root_path_ + "/testSTL10Data/"; + std::shared_ptr ds = STL10(folder_path, "unlabeled"); + EXPECT_NE(ds, nullptr); + + EXPECT_EQ(ds->GetDatasetSize(), 1); +} + +/// Feature: STL10GetTrainUnlabeledDatasetSize. +/// Description: test usage of STL10GetTrainUnlabeledDatasetSize with pipeline. +/// Expectation: get correct number of data. +TEST_F(MindDataTestPipeline, TestSTL10GetTrainUnlabeledDatasetSize) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSTL10GetTrainUnlabeledDatasetSize."; + + // Create a STL10 Dataset + std::string folder_path = datasets_root_path_ + "/testSTL10Data/"; + std::shared_ptr ds = STL10(folder_path, "train+unlabeled"); + EXPECT_NE(ds, nullptr); + + EXPECT_EQ(ds->GetDatasetSize(), 2); +} + +/// Feature: STL10GetAllDatasetSize. +/// Description: test usage of STL10GetAllDatasetSize with pipeline. +/// Expectation: get correct number of data. +TEST_F(MindDataTestPipeline, TestSTL10GetAllDatasetSize) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSTL10GetAllDatasetSize."; + + // Create a STL10 Dataset + std::string folder_path = datasets_root_path_ + "/testSTL10Data/"; + std::shared_ptr ds = STL10(folder_path, "all"); + EXPECT_NE(ds, nullptr); + + EXPECT_EQ(ds->GetDatasetSize(), 3); +} + +/// Feature: STL10TrainDatasetGetters. +/// Description: test usage of getters STL10TrainDataset. +/// Expectation: get correct number of data and correct tensor shape. +TEST_F(MindDataTestPipeline, TestSTL10TrainGetters) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSTL10TrainGetter."; + + // Create a STL10 Dataset + std::string folder_path = datasets_root_path_ + "/testSTL10Data/"; + std::shared_ptr ds = STL10(folder_path, "train"); + EXPECT_NE(ds, nullptr); + + EXPECT_EQ(ds->GetDatasetSize(), 1); + std::vector types = ToDETypes(ds->GetOutputTypes()); + std::vector shapes = ToTensorShapeVec(ds->GetOutputShapes()); + std::vector column_names = {"image", "label"}; + int64_t num_classes = ds->GetNumClasses(); + EXPECT_EQ(types.size(), 2); + EXPECT_EQ(types[0].ToString(), "uint8"); + EXPECT_EQ(types[1].ToString(), "int32"); + EXPECT_EQ(shapes.size(), 2); + EXPECT_EQ(shapes[0].ToString(), "<96,96,3>"); + EXPECT_EQ(shapes[1].ToString(), "<>"); + EXPECT_EQ(num_classes, -1); + EXPECT_EQ(ds->GetBatchSize(), 1); + EXPECT_EQ(ds->GetRepeatCount(), 1); + + EXPECT_EQ(ds->GetDatasetSize(), 1); + EXPECT_EQ(ToDETypes(ds->GetOutputTypes()), types); + EXPECT_EQ(ToTensorShapeVec(ds->GetOutputShapes()), shapes); + EXPECT_EQ(ds->GetNumClasses(), -1); + + EXPECT_EQ(ds->GetColumnNames(), column_names); + EXPECT_EQ(ds->GetDatasetSize(), 1); + EXPECT_EQ(ToDETypes(ds->GetOutputTypes()), types); + EXPECT_EQ(ToTensorShapeVec(ds->GetOutputShapes()), shapes); + EXPECT_EQ(ds->GetBatchSize(), 1); + EXPECT_EQ(ds->GetRepeatCount(), 1); + EXPECT_EQ(ds->GetNumClasses(), -1); + EXPECT_EQ(ds->GetDatasetSize(), 1); +} + +/// Feature: STL10DatasetFail. +/// Description: test failure of STL10Dataset. +/// Expectation: get none piece of data. +TEST_F(MindDataTestPipeline, testSTL10DataFail) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-testSTL10DataFail."; + + // Create a STL10 Dataset + std::shared_ptr ds = STL10("", "train", std::make_shared(false, 2)); + EXPECT_NE(ds, nullptr); + + // Create an iterator over the result of the above dataset + std::shared_ptr iter = ds->CreateIterator(); + // Expect failure: invalid STL10 input + EXPECT_EQ(iter, nullptr); +} + +/// Feature: STL10DatasetWithInvalidUsageFail. +/// Description: test failure of STL10Dataset with invalid usage. +/// Expectation: get none piece of data. +TEST_F(MindDataTestPipeline, testSTL10DataWithInvalidUsageFail) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-testSTL10DataWithNullSamplerFail."; + + // Create a STL10 Dataset + std::string folder_path = datasets_root_path_ + "/testSTL10Data/"; + std::shared_ptr ds = STL10(folder_path, "validation"); + EXPECT_NE(ds, nullptr); + + // Create an iterator over the result of the above dataset + std::shared_ptr iter = ds->CreateIterator(); + // Expect failure: invalid STL10 input, validation is not a valid usage + EXPECT_EQ(iter, nullptr); +} + +/// Feature: STL10DatasetWithNullSamplerFail. +/// Description: test failure of STL10Dataset with null sampler. +/// Expectation: get none piece of data. +TEST_F(MindDataTestPipeline, testSTL10DataWithNullSamplerFail) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-testSTL10DataWithNullSamplerFail."; + + // Create a STL10 Dataset + std::string folder_path = datasets_root_path_ + "/testSTL10Data/"; + std::shared_ptr ds = STL10(folder_path, "train", nullptr); + EXPECT_NE(ds, nullptr); + + // Create an iterator over the result of the above dataset + std::shared_ptr iter = ds->CreateIterator(); + // Expect failure: invalid STL10 input, sampler cannot be nullptr + EXPECT_EQ(iter, nullptr); +} diff --git a/tests/ut/data/dataset/testSTL10Data/test_X.bin b/tests/ut/data/dataset/testSTL10Data/test_X.bin new file mode 100644 index 00000000000..935f5550058 Binary files /dev/null and b/tests/ut/data/dataset/testSTL10Data/test_X.bin differ diff --git a/tests/ut/data/dataset/testSTL10Data/test_y.bin b/tests/ut/data/dataset/testSTL10Data/test_y.bin new file mode 100644 index 00000000000..501a6bbaf1e --- /dev/null +++ b/tests/ut/data/dataset/testSTL10Data/test_y.bin @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/tests/ut/data/dataset/testSTL10Data/train_X.bin b/tests/ut/data/dataset/testSTL10Data/train_X.bin new file mode 100644 index 00000000000..a72f2bb45a9 Binary files /dev/null and b/tests/ut/data/dataset/testSTL10Data/train_X.bin differ diff --git a/tests/ut/data/dataset/testSTL10Data/train_y.bin b/tests/ut/data/dataset/testSTL10Data/train_y.bin new file mode 100644 index 00000000000..f8fa5a23540 --- /dev/null +++ b/tests/ut/data/dataset/testSTL10Data/train_y.bin @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/tests/ut/data/dataset/testSTL10Data/unlabeled_X.bin b/tests/ut/data/dataset/testSTL10Data/unlabeled_X.bin new file mode 100644 index 00000000000..53bd4182af3 Binary files /dev/null and b/tests/ut/data/dataset/testSTL10Data/unlabeled_X.bin differ diff --git a/tests/ut/python/dataset/test_datasets_stl10.py b/tests/ut/python/dataset/test_datasets_stl10.py new file mode 100644 index 00000000000..828e1302c7c --- /dev/null +++ b/tests/ut/python/dataset/test_datasets_stl10.py @@ -0,0 +1,397 @@ +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +""" +Test STL10 dataset operators +""" +import os + +import matplotlib.pyplot as plt +import numpy as np +import pytest + +import mindspore.dataset as ds +import mindspore.dataset.vision.c_transforms as vision +from mindspore import log as logger + +DATA_DIR = "../data/dataset/testSTL10Data" +WRONG_DIR = "../data/dataset/testMnistData" + + +def loadfile(path_to_data, path_to_labels=None): + """ + Feature: loadfile. + Description: parse stl10 file. + Expectation: get image and label of stl10 dataset. + """ + labels = None + if path_to_labels: + with open(os.path.realpath(path_to_labels), 'rb') as f: + labels = np.fromfile(f, dtype=np.uint8) - 1 # 0-based + + with open(path_to_data, 'rb') as f: + # read whole file in uint8 chunks + everything = np.fromfile(f, dtype=np.uint8) + images = np.reshape(everything, (-1, 3, 96, 96)) + images = np.transpose(images, (0, 1, 3, 2)) + return images, labels + + +def load_stl10(path, usage): + """ + Feature: load_stl10. + Description: load stl10. + Expectation: get data of stl10 dataset. + """ + assert usage in ["train", "test", "unlabeled", "train+unlabeled", "all"] + + if usage == "train": + image_path = os.path.join(path, "train_X.bin") + label_path = os.path.join(path, "train_y.bin") + images, labels = loadfile(image_path, label_path) + + elif usage == "train+unlabeled": + image_path = os.path.join(path, "train_X.bin") + label_path = os.path.join(path, "train_y.bin") + images, labels = loadfile(image_path, label_path) + + image_path = os.path.join(path, "unlabeled_X.bin") + unlabeled_image, _ = loadfile(image_path) + + images = np.concatenate((images, unlabeled_image)) + labels = np.concatenate((labels, np.asarray([-1] * unlabeled_image.shape[0]))) + + elif usage == "unlabeled": + image_path = os.path.join(path, "unlabeled_X.bin") + + images, _ = loadfile(image_path) + labels = np.asarray([-1] * images.shape[0]) + + elif usage == "test": + image_path = os.path.join(path, "test_X.bin") + label_path = os.path.join(path, "test_y.bin") + + images, labels = loadfile(image_path, label_path) + + elif usage == "all": + image_path = os.path.join(path, "test_X.bin") + label_path = os.path.join(path, "test_y.bin") + images, labels = loadfile(image_path, label_path) + + image_path = os.path.join(path, "train_X.bin") + label_path = os.path.join(path, "train_y.bin") + + train_image, train_label = loadfile(image_path, label_path) + + images = np.concatenate((images, train_image)) + labels = np.concatenate((labels, train_label)) + + image_path = os.path.join(path, "unlabeled_X.bin") + unlabeled_image, _ = loadfile(image_path) + + images = np.concatenate((images, unlabeled_image)) + labels = np.concatenate((labels, np.asarray([-1] * unlabeled_image.shape[0]))) + + return images, labels + + +def visualize_dataset(images, labels): + """ + Feature: visualize_dataset. + Description: visualize stl10 dataset. + Expectation: plot images. + """ + num_samples = len(images) + for i in range(num_samples): + plt.subplot(1, num_samples, i + 1) + plt.imshow(np.transpose(images[i], (1, 2, 0))) + plt.title(labels[i]) + plt.show() + + +def test_stl10_content_check(): + """ + Feature: test_stl10_content_check. + Description: validate STL10ataset image readings. + Expectation: get correct number of data and correct content. + """ + logger.info("Test STL10Dataset Op with content check") + # 1. train data. + data1 = ds.STL10Dataset(DATA_DIR, usage="train", num_samples=1, shuffle=False) + images, labels = load_stl10(DATA_DIR, "train") + num_iter = 0 + # in this example, each dictionary has keys "image" and "label". + for i, d in enumerate(data1.create_dict_iterator(num_epochs=1, output_numpy=True)): + np.testing.assert_array_equal(d["image"], np.transpose(images[i], (1, 2, 0))) + np.testing.assert_array_equal(d["label"], labels[i]) + num_iter += 1 + assert num_iter == 1 + + # 2. test data. + data1 = ds.STL10Dataset(DATA_DIR, usage="test", num_samples=1, shuffle=False) + images, labels = load_stl10(DATA_DIR, "test") + num_iter = 0 + # in this example, each dictionary has keys "image" and "label". + for i, d in enumerate(data1.create_dict_iterator(num_epochs=1, output_numpy=True)): + np.testing.assert_array_equal(d["image"], np.transpose(images[i], (1, 2, 0))) + np.testing.assert_array_equal(d["label"], labels[i]) + num_iter += 1 + assert num_iter == 1 + + # 3. unlabeled data. + data1 = ds.STL10Dataset(DATA_DIR, usage="unlabeled", num_samples=1, shuffle=False) + images, labels = load_stl10(DATA_DIR, "unlabeled") + num_iter = 0 + # in this example, each dictionary has keys "image" and "label". + for i, d in enumerate(data1.create_dict_iterator(num_epochs=1, output_numpy=True)): + np.testing.assert_array_equal(d["image"], np.transpose(images[i], (1, 2, 0))) + np.testing.assert_array_equal(d["label"], labels[i]) + num_iter += 1 + assert num_iter == 1 + + # 4. train+unlabeled data. + data1 = ds.STL10Dataset(DATA_DIR, usage="train+unlabeled", num_samples=2, shuffle=False) + images, labels = load_stl10(DATA_DIR, "train+unlabeled") + num_iter = 0 + # in this example, each dictionary has keys "image" and "label". + for i, d in enumerate(data1.create_dict_iterator(num_epochs=1, output_numpy=True)): + np.testing.assert_array_equal(d["image"], np.transpose(images[i], (1, 2, 0))) + np.testing.assert_array_equal(d["label"], labels[i]) + num_iter += 1 + assert num_iter == 2 + + # 4. all data. + data1 = ds.STL10Dataset(DATA_DIR, usage="all", num_samples=3, shuffle=False) + images, labels = load_stl10(DATA_DIR, "all") + num_iter = 0 + # in this example, each dictionary has keys "image" and "label". + for i, d in enumerate(data1.create_dict_iterator(num_epochs=1, output_numpy=True)): + np.testing.assert_array_equal(d["image"], np.transpose(images[i], (1, 2, 0))) + np.testing.assert_array_equal(d["label"], labels[i]) + num_iter += 1 + assert num_iter == 3 + + +def test_stl10_basic(): + """ + Feature: test_stl10_basic. + Description: test basic usage of STL10Dataset. + Expectation: get correct number of data. + """ + logger.info("Test STL10Dataset Op") + + # case 1: test loading whole dataset. + all_data = ds.STL10Dataset(DATA_DIR, "all") + num_iter = 0 + for _ in all_data.create_dict_iterator(num_epochs=1): + num_iter += 1 + assert num_iter == 3 + + # case 2: test num_samples. + all_data = ds.STL10Dataset(DATA_DIR, "all", num_samples=1) + num_iter = 0 + for _ in all_data.create_dict_iterator(num_epochs=1): + num_iter += 1 + assert num_iter == 1 + + # case 3: test repeat. + all_data = ds.STL10Dataset(DATA_DIR, "all", num_samples=2) + all_data = all_data.repeat(5) + num_iter = 0 + for _ in all_data.create_dict_iterator(num_epochs=1): + num_iter += 1 + assert num_iter == 10 + + # case 4: test batch with drop_remainder=False. + all_data = ds.STL10Dataset(DATA_DIR, "all", num_samples=2) + assert all_data.get_dataset_size() == 2 + assert all_data.get_batch_size() == 1 + all_data = all_data.batch(batch_size=2) # drop_remainder is default to be False. + assert all_data.get_batch_size() == 2 + assert all_data.get_dataset_size() == 1 + + num_iter = 0 + for _ in all_data.create_dict_iterator(num_epochs=1): + num_iter += 1 + assert num_iter == 1 + + # case 5: test batch with drop_remainder=True. + all_data = ds.STL10Dataset(DATA_DIR, "all", num_samples=2) + assert all_data.get_dataset_size() == 2 + assert all_data.get_batch_size() == 1 + all_data = all_data.batch(batch_size=2, drop_remainder=True) # the rest of incomplete batch will be dropped. + assert all_data.get_dataset_size() == 1 + assert all_data.get_batch_size() == 2 + num_iter = 0 + for _ in all_data.create_dict_iterator(num_epochs=1): + num_iter += 1 + assert num_iter == 1 + + +def test_stl10_sequential_sampler(): + """ + Feature: test_stl10_sequential_sampler. + Description: test usage of STL10Dataset with SequentialSampler. + Expectation: get correct number of data. + """ + logger.info("Test STL10Dataset Op with SequentialSampler") + num_samples = 2 + sampler = ds.SequentialSampler(num_samples=num_samples) + all_data_1 = ds.STL10Dataset(DATA_DIR, "all", sampler=sampler) + all_data_2 = ds.STL10Dataset(DATA_DIR, "all", shuffle=False, num_samples=num_samples) + label_list_1, label_list_2 = [], [] + num_iter = 0 + for item1, item2 in zip(all_data_1.create_dict_iterator(num_epochs=1), + all_data_2.create_dict_iterator(num_epochs=1)): + label_list_1.append(item1["label"].asnumpy()) + label_list_2.append(item2["label"].asnumpy()) + num_iter += 1 + np.testing.assert_array_equal(label_list_1, label_list_2) + assert num_iter == num_samples + + +def test_stl10_exception(): + """ + Feature: test_stl10_exception. + Description: test error cases for STL10Dataset. + Expectation: raise exception. + """ + logger.info("Test error cases for STL10Dataset") + error_msg_1 = "sampler and shuffle cannot be specified at the same time" + with pytest.raises(RuntimeError, match=error_msg_1): + ds.STL10Dataset(DATA_DIR, "all", shuffle=False, sampler=ds.PKSampler(3)) + + error_msg_2 = "sampler and sharding cannot be specified at the same time" + with pytest.raises(RuntimeError, match=error_msg_2): + ds.STL10Dataset(DATA_DIR, "all", sampler=ds.PKSampler(3), num_shards=2, shard_id=0) + + error_msg_3 = "num_shards is specified and currently requires shard_id as well" + with pytest.raises(RuntimeError, match=error_msg_3): + ds.STL10Dataset(DATA_DIR, "all", num_shards=10) + + error_msg_4 = "shard_id is specified but num_shards is not" + with pytest.raises(RuntimeError, match=error_msg_4): + ds.STL10Dataset(DATA_DIR, "all", shard_id=0) + + error_msg_5 = "Input shard_id is not within the required interval" + with pytest.raises(ValueError, match=error_msg_5): + ds.STL10Dataset(DATA_DIR, "all", num_shards=5, shard_id=-1) + with pytest.raises(ValueError, match=error_msg_5): + ds.STL10Dataset(DATA_DIR, "all", num_shards=5, shard_id=5) + with pytest.raises(ValueError, match=error_msg_5): + ds.STL10Dataset(DATA_DIR, "all", num_shards=2, shard_id=5) + error_msg_6 = "num_parallel_workers exceeds" + with pytest.raises(ValueError, match=error_msg_6): + ds.STL10Dataset(DATA_DIR, "all", shuffle=False, num_parallel_workers=0) + with pytest.raises(ValueError, match=error_msg_6): + ds.STL10Dataset(DATA_DIR, "all", shuffle=False, num_parallel_workers=256) + with pytest.raises(ValueError, match=error_msg_6): + ds.STL10Dataset(DATA_DIR, "all", shuffle=False, num_parallel_workers=-2) + error_msg_7 = "Argument shard_id" + with pytest.raises(TypeError, match=error_msg_7): + ds.STL10Dataset(DATA_DIR, "all", num_shards=2, shard_id="0") + + def exception_func(item): + raise Exception("Error occur!") + + error_msg_8 = "The corresponding data files" + with pytest.raises(RuntimeError, match=error_msg_8): + all_data = ds.STL10Dataset(DATA_DIR, "all") + all_data = all_data.map(operations=exception_func, input_columns=["image"], num_parallel_workers=1) + for _ in all_data.__iter__(): + pass + + with pytest.raises(RuntimeError, match=error_msg_8): + all_data = ds.STL10Dataset(DATA_DIR, "all") + all_data = all_data.map(operations=vision.Decode(), input_columns=["image"], num_parallel_workers=1) + for _ in all_data.__iter__(): + pass + + error_msg_9 = "does not exist or permission denied!" + with pytest.raises(ValueError, match=error_msg_9): + all_data = ds.STL10Dataset(WRONG_DIR, "all") + for _ in all_data.__iter__(): + pass + + +def test_stl10_visualize(plot=False): + """ + Feature: test_stl10_visualize. + Description: visualize STL10Dataset results. + Expectation: get correct number of data and plot them. + """ + logger.info("Test STL10Dataset visualization") + all_data = ds.STL10Dataset(DATA_DIR, "all", num_samples=2, shuffle=False) + num_iter = 0 + image_list, label_list = [], [] + for item in all_data.create_dict_iterator(num_epochs=1, output_numpy=True): + image = item["image"] + label = item["label"] + image_list.append(image) + label_list.append("label {}".format(label)) + assert isinstance(image, np.ndarray) + assert image.shape == (96, 96, 3) + assert image.dtype == np.uint8 + assert label.dtype == np.int32 + num_iter += 1 + assert num_iter == 2 + if plot: + visualize_dataset(image_list, label_list) + + +def test_stl10_usage(): + """ + Feature: test_stl10_usage. + Description: validate STL10Dataset image readings. + Expectation: get correct number of data. + """ + logger.info("Test STL10Dataset usage flag") + + def test_config(usage, path=None): + path = DATA_DIR if path is None else path + try: + data = ds.STL10Dataset(path, usage=usage, shuffle=False) + num_rows = 0 + for _ in data.create_dict_iterator(num_epochs=1, output_numpy=True): + num_rows += 1 + except (ValueError, TypeError, RuntimeError) as e: + return str(e) + return num_rows + + assert test_config("train") == 1 + assert test_config("test") == 1 + assert test_config("unlabeled") == 1 + assert test_config("train+unlabeled") == 2 + assert test_config("all") == 3 + + assert "Input usage is not within the valid set of ['train', 'test', 'unlabeled', 'train+unlabeled', 'all']."\ + in test_config("invalid") + assert "Argument usage with value ['list'] is not of type []" in test_config(["list"]) + + # change this directory to the folder that contains all STL10 files. + all_files_path = None + # the following tests on the entire datasets. + if all_files_path is not None: + assert test_config("train", all_files_path) == 1 + + assert ds.STL10Dataset(all_files_path, usage="train").get_dataset_size() == 1 + + +if __name__ == '__main__': + test_stl10_content_check() + test_stl10_basic() + test_stl10_sequential_sampler() + test_stl10_exception() + test_stl10_visualize(plot=True) + test_stl10_usage()