From a86ac516fcf6227eebd87e5a65586d5206d71e3e Mon Sep 17 00:00:00 2001 From: liucunwei Date: Wed, 3 Nov 2021 12:02:00 +0800 Subject: [PATCH] add some of ut about dataset IteratorOneColumn --- .../cpp/dataset/c_api_dataset_album_test.cc | 84 +++++++++++++++++++ .../cpp/dataset/c_api_dataset_div2k_test.cc | 83 ++++++++++++++++++ .../cpp/dataset/c_api_dataset_flickr_test.cc | 81 ++++++++++++++++++ .../dataset/c_api_dataset_manifest_test.cc | 83 ++++++++++++++++++ .../ut/cpp/dataset/c_api_dataset_sbu_test.cc | 74 ++++++++++++++++ .../ut/cpp/dataset/c_api_dataset_usps_test.cc | 82 ++++++++++++++++++ 6 files changed, 487 insertions(+) diff --git a/tests/ut/cpp/dataset/c_api_dataset_album_test.cc b/tests/ut/cpp/dataset/c_api_dataset_album_test.cc index 9ae08cea6d3..2080a458276 100644 --- a/tests/ut/cpp/dataset/c_api_dataset_album_test.cc +++ b/tests/ut/cpp/dataset/c_api_dataset_album_test.cc @@ -23,6 +23,9 @@ class MindDataTestPipeline : public UT::DatasetOpTesting { protected: }; +/// Feature: AlbumDataset. +/// Description: test basic usage of AlbumDataset. +/// Expectation: get correct number of data. TEST_F(MindDataTestPipeline, TestAlbumBasic) { MS_LOG(INFO) << "Doing MindDataTestPipeline-TestAlbumBasic."; @@ -155,6 +158,9 @@ TEST_F(MindDataTestPipeline, TestAlbumWithFullSchema) { iter->Stop(); } +/// Feature: AlbumDatasetWithPipeline. +/// Description: test usage of AlbumDataset with pipeline. +/// Expectation: get correct number of data. TEST_F(MindDataTestPipeline, TestAlbumBasicWithPipeline) { MS_LOG(INFO) << "Doing MindDataTestPipeline-TestAlbumBasicWithPipeline."; @@ -210,6 +216,69 @@ TEST_F(MindDataTestPipeline, TestAlbumBasicWithPipeline) { iter->Stop(); } +/// Feature: AlbumIteratorOneColumn. +/// Description: test iterator of AlbumDataset with only the "image" column. +/// Expectation: get correct data. +TEST_F(MindDataTestPipeline, TestAlbumIteratorOneColumn) { +MS_LOG(INFO) << "Doing MindDataTestPipeline-TestAlbumIteratorOneColumn."; +// Create a Album Dataset +std::string folder_path = datasets_root_path_ + "/testAlbum/images"; +std::string schema_file = datasets_root_path_ + "/testAlbum/datasetSchema.json"; +std::vector column_names = {"image", "label", "id"}; +std::shared_ptr ds = Album(folder_path, schema_file, column_names); +EXPECT_NE(ds, nullptr); + +// Create a Batch operation on ds +int32_t batch_size = 1; +ds = ds->Batch(batch_size); +EXPECT_NE(ds, nullptr); + +// Create an iterator over the result of the above dataset +// Only select "image" column and drop others +std::vector columns = {"image"}; +std::shared_ptr iter = ds->CreateIterator(columns, -1); +EXPECT_NE(iter, nullptr); + +// Iterate the dataset and get each row +std::vector row; +ASSERT_OK(iter->GetNextRow(&row)); + +uint64_t i = 0; +while (row.size() != 0) { +for (auto &v : row) { +MS_LOG(INFO) << "image shape:" << v.Shape(); +} +ASSERT_OK(iter->GetNextRow(&row)); +i++; +} + +EXPECT_EQ(i, 7); + +// Manually terminate the pipeline +iter->Stop(); +} + +/// Feature: AlbumIteratorWrongColumn. +/// Description: test iterator of AlbumDataset with wrong column. +/// Expectation: get none piece of data. +TEST_F(MindDataTestPipeline, TestAlbumIteratorWrongColumn) { +MS_LOG(INFO) << "Doing MindDataTestPipeline-TestAlbumIteratorWrongColumn."; +// Create a Album Dataset +std::string folder_path = datasets_root_path_ + "/testAlbum/images"; +std::string schema_file = datasets_root_path_ + "/testAlbum/datasetSchema.json"; +std::vector column_names = {"image", "label", "id"}; +std::shared_ptr ds = Album(folder_path, schema_file, column_names); +EXPECT_NE(ds, nullptr); + +// Pass wrong column name +std::vector columns = {"digital"}; +std::shared_ptr iter = ds->CreateIterator(columns); +EXPECT_EQ(iter, nullptr); +} + +/// Feature: AlbumDatasetGetters. +/// Description: test usage of getters AlbumDataset. +/// Expectation: get correct number of data and correct tensor shape. TEST_F(MindDataTestPipeline, TestAlbumGetters) { MS_LOG(INFO) << "Doing MindDataTestPipeline-TestAlbumGetters."; @@ -238,6 +307,9 @@ TEST_F(MindDataTestPipeline, TestAlbumGetters) { EXPECT_EQ(num_samples, 7); } +/// Feature: AlbumDecode. +/// Description: test usage of AlbumDecode. +/// Expectation: get correct number of data. TEST_F(MindDataTestPipeline, TestAlbumDecode) { MS_LOG(INFO) << "Doing MindDataTestPipeline-TestAlbumDecode."; std::string folder_path = datasets_root_path_ + "/testAlbum/images"; @@ -273,6 +345,9 @@ TEST_F(MindDataTestPipeline, TestAlbumDecode) { iter->Stop(); } +/// Feature: AlbumNumSampler. +/// Description: test usage of AlbumDataset with num sampler. +/// Expectation: get correct piece of data. TEST_F(MindDataTestPipeline, TestAlbumNumSamplers) { MS_LOG(INFO) << "Doing MindDataTestPipeline-TestAlbumNumSamplers."; @@ -306,6 +381,9 @@ TEST_F(MindDataTestPipeline, TestAlbumNumSamplers) { iter->Stop(); } +/// Feature: AlbumError. +/// Description: test failure of Album Dataset. +/// Expectation: get none piece of data. TEST_F(MindDataTestPipeline, TestAlbumError) { MS_LOG(INFO) << "Doing MindDataTestPipeline-TestAlbumError."; std::string folder_path = datasets_root_path_ + "/testAlbum/ima"; @@ -321,6 +399,9 @@ TEST_F(MindDataTestPipeline, TestAlbumError) { EXPECT_EQ(iter, nullptr); } +/// Feature: AlbumWithNullSamplerError. +/// Description: test failure of Album Dataset. +/// Expectation: get none piece of data. TEST_F(MindDataTestPipeline, TestAlbumWithNullSamplerError) { MS_LOG(INFO) << "Doing MindDataTestPipeline-TestAlbumWithNullSamplerError."; std::string folder_path = datasets_root_path_ + "/testAlbum/images"; @@ -336,6 +417,9 @@ TEST_F(MindDataTestPipeline, TestAlbumWithNullSamplerError) { EXPECT_EQ(iter, nullptr); } +/// Feature: AlbumDuplicateColumnNameError. +/// Description: test failure of Album Dataset. +/// Expectation: get none piece of data. TEST_F(MindDataTestPipeline, TestAlbumDuplicateColumnNameError) { MS_LOG(INFO) << "Doing MindDataTestPipeline-TestAlbumDuplicateColumnNameError."; std::string folder_path = datasets_root_path_ + "/testAlbum/images"; diff --git a/tests/ut/cpp/dataset/c_api_dataset_div2k_test.cc b/tests/ut/cpp/dataset/c_api_dataset_div2k_test.cc index 9583e429fb0..cc902366861 100644 --- a/tests/ut/cpp/dataset/c_api_dataset_div2k_test.cc +++ b/tests/ut/cpp/dataset/c_api_dataset_div2k_test.cc @@ -23,6 +23,9 @@ class MindDataTestPipeline : public UT::DatasetOpTesting { protected: }; +/// Feature: DIV2KDataset. +/// Description: test basic usage of DIV2KDataset. +/// Expectation: get correct number of data. TEST_F(MindDataTestPipeline, TestDIV2KBasic) { MS_LOG(INFO) << "Doing MindDataTestPipeline-TestDIV2KBasic."; @@ -60,6 +63,9 @@ TEST_F(MindDataTestPipeline, TestDIV2KBasic) { iter->Stop(); } +/// Feature: DIV2KDatasetWithPipeline. +/// Description: test usage of DIV2KDataset with pipeline. +/// Expectation: get correct number of data. TEST_F(MindDataTestPipeline, TestDIV2KBasicWithPipeline) { MS_LOG(INFO) << "Doing MindDataTestPipeline-TestDIV2KBasicWithPipeline."; @@ -118,6 +124,71 @@ TEST_F(MindDataTestPipeline, TestDIV2KBasicWithPipeline) { iter->Stop(); } +/// Feature: DIV2KIteratorOneColumn. +/// Description: test iterator of DIV2KDataset with only the "hr_image" column. +/// Expectation: get correct data. +TEST_F(MindDataTestPipeline, TestDIV2KIteratorOneColumn) { +MS_LOG(INFO) << "Doing MindDataTestPipeline-TestDIV2KIteratorOneColumn."; +// Create a DIV2K Dataset +std::string dataset_path = datasets_root_path_ + "/testDIV2KData/div2k"; +std::string usage = "train"; // train valid, all +std::string downgrade = "bicubic"; // bicubic, unknown, mild, difficult, wild +int32_t scale = 2; // 2, 3, 4, 8 +std::shared_ptr ds = DIV2K(dataset_path, usage, downgrade, scale); +EXPECT_NE(ds, nullptr); + +// Create a Batch operation on ds +int32_t batch_size = 1; +ds = ds->Batch(batch_size); +EXPECT_NE(ds, nullptr); + +// Create an iterator over the result of the above dataset +// Only select "image" column and drop others +std::vector columns = {"hr_image"}; +std::shared_ptr iter = ds->CreateIterator(columns, -1); +EXPECT_NE(iter, nullptr); + +// Iterate the dataset and get each row +std::vector row; +ASSERT_OK(iter->GetNextRow(&row)); + +uint64_t i = 0; +while (row.size() != 0) { +for (auto &v : row) { +MS_LOG(INFO) << "image shape:" << v.Shape(); +} +ASSERT_OK(iter->GetNextRow(&row)); +i++; +} + +EXPECT_EQ(i, 5); + +// Manually terminate the pipeline +iter->Stop(); +} + +/// Feature: DIV2KIteratorWrongColumn. +/// Description: test iterator of DIV2KDataset with wrong column. +/// Expectation: get none piece of data. +TEST_F(MindDataTestPipeline, TestDIV2KIteratorWrongColumn) { +MS_LOG(INFO) << "Doing MindDataTestPipeline-TestDIV2KIteratorWrongColumn."; +// Create a DIV2K Dataset +std::string dataset_path = datasets_root_path_ + "/testDIV2KData/div2k"; +std::string usage = "train"; // train valid, all +std::string downgrade = "bicubic"; // bicubic, unknown, mild, difficult, wild +int32_t scale = 2; // 2, 3, 4, 8 +std::shared_ptr ds = DIV2K(dataset_path, usage, downgrade, scale); +EXPECT_NE(ds, nullptr); + +// Pass wrong column name +std::vector columns = {"digital"}; +std::shared_ptr iter = ds->CreateIterator(columns); +EXPECT_EQ(iter, nullptr); +} + +/// Feature: DIV2KDatasetGetters. +/// Description: test usage of getters DIV2KDataset. +/// Expectation: get correct number of data and correct tensor shape. TEST_F(MindDataTestPipeline, TestDIV2KGetters) { MS_LOG(INFO) << "Doing MindDataTestPipeline-TestDIV2KGetters."; @@ -142,6 +213,9 @@ TEST_F(MindDataTestPipeline, TestDIV2KGetters) { EXPECT_EQ(ds2->GetColumnNames(), column_names); } +/// Feature: DIV2KDecode. +/// Description: test usage of DIV2KDecode. +/// Expectation: get correct number of data and correct tensor shape. TEST_F(MindDataTestPipeline, TestDIV2KDecode) { MS_LOG(INFO) << "Doing MindDataTestPipeline-TestDIV2KDecode."; @@ -183,6 +257,9 @@ TEST_F(MindDataTestPipeline, TestDIV2KDecode) { iter->Stop(); } +/// Feature: DIV2KNumSampler. +/// Description: test usage of DIV2KDataset with num sampler. +/// Expectation: get correct piece of data. TEST_F(MindDataTestPipeline, TestDIV2KNumSamplers) { MS_LOG(INFO) << "Doing MindDataTestPipeline-TestDIV2KNumSamplers."; @@ -223,6 +300,9 @@ TEST_F(MindDataTestPipeline, TestDIV2KNumSamplers) { iter->Stop(); } +/// Feature: DIV2KError. +/// Description: test failure of DIV2K Dataset. +/// Expectation: get none piece of data. TEST_F(MindDataTestPipeline, TestDIV2KError) { MS_LOG(INFO) << "Doing MindDataTestPipeline-TestDIV2KError."; @@ -286,6 +366,9 @@ TEST_F(MindDataTestPipeline, TestDIV2KError) { EXPECT_EQ(iter5, nullptr); } +/// Feature: DIV2KWithNullSamplerError. +/// Description: test failure of DIV2K Dataset. +/// Expectation: get none piece of data. TEST_F(MindDataTestPipeline, TestDIV2KWithNullSamplerError) { MS_LOG(INFO) << "Doing MindDataTestPipeline-TestDIV2KWithNullSamplerError."; diff --git a/tests/ut/cpp/dataset/c_api_dataset_flickr_test.cc b/tests/ut/cpp/dataset/c_api_dataset_flickr_test.cc index cdd109ef872..50ea61e40a1 100644 --- a/tests/ut/cpp/dataset/c_api_dataset_flickr_test.cc +++ b/tests/ut/cpp/dataset/c_api_dataset_flickr_test.cc @@ -23,6 +23,9 @@ class MindDataTestPipeline : public UT::DatasetOpTesting { protected: }; +/// Feature: FlickrBasic. +/// Description: test basic usage of FlickrDataset. +/// Expectation: get correct number of data. TEST_F(MindDataTestPipeline, TestFlickrBasic) { MS_LOG(INFO) << "Doing MindDataTestPipeline-TestFlickrBasic."; @@ -56,6 +59,9 @@ TEST_F(MindDataTestPipeline, TestFlickrBasic) { iter->Stop(); } +/// Feature: FlickrBasicWithPipeline. +/// Description: test usage of FlickrDataset with pipeline. +/// Expectation: get correct number of data. TEST_F(MindDataTestPipeline, TestFlickrBasicWithPipeline) { MS_LOG(INFO) << "Doing MindDataTestPipeline-TestFlickrBasicWithPipeline."; @@ -110,6 +116,66 @@ TEST_F(MindDataTestPipeline, TestFlickrBasicWithPipeline) { iter->Stop(); } +/// Feature: FlickrIteratorOneColumn. +/// Description: test iterator of FlickrDataset with only the "image" column. +/// Expectation: get correct data. +TEST_F(MindDataTestPipeline, TestFlickrIteratorOneColumn) { +MS_LOG(INFO) << "Doing MindDataTestPipeline-TestFlickrIteratorOneColumn."; +std::string dataset_path = datasets_root_path_ + "/testFlickrData/flickr30k/flickr30k-images"; +std::string file_path = datasets_root_path_ + "/testFlickrData/flickr30k/test1.token"; + +// Create a Flickr30k Dataset +std::shared_ptr ds = Flickr(dataset_path, file_path); +EXPECT_NE(ds, nullptr); + +// Create a Batch operation on ds +int32_t batch_size = 1; +ds = ds->Batch(batch_size); +EXPECT_NE(ds, nullptr); + +// Create an iterator over the result of the above dataset +// Only select "image" column and drop others +std::vector columns = {"image"}; +std::shared_ptr iter = ds->CreateIterator(columns, -1); +EXPECT_NE(iter, nullptr); + +// Iterate the dataset and get each row +std::vector row; +ASSERT_OK(iter->GetNextRow(&row)); + +uint64_t i = 0; +while (row.size() != 0) { +for (auto &v : row) { +MS_LOG(INFO) << "image shape:" << v.Shape(); +} +ASSERT_OK(iter->GetNextRow(&row)); +i++; +} + +EXPECT_EQ(i, 2); + +// Manually terminate the pipeline +iter->Stop(); +} + +/// Feature: FlickrIteratorWrongColumn. +/// Description: test iterator of FlickrDataset with wrong column. +/// Expectation: get none piece of data. +TEST_F(MindDataTestPipeline, TestFlickrIteratorWrongColumn) { +MS_LOG(INFO) << "Doing MindDataTestPipeline-TestFlickrIteratorWrongColumn."; +std::string dataset_path = datasets_root_path_ + "/testFlickrData/flickr30k/flickr30k-images"; +std::string file_path = datasets_root_path_ + "/testFlickrData/flickr30k/test1.token"; + +// Create a Flickr30k Dataset +std::shared_ptr ds = Flickr(dataset_path, file_path); +EXPECT_NE(ds, nullptr); + +// Pass wrong column name +std::vector columns = {"digital"}; +std::shared_ptr iter = ds->CreateIterator(columns); +EXPECT_EQ(iter, nullptr); +} + TEST_F(MindDataTestPipeline, TestFlickrGetters) { MS_LOG(INFO) << "Doing MindDataTestPipeline-TestFlickrGetters."; @@ -131,6 +197,9 @@ TEST_F(MindDataTestPipeline, TestFlickrGetters) { EXPECT_EQ(ds2->GetColumnNames(), column_names); } +/// Feature: FlickrAnnotations. +/// Description: test usage of FlickrAnnotations. +/// Expectation: get correct number of data. TEST_F(MindDataTestPipeline, TestFlickrAnnotations) { MS_LOG(INFO) << "Doing MindDataTestPipeline-TestFlickrGetters."; @@ -177,6 +246,9 @@ TEST_F(MindDataTestPipeline, TestFlickrAnnotations) { iter->Stop(); } +/// Feature: FlickrDecode. +/// Description: test usage of FlickrDecode. +/// Expectation: get correct number of data. TEST_F(MindDataTestPipeline, TestFlickrDecode) { MS_LOG(INFO) << "Doing MindDataTestPipeline-TestFlickrDecode."; @@ -212,6 +284,9 @@ TEST_F(MindDataTestPipeline, TestFlickrDecode) { iter->Stop(); } +/// Feature: FlickrNumSamplers. +/// Description: test usage of FlickrDataset with num sampler. +/// Expectation: get correct piece of data. TEST_F(MindDataTestPipeline, TestFlickrNumSamplers) { MS_LOG(INFO) << "Doing MindDataTestPipeline-TestFlickrNumSamplers."; @@ -250,6 +325,9 @@ TEST_F(MindDataTestPipeline, TestFlickrNumSamplers) { iter->Stop(); } +/// Feature: FlickrError. +/// Description: test failure of Flickr Dataset. +/// Expectation: get none piece of data. TEST_F(MindDataTestPipeline, TestFlickrError) { MS_LOG(INFO) << "Doing MindDataTestPipeline-TestFlickrError."; @@ -292,6 +370,9 @@ TEST_F(MindDataTestPipeline, TestFlickrError) { EXPECT_EQ(iter3, nullptr); } +/// Feature: FlickrWithNullSamplerError. +/// Description: test failure of FlickrDataset with null sampler. +/// Expectation: get none piece of data. TEST_F(MindDataTestPipeline, TestFlickrWithNullSamplerError) { MS_LOG(INFO) << "Doing MindDataTestPipeline-TestFlickrWithNullSamplerError."; diff --git a/tests/ut/cpp/dataset/c_api_dataset_manifest_test.cc b/tests/ut/cpp/dataset/c_api_dataset_manifest_test.cc index 8e85ff05681..658e15a2edc 100644 --- a/tests/ut/cpp/dataset/c_api_dataset_manifest_test.cc +++ b/tests/ut/cpp/dataset/c_api_dataset_manifest_test.cc @@ -23,6 +23,9 @@ class MindDataTestPipeline : public UT::DatasetOpTesting { protected: }; +/// Feature: ManifestBasic. +/// Description: test basic usage of ManifestDataset. +/// Expectation: get correct number of data. TEST_F(MindDataTestPipeline, TestManifestBasic) { MS_LOG(INFO) << "Doing MindDataTestPipeline-TestManifestBasic."; @@ -54,6 +57,9 @@ TEST_F(MindDataTestPipeline, TestManifestBasic) { iter->Stop(); } +/// Feature: ManifestBasicWithPipeline. +/// Description: test usage of ManifestDataset with pipeline. +/// Expectation: get correct number of data. TEST_F(MindDataTestPipeline, TestManifestBasicWithPipeline) { MS_LOG(INFO) << "Doing MindDataTestPipeline-TestManifestBasicWithPipeline."; @@ -106,6 +112,65 @@ TEST_F(MindDataTestPipeline, TestManifestBasicWithPipeline) { iter->Stop(); } +/// Feature: ManifestIteratorOneColumn. +/// Description: test iterator of ManifestDataset with only the "image" column. +/// Expectation: get correct data. +TEST_F(MindDataTestPipeline, TestManifestIteratorOneColumn) { +MS_LOG(INFO) << "Doing MindDataTestPipeline-TestManifestIteratorOneColumn."; +std::string file_path = datasets_root_path_ + "/testManifestData/cpp.json"; +// Create a Manifest Dataset +std::shared_ptr ds = Manifest(file_path); +EXPECT_NE(ds, nullptr); + +// Create a Batch operation on ds +int32_t batch_size = 1; +ds = ds->Batch(batch_size); +EXPECT_NE(ds, nullptr); + +// Create an iterator over the result of the above dataset +// Only select "image" column and drop others +std::vector columns = {"image"}; +std::shared_ptr iter = ds->CreateIterator(columns, -1); +EXPECT_NE(iter, nullptr); + +// Iterate the dataset and get each row +std::vector row; +ASSERT_OK(iter->GetNextRow(&row)); + +uint64_t i = 0; +while (row.size() != 0) { +for (auto &v : row) { +MS_LOG(INFO) << "image shape:" << v.Shape(); +} +ASSERT_OK(iter->GetNextRow(&row)); +i++; +} + +EXPECT_EQ(i, 2); + +// Manually terminate the pipeline +iter->Stop(); +} + +/// Feature: ManifestIteratorWrongColumn. +/// Description: test iterator of ManifestDataset with wrong column. +/// Expectation: get none piece of data. +TEST_F(MindDataTestPipeline, TestManifestIteratorWrongColumn) { +MS_LOG(INFO) << "Doing MindDataTestPipeline-TestManifestIteratorWrongColumn."; +std::string file_path = datasets_root_path_ + "/testManifestData/cpp.json"; +// Create a Manifest Dataset +std::shared_ptr ds = Manifest(file_path); +EXPECT_NE(ds, nullptr); + +// Pass wrong column name +std::vector columns = {"digital"}; +std::shared_ptr iter = ds->CreateIterator(columns); +EXPECT_EQ(iter, nullptr); +} + +/// Feature: ManifestGetters. +/// Description: test usage of getters ManifestDataset. +/// Expectation: get correct number of data and correct tensor shape. TEST_F(MindDataTestPipeline, TestManifestGetters) { MS_LOG(INFO) << "Doing MindDataTestPipeline-TestManifestGetters."; @@ -142,6 +207,9 @@ TEST_F(MindDataTestPipeline, TestManifestGetters) { EXPECT_EQ(class_index2[2].second[0], 2); } +/// Feature: ManifestDecode. +/// Description: test usage of ManifestDecode. +/// Expectation: get correct number of data. TEST_F(MindDataTestPipeline, TestManifestDecode) { MS_LOG(INFO) << "Doing MindDataTestPipeline-TestManifestDecode."; @@ -176,6 +244,9 @@ TEST_F(MindDataTestPipeline, TestManifestDecode) { iter->Stop(); } +/// Feature: ManifestEval. +/// Description: test usage of ManifestEval. +/// Expectation: get correct number of data. TEST_F(MindDataTestPipeline, TestManifestEval) { MS_LOG(INFO) << "Doing MindDataTestPipeline-TestManifestEval."; @@ -207,6 +278,9 @@ TEST_F(MindDataTestPipeline, TestManifestEval) { iter->Stop(); } +/// Feature: ManifestClassIndex. +/// Description: test usage of ManifestClassIndex. +/// Expectation: get correct number of data. TEST_F(MindDataTestPipeline, TestManifestClassIndex) { MS_LOG(INFO) << "Doing MindDataTestPipeline-TestManifestClassIndex."; @@ -261,6 +335,9 @@ TEST_F(MindDataTestPipeline, TestManifestClassIndex) { iter->Stop(); } +/// Feature: ManifestNumSamplers. +/// Description: test usage of ManifestDataset with num sampler. +/// Expectation: get correct piece of data. TEST_F(MindDataTestPipeline, TestManifestNumSamplers) { MS_LOG(INFO) << "Doing MindDataTestPipeline-TestManifestNumSamplers."; @@ -292,6 +369,9 @@ TEST_F(MindDataTestPipeline, TestManifestNumSamplers) { iter->Stop(); } +/// Feature: ManifestError. +/// Description: test failure of Manifest Dataset. +/// Expectation: get none piece of data. TEST_F(MindDataTestPipeline, TestManifestError) { MS_LOG(INFO) << "Doing MindDataTestPipeline-TestManifestError."; @@ -324,6 +404,9 @@ TEST_F(MindDataTestPipeline, TestManifestError) { EXPECT_EQ(iter2, nullptr); } +/// Feature: ManifestWithNullSamplerError. +/// Description: test failure of ManifestDataset with null sampler. +/// Expectation: get none piece of data. TEST_F(MindDataTestPipeline, TestManifestWithNullSamplerError) { MS_LOG(INFO) << "Doing MindDataTestPipeline-TestManifestWithNullSamplerError."; std::string file_path = datasets_root_path_ + "/testManifestData/cpp.json"; diff --git a/tests/ut/cpp/dataset/c_api_dataset_sbu_test.cc b/tests/ut/cpp/dataset/c_api_dataset_sbu_test.cc index ec87a4209dc..63857dd49df 100644 --- a/tests/ut/cpp/dataset/c_api_dataset_sbu_test.cc +++ b/tests/ut/cpp/dataset/c_api_dataset_sbu_test.cc @@ -26,6 +26,9 @@ class MindDataTestPipeline : public UT::DatasetOpTesting { protected: }; +/// Feature: SBUDataset. +/// Description: test basic usage of SBUDataset. +/// Expectation: get correct number of data. TEST_F(MindDataTestPipeline, TestSBUDataset) { MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSBUDataset."; @@ -60,6 +63,9 @@ TEST_F(MindDataTestPipeline, TestSBUDataset) { iter->Stop(); } +/// Feature: SBUDatasetWithPipeline. +/// Description: test usage of SBUDataset with pipeline. +/// Expectation: get correct number of data. TEST_F(MindDataTestPipeline, TestSBUDatasetWithPipeline) { MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSBUDatasetWithPipeline."; @@ -115,6 +121,65 @@ TEST_F(MindDataTestPipeline, TestSBUDatasetWithPipeline) { iter->Stop(); } +/// Feature: SBUIteratorOneColumn. +/// Description: test iterator of SBUDataset with only the "image" column. +/// Expectation: get correct data. +TEST_F(MindDataTestPipeline, TestSBUIteratorOneColumn) { +MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSBUIteratorOneColumn."; +// Create a SBU Dataset +std::string folder_path = datasets_root_path_ + "/testSBUDataset/"; +std::shared_ptr ds = SBU(folder_path, true, std::make_shared(false, 5)); +EXPECT_NE(ds, nullptr); + +// Create a Batch operation on ds +int32_t batch_size = 1; +ds = ds->Batch(batch_size); +EXPECT_NE(ds, nullptr); + +// Create an iterator over the result of the above dataset +// Only select "image" column and drop others +std::vector columns = {"image"}; +std::shared_ptr iter = ds->CreateIterator(columns, -1); +EXPECT_NE(iter, nullptr); + +// Iterate the dataset and get each row +std::vector row; +ASSERT_OK(iter->GetNextRow(&row)); + +uint64_t i = 0; +while (row.size() != 0) { +for (auto &v : row) { +MS_LOG(INFO) << "image shape:" << v.Shape(); +} +ASSERT_OK(iter->GetNextRow(&row)); +i++; +} + +EXPECT_EQ(i, 5); + +// Manually terminate the pipeline +iter->Stop(); +} + +/// Feature: SBUIteratorWrongColumn. +/// Description: test iterator of SBUtDataset with wrong column. +/// Expectation: get none piece of data. +TEST_F(MindDataTestPipeline, TestSBUIteratorWrongColumn) { +MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSBUIteratorWrongColumn."; +// Create a SBU Dataset +std::string folder_path = datasets_root_path_ + "/testSBUDataset/"; +std::shared_ptr ds = SBU(folder_path, true, std::make_shared(false, 5)); +EXPECT_NE(ds, nullptr); + +// Pass wrong column name +std::vector columns = {"digital"}; +std::shared_ptr iter = ds->CreateIterator(columns); +EXPECT_EQ(iter, nullptr); +} + +/// Feature: SBUDatasetSize. +/// Description: test usage of get the size of SBUDataset. +/// Expectation: get correct number of data. TEST_F(MindDataTestPipeline, TestGetSBUDatasetSize) { MS_LOG(INFO) << "Doing MindDataTestPipeline-TestGetSBUDatasetSize."; @@ -126,6 +191,9 @@ TEST_F(MindDataTestPipeline, TestGetSBUDatasetSize) { EXPECT_EQ(ds->GetDatasetSize(), 5); } +/// Feature: SBUDatasetGetters. +/// Description: test usage of getters SBUDataset. +/// Expectation: get correct number of data and correct tensor shape. TEST_F(MindDataTestPipeline, TestSBUDatasetGetters) { MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSBUDatasetGetters."; @@ -160,6 +228,9 @@ TEST_F(MindDataTestPipeline, TestSBUDatasetGetters) { EXPECT_EQ(ds->GetDatasetSize(), 5); } +/// Feature: SBUDataFail. +/// Description: test failure of SBUDataset. +/// Expectation: get none piece of data. TEST_F(MindDataTestPipeline, TestSBUDatasetFail) { MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSBUDatasetFail."; @@ -173,6 +244,9 @@ TEST_F(MindDataTestPipeline, TestSBUDatasetFail) { EXPECT_EQ(iter, nullptr); } +/// Feature: SBUDataWithNullSamplerFail. +/// Description: test failure of SBUDataset with null sampler. +/// Expectation: get none piece of data. TEST_F(MindDataTestPipeline, TestSBUDatasetWithNullSamplerFail) { MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSBUDatasetWithNullSamplerFail."; diff --git a/tests/ut/cpp/dataset/c_api_dataset_usps_test.cc b/tests/ut/cpp/dataset/c_api_dataset_usps_test.cc index 275696671d7..c4d7428296b 100644 --- a/tests/ut/cpp/dataset/c_api_dataset_usps_test.cc +++ b/tests/ut/cpp/dataset/c_api_dataset_usps_test.cc @@ -25,6 +25,9 @@ class MindDataTestPipeline : public UT::DatasetOpTesting { protected: }; +/// Feature: USPSTrainDataset. +/// Description: test basic usage of USPSTrainDataset. +/// Expectation: get correct number of data. TEST_F(MindDataTestPipeline, TestUSPSTrainDataset) { MS_LOG(INFO) << "Doing MindDataTestPipeline-TestUSPSTrainDataset."; @@ -59,6 +62,9 @@ TEST_F(MindDataTestPipeline, TestUSPSTrainDataset) { iter->Stop(); } +/// Feature: USPSTestDataset. +/// Description: test basic usage of USPSTestDataset. +/// Expectation: get correct number of data. TEST_F(MindDataTestPipeline, TestUSPSTestDataset) { MS_LOG(INFO) << "Doing MindDataTestPipeline-TestUSPSTestDataset."; @@ -93,6 +99,9 @@ TEST_F(MindDataTestPipeline, TestUSPSTestDataset) { iter->Stop(); } +/// Feature: USPSAllDataset. +/// Description: test basic usage of USPSAllDataset. +/// Expectation: get correct number of data. TEST_F(MindDataTestPipeline, TestUSPSAllDataset) { MS_LOG(INFO) << "Doing MindDataTestPipeline-TestUSPSAllDataset."; @@ -127,6 +136,9 @@ TEST_F(MindDataTestPipeline, TestUSPSAllDataset) { iter->Stop(); } +/// Feature: USPSDatasetWithPipeline. +/// Description: test usage of USPSDataset with pipeline. +/// Expectation: get correct number of data. TEST_F(MindDataTestPipeline, TestUSPSDatasetWithPipeline) { MS_LOG(INFO) << "Doing MindDataTestPipeline-TestUSPSTrainDatasetWithPipeline."; @@ -182,6 +194,67 @@ TEST_F(MindDataTestPipeline, TestUSPSDatasetWithPipeline) { iter->Stop(); } +/// Feature: USPSIteratorOneColumn. +/// Description: test iterator of USPSDataset with only the "image" column. +/// Expectation: get correct data. +TEST_F(MindDataTestPipeline, TestUSPSIteratorOneColumn) { +MS_LOG(INFO) << "Doing MindDataTestPipeline-TestUSPSIteratorOneColumn."; +// Create a USPS Dataset +std::string folder_path = datasets_root_path_ + "/testUSPSDataset/"; +std::shared_ptr ds = USPS(folder_path, "train"); +EXPECT_NE(ds, nullptr); + +// Create a Batch operation on ds +int32_t batch_size = 1; +ds = ds->Batch(batch_size); +EXPECT_NE(ds, nullptr); + +// Create an iterator over the result of the above dataset +// Only select "image" column and drop others +std::vector columns = {"image"}; +std::shared_ptr iter = ds->CreateIterator(columns, -1); +EXPECT_NE(iter, nullptr); + +// Iterate the dataset and get each row +std::vector row; +ASSERT_OK(iter->GetNextRow(&row)); +std::vector expect_image = {1, 16, 16, 1}; + +uint64_t i = 0; +while (row.size() != 0) { +for (auto &v : row) { +MS_LOG(INFO) << "image shape:" << v.Shape(); +EXPECT_EQ(expect_image, v.Shape()); +} +ASSERT_OK(iter->GetNextRow(&row)); +i++; +} + +EXPECT_EQ(i, 3); + +// Manually terminate the pipeline +iter->Stop(); +} + +/// Feature: USPSIteratorWrongColumn. +/// Description: test iterator of USPSDataset with wrong column. +/// Expectation: get none piece of data. +TEST_F(MindDataTestPipeline, TestUSPSIteratorWrongColumn) { +MS_LOG(INFO) << "Doing MindDataTestPipeline-TestUSPSIteratorWrongColumn."; +// Create a USPS Dataset +std::string folder_path = datasets_root_path_ + "/testUSPSDataset/"; +std::shared_ptr ds = USPS(folder_path, "train"); +EXPECT_NE(ds, nullptr); + +// Pass wrong column name +std::vector columns = {"digital"}; +std::shared_ptr iter = ds->CreateIterator(columns); +EXPECT_EQ(iter, nullptr); +} + +/// Feature: GetUSPSDatasetSize. +/// Description: test usage of get the size of USPSDataset. +/// Expectation: get correct number of data. TEST_F(MindDataTestPipeline, TestGetUSPSDatasetSize) { MS_LOG(INFO) << "Doing MindDataTestPipeline-TestGetUSPSTrainDatasetSize."; @@ -193,6 +266,9 @@ TEST_F(MindDataTestPipeline, TestGetUSPSDatasetSize) { EXPECT_EQ(ds->GetDatasetSize(), 3); } +/// Feature: USPSDatasetGetters. +/// Description: test usage of getters USPSDataset. +/// Expectation: get correct number of data and correct tensor shape. TEST_F(MindDataTestPipeline, TestUSPSDatasetGetters) { MS_LOG(INFO) << "Doing MindDataTestPipeline-TestUSPSTrainDatasetGetters."; @@ -227,6 +303,9 @@ TEST_F(MindDataTestPipeline, TestUSPSDatasetGetters) { EXPECT_EQ(ds->GetDatasetSize(), 3); } +/// Feature: USPSDataFail. +/// Description: test failure of USPSDataset. +/// Expectation: get none piece of data. TEST_F(MindDataTestPipeline, TestUSPSDatasetFail) { MS_LOG(INFO) << "Doing MindDataTestPipeline-TestUSPSDatasetFail."; @@ -240,6 +319,9 @@ TEST_F(MindDataTestPipeline, TestUSPSDatasetFail) { EXPECT_EQ(iter, nullptr); } +/// Feature: USPSDatasetWithInvalidUsageFail. +/// Description: test failure of USPSDataset with invalid usage. +/// Expectation: get none piece of data. TEST_F(MindDataTestPipeline, TestUSPSDatasetWithInvalidUsageFail) { MS_LOG(INFO) << "Doing MindDataTestPipeline-TestUSPSDatasetWithInvalidUsageFail.";