add some of ut about dataset IteratorOneColumn

This commit is contained in:
liucunwei 2021-11-03 12:02:00 +08:00
parent ae1ebf1430
commit a86ac516fc
6 changed files with 487 additions and 0 deletions

View File

@ -23,6 +23,9 @@ class MindDataTestPipeline : public UT::DatasetOpTesting {
protected:
};
/// Feature: AlbumDataset.
/// Description: test basic usage of AlbumDataset.
/// Expectation: get correct number of data.
TEST_F(MindDataTestPipeline, TestAlbumBasic) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestAlbumBasic.";
@ -155,6 +158,9 @@ TEST_F(MindDataTestPipeline, TestAlbumWithFullSchema) {
iter->Stop();
}
/// Feature: AlbumDatasetWithPipeline.
/// Description: test usage of AlbumDataset with pipeline.
/// Expectation: get correct number of data.
TEST_F(MindDataTestPipeline, TestAlbumBasicWithPipeline) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestAlbumBasicWithPipeline.";
@ -210,6 +216,69 @@ TEST_F(MindDataTestPipeline, TestAlbumBasicWithPipeline) {
iter->Stop();
}
/// Feature: AlbumIteratorOneColumn.
/// Description: test iterator of AlbumDataset with only the "image" column.
/// Expectation: get correct data.
TEST_F(MindDataTestPipeline, TestAlbumIteratorOneColumn) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestAlbumIteratorOneColumn.";
// Create a Album Dataset
std::string folder_path = datasets_root_path_ + "/testAlbum/images";
std::string schema_file = datasets_root_path_ + "/testAlbum/datasetSchema.json";
std::vector<std::string> column_names = {"image", "label", "id"};
std::shared_ptr<Dataset> ds = Album(folder_path, schema_file, column_names);
EXPECT_NE(ds, nullptr);
// Create a Batch operation on ds
int32_t batch_size = 1;
ds = ds->Batch(batch_size);
EXPECT_NE(ds, nullptr);
// Create an iterator over the result of the above dataset
// Only select "image" column and drop others
std::vector<std::string> columns = {"image"};
std::shared_ptr<Iterator> iter = ds->CreateIterator(columns, -1);
EXPECT_NE(iter, nullptr);
// Iterate the dataset and get each row
std::vector<mindspore::MSTensor> row;
ASSERT_OK(iter->GetNextRow(&row));
uint64_t i = 0;
while (row.size() != 0) {
for (auto &v : row) {
MS_LOG(INFO) << "image shape:" << v.Shape();
}
ASSERT_OK(iter->GetNextRow(&row));
i++;
}
EXPECT_EQ(i, 7);
// Manually terminate the pipeline
iter->Stop();
}
/// Feature: AlbumIteratorWrongColumn.
/// Description: test iterator of AlbumDataset with wrong column.
/// Expectation: get none piece of data.
TEST_F(MindDataTestPipeline, TestAlbumIteratorWrongColumn) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestAlbumIteratorWrongColumn.";
// Create a Album Dataset
std::string folder_path = datasets_root_path_ + "/testAlbum/images";
std::string schema_file = datasets_root_path_ + "/testAlbum/datasetSchema.json";
std::vector<std::string> column_names = {"image", "label", "id"};
std::shared_ptr<Dataset> ds = Album(folder_path, schema_file, column_names);
EXPECT_NE(ds, nullptr);
// Pass wrong column name
std::vector<std::string> columns = {"digital"};
std::shared_ptr<Iterator> iter = ds->CreateIterator(columns);
EXPECT_EQ(iter, nullptr);
}
/// Feature: AlbumDatasetGetters.
/// Description: test usage of getters AlbumDataset.
/// Expectation: get correct number of data and correct tensor shape.
TEST_F(MindDataTestPipeline, TestAlbumGetters) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestAlbumGetters.";
@ -238,6 +307,9 @@ TEST_F(MindDataTestPipeline, TestAlbumGetters) {
EXPECT_EQ(num_samples, 7);
}
/// Feature: AlbumDecode.
/// Description: test usage of AlbumDecode.
/// Expectation: get correct number of data.
TEST_F(MindDataTestPipeline, TestAlbumDecode) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestAlbumDecode.";
std::string folder_path = datasets_root_path_ + "/testAlbum/images";
@ -273,6 +345,9 @@ TEST_F(MindDataTestPipeline, TestAlbumDecode) {
iter->Stop();
}
/// Feature: AlbumNumSampler.
/// Description: test usage of AlbumDataset with num sampler.
/// Expectation: get correct piece of data.
TEST_F(MindDataTestPipeline, TestAlbumNumSamplers) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestAlbumNumSamplers.";
@ -306,6 +381,9 @@ TEST_F(MindDataTestPipeline, TestAlbumNumSamplers) {
iter->Stop();
}
/// Feature: AlbumError.
/// Description: test failure of Album Dataset.
/// Expectation: get none piece of data.
TEST_F(MindDataTestPipeline, TestAlbumError) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestAlbumError.";
std::string folder_path = datasets_root_path_ + "/testAlbum/ima";
@ -321,6 +399,9 @@ TEST_F(MindDataTestPipeline, TestAlbumError) {
EXPECT_EQ(iter, nullptr);
}
/// Feature: AlbumWithNullSamplerError.
/// Description: test failure of Album Dataset.
/// Expectation: get none piece of data.
TEST_F(MindDataTestPipeline, TestAlbumWithNullSamplerError) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestAlbumWithNullSamplerError.";
std::string folder_path = datasets_root_path_ + "/testAlbum/images";
@ -336,6 +417,9 @@ TEST_F(MindDataTestPipeline, TestAlbumWithNullSamplerError) {
EXPECT_EQ(iter, nullptr);
}
/// Feature: AlbumDuplicateColumnNameError.
/// Description: test failure of Album Dataset.
/// Expectation: get none piece of data.
TEST_F(MindDataTestPipeline, TestAlbumDuplicateColumnNameError) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestAlbumDuplicateColumnNameError.";
std::string folder_path = datasets_root_path_ + "/testAlbum/images";

View File

@ -23,6 +23,9 @@ class MindDataTestPipeline : public UT::DatasetOpTesting {
protected:
};
/// Feature: DIV2KDataset.
/// Description: test basic usage of DIV2KDataset.
/// Expectation: get correct number of data.
TEST_F(MindDataTestPipeline, TestDIV2KBasic) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestDIV2KBasic.";
@ -60,6 +63,9 @@ TEST_F(MindDataTestPipeline, TestDIV2KBasic) {
iter->Stop();
}
/// Feature: DIV2KDatasetWithPipeline.
/// Description: test usage of DIV2KDataset with pipeline.
/// Expectation: get correct number of data.
TEST_F(MindDataTestPipeline, TestDIV2KBasicWithPipeline) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestDIV2KBasicWithPipeline.";
@ -118,6 +124,71 @@ TEST_F(MindDataTestPipeline, TestDIV2KBasicWithPipeline) {
iter->Stop();
}
/// Feature: DIV2KIteratorOneColumn.
/// Description: test iterator of DIV2KDataset with only the "hr_image" column.
/// Expectation: get correct data.
TEST_F(MindDataTestPipeline, TestDIV2KIteratorOneColumn) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestDIV2KIteratorOneColumn.";
// Create a DIV2K Dataset
std::string dataset_path = datasets_root_path_ + "/testDIV2KData/div2k";
std::string usage = "train"; // train valid, all
std::string downgrade = "bicubic"; // bicubic, unknown, mild, difficult, wild
int32_t scale = 2; // 2, 3, 4, 8
std::shared_ptr<Dataset> ds = DIV2K(dataset_path, usage, downgrade, scale);
EXPECT_NE(ds, nullptr);
// Create a Batch operation on ds
int32_t batch_size = 1;
ds = ds->Batch(batch_size);
EXPECT_NE(ds, nullptr);
// Create an iterator over the result of the above dataset
// Only select "image" column and drop others
std::vector<std::string> columns = {"hr_image"};
std::shared_ptr<Iterator> iter = ds->CreateIterator(columns, -1);
EXPECT_NE(iter, nullptr);
// Iterate the dataset and get each row
std::vector<mindspore::MSTensor> row;
ASSERT_OK(iter->GetNextRow(&row));
uint64_t i = 0;
while (row.size() != 0) {
for (auto &v : row) {
MS_LOG(INFO) << "image shape:" << v.Shape();
}
ASSERT_OK(iter->GetNextRow(&row));
i++;
}
EXPECT_EQ(i, 5);
// Manually terminate the pipeline
iter->Stop();
}
/// Feature: DIV2KIteratorWrongColumn.
/// Description: test iterator of DIV2KDataset with wrong column.
/// Expectation: get none piece of data.
TEST_F(MindDataTestPipeline, TestDIV2KIteratorWrongColumn) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestDIV2KIteratorWrongColumn.";
// Create a DIV2K Dataset
std::string dataset_path = datasets_root_path_ + "/testDIV2KData/div2k";
std::string usage = "train"; // train valid, all
std::string downgrade = "bicubic"; // bicubic, unknown, mild, difficult, wild
int32_t scale = 2; // 2, 3, 4, 8
std::shared_ptr<Dataset> ds = DIV2K(dataset_path, usage, downgrade, scale);
EXPECT_NE(ds, nullptr);
// Pass wrong column name
std::vector<std::string> columns = {"digital"};
std::shared_ptr<Iterator> iter = ds->CreateIterator(columns);
EXPECT_EQ(iter, nullptr);
}
/// Feature: DIV2KDatasetGetters.
/// Description: test usage of getters DIV2KDataset.
/// Expectation: get correct number of data and correct tensor shape.
TEST_F(MindDataTestPipeline, TestDIV2KGetters) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestDIV2KGetters.";
@ -142,6 +213,9 @@ TEST_F(MindDataTestPipeline, TestDIV2KGetters) {
EXPECT_EQ(ds2->GetColumnNames(), column_names);
}
/// Feature: DIV2KDecode.
/// Description: test usage of DIV2KDecode.
/// Expectation: get correct number of data and correct tensor shape.
TEST_F(MindDataTestPipeline, TestDIV2KDecode) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestDIV2KDecode.";
@ -183,6 +257,9 @@ TEST_F(MindDataTestPipeline, TestDIV2KDecode) {
iter->Stop();
}
/// Feature: DIV2KNumSampler.
/// Description: test usage of DIV2KDataset with num sampler.
/// Expectation: get correct piece of data.
TEST_F(MindDataTestPipeline, TestDIV2KNumSamplers) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestDIV2KNumSamplers.";
@ -223,6 +300,9 @@ TEST_F(MindDataTestPipeline, TestDIV2KNumSamplers) {
iter->Stop();
}
/// Feature: DIV2KError.
/// Description: test failure of DIV2K Dataset.
/// Expectation: get none piece of data.
TEST_F(MindDataTestPipeline, TestDIV2KError) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestDIV2KError.";
@ -286,6 +366,9 @@ TEST_F(MindDataTestPipeline, TestDIV2KError) {
EXPECT_EQ(iter5, nullptr);
}
/// Feature: DIV2KWithNullSamplerError.
/// Description: test failure of DIV2K Dataset.
/// Expectation: get none piece of data.
TEST_F(MindDataTestPipeline, TestDIV2KWithNullSamplerError) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestDIV2KWithNullSamplerError.";

View File

@ -23,6 +23,9 @@ class MindDataTestPipeline : public UT::DatasetOpTesting {
protected:
};
/// Feature: FlickrBasic.
/// Description: test basic usage of FlickrDataset.
/// Expectation: get correct number of data.
TEST_F(MindDataTestPipeline, TestFlickrBasic) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestFlickrBasic.";
@ -56,6 +59,9 @@ TEST_F(MindDataTestPipeline, TestFlickrBasic) {
iter->Stop();
}
/// Feature: FlickrBasicWithPipeline.
/// Description: test usage of FlickrDataset with pipeline.
/// Expectation: get correct number of data.
TEST_F(MindDataTestPipeline, TestFlickrBasicWithPipeline) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestFlickrBasicWithPipeline.";
@ -110,6 +116,66 @@ TEST_F(MindDataTestPipeline, TestFlickrBasicWithPipeline) {
iter->Stop();
}
/// Feature: FlickrIteratorOneColumn.
/// Description: test iterator of FlickrDataset with only the "image" column.
/// Expectation: get correct data.
TEST_F(MindDataTestPipeline, TestFlickrIteratorOneColumn) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestFlickrIteratorOneColumn.";
std::string dataset_path = datasets_root_path_ + "/testFlickrData/flickr30k/flickr30k-images";
std::string file_path = datasets_root_path_ + "/testFlickrData/flickr30k/test1.token";
// Create a Flickr30k Dataset
std::shared_ptr<Dataset> ds = Flickr(dataset_path, file_path);
EXPECT_NE(ds, nullptr);
// Create a Batch operation on ds
int32_t batch_size = 1;
ds = ds->Batch(batch_size);
EXPECT_NE(ds, nullptr);
// Create an iterator over the result of the above dataset
// Only select "image" column and drop others
std::vector<std::string> columns = {"image"};
std::shared_ptr<Iterator> iter = ds->CreateIterator(columns, -1);
EXPECT_NE(iter, nullptr);
// Iterate the dataset and get each row
std::vector<mindspore::MSTensor> row;
ASSERT_OK(iter->GetNextRow(&row));
uint64_t i = 0;
while (row.size() != 0) {
for (auto &v : row) {
MS_LOG(INFO) << "image shape:" << v.Shape();
}
ASSERT_OK(iter->GetNextRow(&row));
i++;
}
EXPECT_EQ(i, 2);
// Manually terminate the pipeline
iter->Stop();
}
/// Feature: FlickrIteratorWrongColumn.
/// Description: test iterator of FlickrDataset with wrong column.
/// Expectation: get none piece of data.
TEST_F(MindDataTestPipeline, TestFlickrIteratorWrongColumn) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestFlickrIteratorWrongColumn.";
std::string dataset_path = datasets_root_path_ + "/testFlickrData/flickr30k/flickr30k-images";
std::string file_path = datasets_root_path_ + "/testFlickrData/flickr30k/test1.token";
// Create a Flickr30k Dataset
std::shared_ptr<Dataset> ds = Flickr(dataset_path, file_path);
EXPECT_NE(ds, nullptr);
// Pass wrong column name
std::vector<std::string> columns = {"digital"};
std::shared_ptr<Iterator> iter = ds->CreateIterator(columns);
EXPECT_EQ(iter, nullptr);
}
TEST_F(MindDataTestPipeline, TestFlickrGetters) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestFlickrGetters.";
@ -131,6 +197,9 @@ TEST_F(MindDataTestPipeline, TestFlickrGetters) {
EXPECT_EQ(ds2->GetColumnNames(), column_names);
}
/// Feature: FlickrAnnotations.
/// Description: test usage of FlickrAnnotations.
/// Expectation: get correct number of data.
TEST_F(MindDataTestPipeline, TestFlickrAnnotations) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestFlickrGetters.";
@ -177,6 +246,9 @@ TEST_F(MindDataTestPipeline, TestFlickrAnnotations) {
iter->Stop();
}
/// Feature: FlickrDecode.
/// Description: test usage of FlickrDecode.
/// Expectation: get correct number of data.
TEST_F(MindDataTestPipeline, TestFlickrDecode) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestFlickrDecode.";
@ -212,6 +284,9 @@ TEST_F(MindDataTestPipeline, TestFlickrDecode) {
iter->Stop();
}
/// Feature: FlickrNumSamplers.
/// Description: test usage of FlickrDataset with num sampler.
/// Expectation: get correct piece of data.
TEST_F(MindDataTestPipeline, TestFlickrNumSamplers) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestFlickrNumSamplers.";
@ -250,6 +325,9 @@ TEST_F(MindDataTestPipeline, TestFlickrNumSamplers) {
iter->Stop();
}
/// Feature: FlickrError.
/// Description: test failure of Flickr Dataset.
/// Expectation: get none piece of data.
TEST_F(MindDataTestPipeline, TestFlickrError) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestFlickrError.";
@ -292,6 +370,9 @@ TEST_F(MindDataTestPipeline, TestFlickrError) {
EXPECT_EQ(iter3, nullptr);
}
/// Feature: FlickrWithNullSamplerError.
/// Description: test failure of FlickrDataset with null sampler.
/// Expectation: get none piece of data.
TEST_F(MindDataTestPipeline, TestFlickrWithNullSamplerError) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestFlickrWithNullSamplerError.";

View File

@ -23,6 +23,9 @@ class MindDataTestPipeline : public UT::DatasetOpTesting {
protected:
};
/// Feature: ManifestBasic.
/// Description: test basic usage of ManifestDataset.
/// Expectation: get correct number of data.
TEST_F(MindDataTestPipeline, TestManifestBasic) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestManifestBasic.";
@ -54,6 +57,9 @@ TEST_F(MindDataTestPipeline, TestManifestBasic) {
iter->Stop();
}
/// Feature: ManifestBasicWithPipeline.
/// Description: test usage of ManifestDataset with pipeline.
/// Expectation: get correct number of data.
TEST_F(MindDataTestPipeline, TestManifestBasicWithPipeline) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestManifestBasicWithPipeline.";
@ -106,6 +112,65 @@ TEST_F(MindDataTestPipeline, TestManifestBasicWithPipeline) {
iter->Stop();
}
/// Feature: ManifestIteratorOneColumn.
/// Description: test iterator of ManifestDataset with only the "image" column.
/// Expectation: get correct data.
TEST_F(MindDataTestPipeline, TestManifestIteratorOneColumn) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestManifestIteratorOneColumn.";
std::string file_path = datasets_root_path_ + "/testManifestData/cpp.json";
// Create a Manifest Dataset
std::shared_ptr<Dataset> ds = Manifest(file_path);
EXPECT_NE(ds, nullptr);
// Create a Batch operation on ds
int32_t batch_size = 1;
ds = ds->Batch(batch_size);
EXPECT_NE(ds, nullptr);
// Create an iterator over the result of the above dataset
// Only select "image" column and drop others
std::vector<std::string> columns = {"image"};
std::shared_ptr<Iterator> iter = ds->CreateIterator(columns, -1);
EXPECT_NE(iter, nullptr);
// Iterate the dataset and get each row
std::vector<mindspore::MSTensor> row;
ASSERT_OK(iter->GetNextRow(&row));
uint64_t i = 0;
while (row.size() != 0) {
for (auto &v : row) {
MS_LOG(INFO) << "image shape:" << v.Shape();
}
ASSERT_OK(iter->GetNextRow(&row));
i++;
}
EXPECT_EQ(i, 2);
// Manually terminate the pipeline
iter->Stop();
}
/// Feature: ManifestIteratorWrongColumn.
/// Description: test iterator of ManifestDataset with wrong column.
/// Expectation: get none piece of data.
TEST_F(MindDataTestPipeline, TestManifestIteratorWrongColumn) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestManifestIteratorWrongColumn.";
std::string file_path = datasets_root_path_ + "/testManifestData/cpp.json";
// Create a Manifest Dataset
std::shared_ptr<Dataset> ds = Manifest(file_path);
EXPECT_NE(ds, nullptr);
// Pass wrong column name
std::vector<std::string> columns = {"digital"};
std::shared_ptr<Iterator> iter = ds->CreateIterator(columns);
EXPECT_EQ(iter, nullptr);
}
/// Feature: ManifestGetters.
/// Description: test usage of getters ManifestDataset.
/// Expectation: get correct number of data and correct tensor shape.
TEST_F(MindDataTestPipeline, TestManifestGetters) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestManifestGetters.";
@ -142,6 +207,9 @@ TEST_F(MindDataTestPipeline, TestManifestGetters) {
EXPECT_EQ(class_index2[2].second[0], 2);
}
/// Feature: ManifestDecode.
/// Description: test usage of ManifestDecode.
/// Expectation: get correct number of data.
TEST_F(MindDataTestPipeline, TestManifestDecode) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestManifestDecode.";
@ -176,6 +244,9 @@ TEST_F(MindDataTestPipeline, TestManifestDecode) {
iter->Stop();
}
/// Feature: ManifestEval.
/// Description: test usage of ManifestEval.
/// Expectation: get correct number of data.
TEST_F(MindDataTestPipeline, TestManifestEval) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestManifestEval.";
@ -207,6 +278,9 @@ TEST_F(MindDataTestPipeline, TestManifestEval) {
iter->Stop();
}
/// Feature: ManifestClassIndex.
/// Description: test usage of ManifestClassIndex.
/// Expectation: get correct number of data.
TEST_F(MindDataTestPipeline, TestManifestClassIndex) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestManifestClassIndex.";
@ -261,6 +335,9 @@ TEST_F(MindDataTestPipeline, TestManifestClassIndex) {
iter->Stop();
}
/// Feature: ManifestNumSamplers.
/// Description: test usage of ManifestDataset with num sampler.
/// Expectation: get correct piece of data.
TEST_F(MindDataTestPipeline, TestManifestNumSamplers) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestManifestNumSamplers.";
@ -292,6 +369,9 @@ TEST_F(MindDataTestPipeline, TestManifestNumSamplers) {
iter->Stop();
}
/// Feature: ManifestError.
/// Description: test failure of Manifest Dataset.
/// Expectation: get none piece of data.
TEST_F(MindDataTestPipeline, TestManifestError) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestManifestError.";
@ -324,6 +404,9 @@ TEST_F(MindDataTestPipeline, TestManifestError) {
EXPECT_EQ(iter2, nullptr);
}
/// Feature: ManifestWithNullSamplerError.
/// Description: test failure of ManifestDataset with null sampler.
/// Expectation: get none piece of data.
TEST_F(MindDataTestPipeline, TestManifestWithNullSamplerError) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestManifestWithNullSamplerError.";
std::string file_path = datasets_root_path_ + "/testManifestData/cpp.json";

View File

@ -26,6 +26,9 @@ class MindDataTestPipeline : public UT::DatasetOpTesting {
protected:
};
/// Feature: SBUDataset.
/// Description: test basic usage of SBUDataset.
/// Expectation: get correct number of data.
TEST_F(MindDataTestPipeline, TestSBUDataset) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSBUDataset.";
@ -60,6 +63,9 @@ TEST_F(MindDataTestPipeline, TestSBUDataset) {
iter->Stop();
}
/// Feature: SBUDatasetWithPipeline.
/// Description: test usage of SBUDataset with pipeline.
/// Expectation: get correct number of data.
TEST_F(MindDataTestPipeline, TestSBUDatasetWithPipeline) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSBUDatasetWithPipeline.";
@ -115,6 +121,65 @@ TEST_F(MindDataTestPipeline, TestSBUDatasetWithPipeline) {
iter->Stop();
}
/// Feature: SBUIteratorOneColumn.
/// Description: test iterator of SBUDataset with only the "image" column.
/// Expectation: get correct data.
TEST_F(MindDataTestPipeline, TestSBUIteratorOneColumn) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSBUIteratorOneColumn.";
// Create a SBU Dataset
std::string folder_path = datasets_root_path_ + "/testSBUDataset/";
std::shared_ptr<Dataset> ds = SBU(folder_path, true, std::make_shared<RandomSampler>(false, 5));
EXPECT_NE(ds, nullptr);
// Create a Batch operation on ds
int32_t batch_size = 1;
ds = ds->Batch(batch_size);
EXPECT_NE(ds, nullptr);
// Create an iterator over the result of the above dataset
// Only select "image" column and drop others
std::vector<std::string> columns = {"image"};
std::shared_ptr<Iterator> iter = ds->CreateIterator(columns, -1);
EXPECT_NE(iter, nullptr);
// Iterate the dataset and get each row
std::vector<mindspore::MSTensor> row;
ASSERT_OK(iter->GetNextRow(&row));
uint64_t i = 0;
while (row.size() != 0) {
for (auto &v : row) {
MS_LOG(INFO) << "image shape:" << v.Shape();
}
ASSERT_OK(iter->GetNextRow(&row));
i++;
}
EXPECT_EQ(i, 5);
// Manually terminate the pipeline
iter->Stop();
}
/// Feature: SBUIteratorWrongColumn.
/// Description: test iterator of SBUtDataset with wrong column.
/// Expectation: get none piece of data.
TEST_F(MindDataTestPipeline, TestSBUIteratorWrongColumn) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSBUIteratorWrongColumn.";
// Create a SBU Dataset
std::string folder_path = datasets_root_path_ + "/testSBUDataset/";
std::shared_ptr<Dataset> ds = SBU(folder_path, true, std::make_shared<RandomSampler>(false, 5));
EXPECT_NE(ds, nullptr);
// Pass wrong column name
std::vector<std::string> columns = {"digital"};
std::shared_ptr<Iterator> iter = ds->CreateIterator(columns);
EXPECT_EQ(iter, nullptr);
}
/// Feature: SBUDatasetSize.
/// Description: test usage of get the size of SBUDataset.
/// Expectation: get correct number of data.
TEST_F(MindDataTestPipeline, TestGetSBUDatasetSize) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestGetSBUDatasetSize.";
@ -126,6 +191,9 @@ TEST_F(MindDataTestPipeline, TestGetSBUDatasetSize) {
EXPECT_EQ(ds->GetDatasetSize(), 5);
}
/// Feature: SBUDatasetGetters.
/// Description: test usage of getters SBUDataset.
/// Expectation: get correct number of data and correct tensor shape.
TEST_F(MindDataTestPipeline, TestSBUDatasetGetters) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSBUDatasetGetters.";
@ -160,6 +228,9 @@ TEST_F(MindDataTestPipeline, TestSBUDatasetGetters) {
EXPECT_EQ(ds->GetDatasetSize(), 5);
}
/// Feature: SBUDataFail.
/// Description: test failure of SBUDataset.
/// Expectation: get none piece of data.
TEST_F(MindDataTestPipeline, TestSBUDatasetFail) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSBUDatasetFail.";
@ -173,6 +244,9 @@ TEST_F(MindDataTestPipeline, TestSBUDatasetFail) {
EXPECT_EQ(iter, nullptr);
}
/// Feature: SBUDataWithNullSamplerFail.
/// Description: test failure of SBUDataset with null sampler.
/// Expectation: get none piece of data.
TEST_F(MindDataTestPipeline, TestSBUDatasetWithNullSamplerFail) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSBUDatasetWithNullSamplerFail.";

View File

@ -25,6 +25,9 @@ class MindDataTestPipeline : public UT::DatasetOpTesting {
protected:
};
/// Feature: USPSTrainDataset.
/// Description: test basic usage of USPSTrainDataset.
/// Expectation: get correct number of data.
TEST_F(MindDataTestPipeline, TestUSPSTrainDataset) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestUSPSTrainDataset.";
@ -59,6 +62,9 @@ TEST_F(MindDataTestPipeline, TestUSPSTrainDataset) {
iter->Stop();
}
/// Feature: USPSTestDataset.
/// Description: test basic usage of USPSTestDataset.
/// Expectation: get correct number of data.
TEST_F(MindDataTestPipeline, TestUSPSTestDataset) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestUSPSTestDataset.";
@ -93,6 +99,9 @@ TEST_F(MindDataTestPipeline, TestUSPSTestDataset) {
iter->Stop();
}
/// Feature: USPSAllDataset.
/// Description: test basic usage of USPSAllDataset.
/// Expectation: get correct number of data.
TEST_F(MindDataTestPipeline, TestUSPSAllDataset) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestUSPSAllDataset.";
@ -127,6 +136,9 @@ TEST_F(MindDataTestPipeline, TestUSPSAllDataset) {
iter->Stop();
}
/// Feature: USPSDatasetWithPipeline.
/// Description: test usage of USPSDataset with pipeline.
/// Expectation: get correct number of data.
TEST_F(MindDataTestPipeline, TestUSPSDatasetWithPipeline) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestUSPSTrainDatasetWithPipeline.";
@ -182,6 +194,67 @@ TEST_F(MindDataTestPipeline, TestUSPSDatasetWithPipeline) {
iter->Stop();
}
/// Feature: USPSIteratorOneColumn.
/// Description: test iterator of USPSDataset with only the "image" column.
/// Expectation: get correct data.
TEST_F(MindDataTestPipeline, TestUSPSIteratorOneColumn) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestUSPSIteratorOneColumn.";
// Create a USPS Dataset
std::string folder_path = datasets_root_path_ + "/testUSPSDataset/";
std::shared_ptr<Dataset> ds = USPS(folder_path, "train");
EXPECT_NE(ds, nullptr);
// Create a Batch operation on ds
int32_t batch_size = 1;
ds = ds->Batch(batch_size);
EXPECT_NE(ds, nullptr);
// Create an iterator over the result of the above dataset
// Only select "image" column and drop others
std::vector<std::string> columns = {"image"};
std::shared_ptr<Iterator> iter = ds->CreateIterator(columns, -1);
EXPECT_NE(iter, nullptr);
// Iterate the dataset and get each row
std::vector<mindspore::MSTensor> row;
ASSERT_OK(iter->GetNextRow(&row));
std::vector<int64_t> expect_image = {1, 16, 16, 1};
uint64_t i = 0;
while (row.size() != 0) {
for (auto &v : row) {
MS_LOG(INFO) << "image shape:" << v.Shape();
EXPECT_EQ(expect_image, v.Shape());
}
ASSERT_OK(iter->GetNextRow(&row));
i++;
}
EXPECT_EQ(i, 3);
// Manually terminate the pipeline
iter->Stop();
}
/// Feature: USPSIteratorWrongColumn.
/// Description: test iterator of USPSDataset with wrong column.
/// Expectation: get none piece of data.
TEST_F(MindDataTestPipeline, TestUSPSIteratorWrongColumn) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestUSPSIteratorWrongColumn.";
// Create a USPS Dataset
std::string folder_path = datasets_root_path_ + "/testUSPSDataset/";
std::shared_ptr<Dataset> ds = USPS(folder_path, "train");
EXPECT_NE(ds, nullptr);
// Pass wrong column name
std::vector<std::string> columns = {"digital"};
std::shared_ptr<Iterator> iter = ds->CreateIterator(columns);
EXPECT_EQ(iter, nullptr);
}
/// Feature: GetUSPSDatasetSize.
/// Description: test usage of get the size of USPSDataset.
/// Expectation: get correct number of data.
TEST_F(MindDataTestPipeline, TestGetUSPSDatasetSize) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestGetUSPSTrainDatasetSize.";
@ -193,6 +266,9 @@ TEST_F(MindDataTestPipeline, TestGetUSPSDatasetSize) {
EXPECT_EQ(ds->GetDatasetSize(), 3);
}
/// Feature: USPSDatasetGetters.
/// Description: test usage of getters USPSDataset.
/// Expectation: get correct number of data and correct tensor shape.
TEST_F(MindDataTestPipeline, TestUSPSDatasetGetters) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestUSPSTrainDatasetGetters.";
@ -227,6 +303,9 @@ TEST_F(MindDataTestPipeline, TestUSPSDatasetGetters) {
EXPECT_EQ(ds->GetDatasetSize(), 3);
}
/// Feature: USPSDataFail.
/// Description: test failure of USPSDataset.
/// Expectation: get none piece of data.
TEST_F(MindDataTestPipeline, TestUSPSDatasetFail) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestUSPSDatasetFail.";
@ -240,6 +319,9 @@ TEST_F(MindDataTestPipeline, TestUSPSDatasetFail) {
EXPECT_EQ(iter, nullptr);
}
/// Feature: USPSDatasetWithInvalidUsageFail.
/// Description: test failure of USPSDataset with invalid usage.
/// Expectation: get none piece of data.
TEST_F(MindDataTestPipeline, TestUSPSDatasetWithInvalidUsageFail) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestUSPSDatasetWithInvalidUsageFail.";