forked from huawei/mindspore2022
add some of ut about dataset IteratorOneColumn
This commit is contained in:
parent
ae1ebf1430
commit
a86ac516fc
|
|
@ -23,6 +23,9 @@ class MindDataTestPipeline : public UT::DatasetOpTesting {
|
|||
protected:
|
||||
};
|
||||
|
||||
/// Feature: AlbumDataset.
|
||||
/// Description: test basic usage of AlbumDataset.
|
||||
/// Expectation: get correct number of data.
|
||||
TEST_F(MindDataTestPipeline, TestAlbumBasic) {
|
||||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestAlbumBasic.";
|
||||
|
||||
|
|
@ -155,6 +158,9 @@ TEST_F(MindDataTestPipeline, TestAlbumWithFullSchema) {
|
|||
iter->Stop();
|
||||
}
|
||||
|
||||
/// Feature: AlbumDatasetWithPipeline.
|
||||
/// Description: test usage of AlbumDataset with pipeline.
|
||||
/// Expectation: get correct number of data.
|
||||
TEST_F(MindDataTestPipeline, TestAlbumBasicWithPipeline) {
|
||||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestAlbumBasicWithPipeline.";
|
||||
|
||||
|
|
@ -210,6 +216,69 @@ TEST_F(MindDataTestPipeline, TestAlbumBasicWithPipeline) {
|
|||
iter->Stop();
|
||||
}
|
||||
|
||||
/// Feature: AlbumIteratorOneColumn.
|
||||
/// Description: test iterator of AlbumDataset with only the "image" column.
|
||||
/// Expectation: get correct data.
|
||||
TEST_F(MindDataTestPipeline, TestAlbumIteratorOneColumn) {
|
||||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestAlbumIteratorOneColumn.";
|
||||
// Create a Album Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testAlbum/images";
|
||||
std::string schema_file = datasets_root_path_ + "/testAlbum/datasetSchema.json";
|
||||
std::vector<std::string> column_names = {"image", "label", "id"};
|
||||
std::shared_ptr<Dataset> ds = Album(folder_path, schema_file, column_names);
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create a Batch operation on ds
|
||||
int32_t batch_size = 1;
|
||||
ds = ds->Batch(batch_size);
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create an iterator over the result of the above dataset
|
||||
// Only select "image" column and drop others
|
||||
std::vector<std::string> columns = {"image"};
|
||||
std::shared_ptr<Iterator> iter = ds->CreateIterator(columns, -1);
|
||||
EXPECT_NE(iter, nullptr);
|
||||
|
||||
// Iterate the dataset and get each row
|
||||
std::vector<mindspore::MSTensor> row;
|
||||
ASSERT_OK(iter->GetNextRow(&row));
|
||||
|
||||
uint64_t i = 0;
|
||||
while (row.size() != 0) {
|
||||
for (auto &v : row) {
|
||||
MS_LOG(INFO) << "image shape:" << v.Shape();
|
||||
}
|
||||
ASSERT_OK(iter->GetNextRow(&row));
|
||||
i++;
|
||||
}
|
||||
|
||||
EXPECT_EQ(i, 7);
|
||||
|
||||
// Manually terminate the pipeline
|
||||
iter->Stop();
|
||||
}
|
||||
|
||||
/// Feature: AlbumIteratorWrongColumn.
|
||||
/// Description: test iterator of AlbumDataset with wrong column.
|
||||
/// Expectation: get none piece of data.
|
||||
TEST_F(MindDataTestPipeline, TestAlbumIteratorWrongColumn) {
|
||||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestAlbumIteratorWrongColumn.";
|
||||
// Create a Album Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testAlbum/images";
|
||||
std::string schema_file = datasets_root_path_ + "/testAlbum/datasetSchema.json";
|
||||
std::vector<std::string> column_names = {"image", "label", "id"};
|
||||
std::shared_ptr<Dataset> ds = Album(folder_path, schema_file, column_names);
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Pass wrong column name
|
||||
std::vector<std::string> columns = {"digital"};
|
||||
std::shared_ptr<Iterator> iter = ds->CreateIterator(columns);
|
||||
EXPECT_EQ(iter, nullptr);
|
||||
}
|
||||
|
||||
/// Feature: AlbumDatasetGetters.
|
||||
/// Description: test usage of getters AlbumDataset.
|
||||
/// Expectation: get correct number of data and correct tensor shape.
|
||||
TEST_F(MindDataTestPipeline, TestAlbumGetters) {
|
||||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestAlbumGetters.";
|
||||
|
||||
|
|
@ -238,6 +307,9 @@ TEST_F(MindDataTestPipeline, TestAlbumGetters) {
|
|||
EXPECT_EQ(num_samples, 7);
|
||||
}
|
||||
|
||||
/// Feature: AlbumDecode.
|
||||
/// Description: test usage of AlbumDecode.
|
||||
/// Expectation: get correct number of data.
|
||||
TEST_F(MindDataTestPipeline, TestAlbumDecode) {
|
||||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestAlbumDecode.";
|
||||
std::string folder_path = datasets_root_path_ + "/testAlbum/images";
|
||||
|
|
@ -273,6 +345,9 @@ TEST_F(MindDataTestPipeline, TestAlbumDecode) {
|
|||
iter->Stop();
|
||||
}
|
||||
|
||||
/// Feature: AlbumNumSampler.
|
||||
/// Description: test usage of AlbumDataset with num sampler.
|
||||
/// Expectation: get correct piece of data.
|
||||
TEST_F(MindDataTestPipeline, TestAlbumNumSamplers) {
|
||||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestAlbumNumSamplers.";
|
||||
|
||||
|
|
@ -306,6 +381,9 @@ TEST_F(MindDataTestPipeline, TestAlbumNumSamplers) {
|
|||
iter->Stop();
|
||||
}
|
||||
|
||||
/// Feature: AlbumError.
|
||||
/// Description: test failure of Album Dataset.
|
||||
/// Expectation: get none piece of data.
|
||||
TEST_F(MindDataTestPipeline, TestAlbumError) {
|
||||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestAlbumError.";
|
||||
std::string folder_path = datasets_root_path_ + "/testAlbum/ima";
|
||||
|
|
@ -321,6 +399,9 @@ TEST_F(MindDataTestPipeline, TestAlbumError) {
|
|||
EXPECT_EQ(iter, nullptr);
|
||||
}
|
||||
|
||||
/// Feature: AlbumWithNullSamplerError.
|
||||
/// Description: test failure of Album Dataset.
|
||||
/// Expectation: get none piece of data.
|
||||
TEST_F(MindDataTestPipeline, TestAlbumWithNullSamplerError) {
|
||||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestAlbumWithNullSamplerError.";
|
||||
std::string folder_path = datasets_root_path_ + "/testAlbum/images";
|
||||
|
|
@ -336,6 +417,9 @@ TEST_F(MindDataTestPipeline, TestAlbumWithNullSamplerError) {
|
|||
EXPECT_EQ(iter, nullptr);
|
||||
}
|
||||
|
||||
/// Feature: AlbumDuplicateColumnNameError.
|
||||
/// Description: test failure of Album Dataset.
|
||||
/// Expectation: get none piece of data.
|
||||
TEST_F(MindDataTestPipeline, TestAlbumDuplicateColumnNameError) {
|
||||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestAlbumDuplicateColumnNameError.";
|
||||
std::string folder_path = datasets_root_path_ + "/testAlbum/images";
|
||||
|
|
|
|||
|
|
@ -23,6 +23,9 @@ class MindDataTestPipeline : public UT::DatasetOpTesting {
|
|||
protected:
|
||||
};
|
||||
|
||||
/// Feature: DIV2KDataset.
|
||||
/// Description: test basic usage of DIV2KDataset.
|
||||
/// Expectation: get correct number of data.
|
||||
TEST_F(MindDataTestPipeline, TestDIV2KBasic) {
|
||||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestDIV2KBasic.";
|
||||
|
||||
|
|
@ -60,6 +63,9 @@ TEST_F(MindDataTestPipeline, TestDIV2KBasic) {
|
|||
iter->Stop();
|
||||
}
|
||||
|
||||
/// Feature: DIV2KDatasetWithPipeline.
|
||||
/// Description: test usage of DIV2KDataset with pipeline.
|
||||
/// Expectation: get correct number of data.
|
||||
TEST_F(MindDataTestPipeline, TestDIV2KBasicWithPipeline) {
|
||||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestDIV2KBasicWithPipeline.";
|
||||
|
||||
|
|
@ -118,6 +124,71 @@ TEST_F(MindDataTestPipeline, TestDIV2KBasicWithPipeline) {
|
|||
iter->Stop();
|
||||
}
|
||||
|
||||
/// Feature: DIV2KIteratorOneColumn.
|
||||
/// Description: test iterator of DIV2KDataset with only the "hr_image" column.
|
||||
/// Expectation: get correct data.
|
||||
TEST_F(MindDataTestPipeline, TestDIV2KIteratorOneColumn) {
|
||||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestDIV2KIteratorOneColumn.";
|
||||
// Create a DIV2K Dataset
|
||||
std::string dataset_path = datasets_root_path_ + "/testDIV2KData/div2k";
|
||||
std::string usage = "train"; // train valid, all
|
||||
std::string downgrade = "bicubic"; // bicubic, unknown, mild, difficult, wild
|
||||
int32_t scale = 2; // 2, 3, 4, 8
|
||||
std::shared_ptr<Dataset> ds = DIV2K(dataset_path, usage, downgrade, scale);
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create a Batch operation on ds
|
||||
int32_t batch_size = 1;
|
||||
ds = ds->Batch(batch_size);
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create an iterator over the result of the above dataset
|
||||
// Only select "image" column and drop others
|
||||
std::vector<std::string> columns = {"hr_image"};
|
||||
std::shared_ptr<Iterator> iter = ds->CreateIterator(columns, -1);
|
||||
EXPECT_NE(iter, nullptr);
|
||||
|
||||
// Iterate the dataset and get each row
|
||||
std::vector<mindspore::MSTensor> row;
|
||||
ASSERT_OK(iter->GetNextRow(&row));
|
||||
|
||||
uint64_t i = 0;
|
||||
while (row.size() != 0) {
|
||||
for (auto &v : row) {
|
||||
MS_LOG(INFO) << "image shape:" << v.Shape();
|
||||
}
|
||||
ASSERT_OK(iter->GetNextRow(&row));
|
||||
i++;
|
||||
}
|
||||
|
||||
EXPECT_EQ(i, 5);
|
||||
|
||||
// Manually terminate the pipeline
|
||||
iter->Stop();
|
||||
}
|
||||
|
||||
/// Feature: DIV2KIteratorWrongColumn.
|
||||
/// Description: test iterator of DIV2KDataset with wrong column.
|
||||
/// Expectation: get none piece of data.
|
||||
TEST_F(MindDataTestPipeline, TestDIV2KIteratorWrongColumn) {
|
||||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestDIV2KIteratorWrongColumn.";
|
||||
// Create a DIV2K Dataset
|
||||
std::string dataset_path = datasets_root_path_ + "/testDIV2KData/div2k";
|
||||
std::string usage = "train"; // train valid, all
|
||||
std::string downgrade = "bicubic"; // bicubic, unknown, mild, difficult, wild
|
||||
int32_t scale = 2; // 2, 3, 4, 8
|
||||
std::shared_ptr<Dataset> ds = DIV2K(dataset_path, usage, downgrade, scale);
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Pass wrong column name
|
||||
std::vector<std::string> columns = {"digital"};
|
||||
std::shared_ptr<Iterator> iter = ds->CreateIterator(columns);
|
||||
EXPECT_EQ(iter, nullptr);
|
||||
}
|
||||
|
||||
/// Feature: DIV2KDatasetGetters.
|
||||
/// Description: test usage of getters DIV2KDataset.
|
||||
/// Expectation: get correct number of data and correct tensor shape.
|
||||
TEST_F(MindDataTestPipeline, TestDIV2KGetters) {
|
||||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestDIV2KGetters.";
|
||||
|
||||
|
|
@ -142,6 +213,9 @@ TEST_F(MindDataTestPipeline, TestDIV2KGetters) {
|
|||
EXPECT_EQ(ds2->GetColumnNames(), column_names);
|
||||
}
|
||||
|
||||
/// Feature: DIV2KDecode.
|
||||
/// Description: test usage of DIV2KDecode.
|
||||
/// Expectation: get correct number of data and correct tensor shape.
|
||||
TEST_F(MindDataTestPipeline, TestDIV2KDecode) {
|
||||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestDIV2KDecode.";
|
||||
|
||||
|
|
@ -183,6 +257,9 @@ TEST_F(MindDataTestPipeline, TestDIV2KDecode) {
|
|||
iter->Stop();
|
||||
}
|
||||
|
||||
/// Feature: DIV2KNumSampler.
|
||||
/// Description: test usage of DIV2KDataset with num sampler.
|
||||
/// Expectation: get correct piece of data.
|
||||
TEST_F(MindDataTestPipeline, TestDIV2KNumSamplers) {
|
||||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestDIV2KNumSamplers.";
|
||||
|
||||
|
|
@ -223,6 +300,9 @@ TEST_F(MindDataTestPipeline, TestDIV2KNumSamplers) {
|
|||
iter->Stop();
|
||||
}
|
||||
|
||||
/// Feature: DIV2KError.
|
||||
/// Description: test failure of DIV2K Dataset.
|
||||
/// Expectation: get none piece of data.
|
||||
TEST_F(MindDataTestPipeline, TestDIV2KError) {
|
||||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestDIV2KError.";
|
||||
|
||||
|
|
@ -286,6 +366,9 @@ TEST_F(MindDataTestPipeline, TestDIV2KError) {
|
|||
EXPECT_EQ(iter5, nullptr);
|
||||
}
|
||||
|
||||
/// Feature: DIV2KWithNullSamplerError.
|
||||
/// Description: test failure of DIV2K Dataset.
|
||||
/// Expectation: get none piece of data.
|
||||
TEST_F(MindDataTestPipeline, TestDIV2KWithNullSamplerError) {
|
||||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestDIV2KWithNullSamplerError.";
|
||||
|
||||
|
|
|
|||
|
|
@ -23,6 +23,9 @@ class MindDataTestPipeline : public UT::DatasetOpTesting {
|
|||
protected:
|
||||
};
|
||||
|
||||
/// Feature: FlickrBasic.
|
||||
/// Description: test basic usage of FlickrDataset.
|
||||
/// Expectation: get correct number of data.
|
||||
TEST_F(MindDataTestPipeline, TestFlickrBasic) {
|
||||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestFlickrBasic.";
|
||||
|
||||
|
|
@ -56,6 +59,9 @@ TEST_F(MindDataTestPipeline, TestFlickrBasic) {
|
|||
iter->Stop();
|
||||
}
|
||||
|
||||
/// Feature: FlickrBasicWithPipeline.
|
||||
/// Description: test usage of FlickrDataset with pipeline.
|
||||
/// Expectation: get correct number of data.
|
||||
TEST_F(MindDataTestPipeline, TestFlickrBasicWithPipeline) {
|
||||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestFlickrBasicWithPipeline.";
|
||||
|
||||
|
|
@ -110,6 +116,66 @@ TEST_F(MindDataTestPipeline, TestFlickrBasicWithPipeline) {
|
|||
iter->Stop();
|
||||
}
|
||||
|
||||
/// Feature: FlickrIteratorOneColumn.
|
||||
/// Description: test iterator of FlickrDataset with only the "image" column.
|
||||
/// Expectation: get correct data.
|
||||
TEST_F(MindDataTestPipeline, TestFlickrIteratorOneColumn) {
|
||||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestFlickrIteratorOneColumn.";
|
||||
std::string dataset_path = datasets_root_path_ + "/testFlickrData/flickr30k/flickr30k-images";
|
||||
std::string file_path = datasets_root_path_ + "/testFlickrData/flickr30k/test1.token";
|
||||
|
||||
// Create a Flickr30k Dataset
|
||||
std::shared_ptr<Dataset> ds = Flickr(dataset_path, file_path);
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create a Batch operation on ds
|
||||
int32_t batch_size = 1;
|
||||
ds = ds->Batch(batch_size);
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create an iterator over the result of the above dataset
|
||||
// Only select "image" column and drop others
|
||||
std::vector<std::string> columns = {"image"};
|
||||
std::shared_ptr<Iterator> iter = ds->CreateIterator(columns, -1);
|
||||
EXPECT_NE(iter, nullptr);
|
||||
|
||||
// Iterate the dataset and get each row
|
||||
std::vector<mindspore::MSTensor> row;
|
||||
ASSERT_OK(iter->GetNextRow(&row));
|
||||
|
||||
uint64_t i = 0;
|
||||
while (row.size() != 0) {
|
||||
for (auto &v : row) {
|
||||
MS_LOG(INFO) << "image shape:" << v.Shape();
|
||||
}
|
||||
ASSERT_OK(iter->GetNextRow(&row));
|
||||
i++;
|
||||
}
|
||||
|
||||
EXPECT_EQ(i, 2);
|
||||
|
||||
// Manually terminate the pipeline
|
||||
iter->Stop();
|
||||
}
|
||||
|
||||
/// Feature: FlickrIteratorWrongColumn.
|
||||
/// Description: test iterator of FlickrDataset with wrong column.
|
||||
/// Expectation: get none piece of data.
|
||||
TEST_F(MindDataTestPipeline, TestFlickrIteratorWrongColumn) {
|
||||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestFlickrIteratorWrongColumn.";
|
||||
std::string dataset_path = datasets_root_path_ + "/testFlickrData/flickr30k/flickr30k-images";
|
||||
std::string file_path = datasets_root_path_ + "/testFlickrData/flickr30k/test1.token";
|
||||
|
||||
// Create a Flickr30k Dataset
|
||||
std::shared_ptr<Dataset> ds = Flickr(dataset_path, file_path);
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Pass wrong column name
|
||||
std::vector<std::string> columns = {"digital"};
|
||||
std::shared_ptr<Iterator> iter = ds->CreateIterator(columns);
|
||||
EXPECT_EQ(iter, nullptr);
|
||||
}
|
||||
|
||||
TEST_F(MindDataTestPipeline, TestFlickrGetters) {
|
||||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestFlickrGetters.";
|
||||
|
||||
|
|
@ -131,6 +197,9 @@ TEST_F(MindDataTestPipeline, TestFlickrGetters) {
|
|||
EXPECT_EQ(ds2->GetColumnNames(), column_names);
|
||||
}
|
||||
|
||||
/// Feature: FlickrAnnotations.
|
||||
/// Description: test usage of FlickrAnnotations.
|
||||
/// Expectation: get correct number of data.
|
||||
TEST_F(MindDataTestPipeline, TestFlickrAnnotations) {
|
||||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestFlickrGetters.";
|
||||
|
||||
|
|
@ -177,6 +246,9 @@ TEST_F(MindDataTestPipeline, TestFlickrAnnotations) {
|
|||
iter->Stop();
|
||||
}
|
||||
|
||||
/// Feature: FlickrDecode.
|
||||
/// Description: test usage of FlickrDecode.
|
||||
/// Expectation: get correct number of data.
|
||||
TEST_F(MindDataTestPipeline, TestFlickrDecode) {
|
||||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestFlickrDecode.";
|
||||
|
||||
|
|
@ -212,6 +284,9 @@ TEST_F(MindDataTestPipeline, TestFlickrDecode) {
|
|||
iter->Stop();
|
||||
}
|
||||
|
||||
/// Feature: FlickrNumSamplers.
|
||||
/// Description: test usage of FlickrDataset with num sampler.
|
||||
/// Expectation: get correct piece of data.
|
||||
TEST_F(MindDataTestPipeline, TestFlickrNumSamplers) {
|
||||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestFlickrNumSamplers.";
|
||||
|
||||
|
|
@ -250,6 +325,9 @@ TEST_F(MindDataTestPipeline, TestFlickrNumSamplers) {
|
|||
iter->Stop();
|
||||
}
|
||||
|
||||
/// Feature: FlickrError.
|
||||
/// Description: test failure of Flickr Dataset.
|
||||
/// Expectation: get none piece of data.
|
||||
TEST_F(MindDataTestPipeline, TestFlickrError) {
|
||||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestFlickrError.";
|
||||
|
||||
|
|
@ -292,6 +370,9 @@ TEST_F(MindDataTestPipeline, TestFlickrError) {
|
|||
EXPECT_EQ(iter3, nullptr);
|
||||
}
|
||||
|
||||
/// Feature: FlickrWithNullSamplerError.
|
||||
/// Description: test failure of FlickrDataset with null sampler.
|
||||
/// Expectation: get none piece of data.
|
||||
TEST_F(MindDataTestPipeline, TestFlickrWithNullSamplerError) {
|
||||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestFlickrWithNullSamplerError.";
|
||||
|
||||
|
|
|
|||
|
|
@ -23,6 +23,9 @@ class MindDataTestPipeline : public UT::DatasetOpTesting {
|
|||
protected:
|
||||
};
|
||||
|
||||
/// Feature: ManifestBasic.
|
||||
/// Description: test basic usage of ManifestDataset.
|
||||
/// Expectation: get correct number of data.
|
||||
TEST_F(MindDataTestPipeline, TestManifestBasic) {
|
||||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestManifestBasic.";
|
||||
|
||||
|
|
@ -54,6 +57,9 @@ TEST_F(MindDataTestPipeline, TestManifestBasic) {
|
|||
iter->Stop();
|
||||
}
|
||||
|
||||
/// Feature: ManifestBasicWithPipeline.
|
||||
/// Description: test usage of ManifestDataset with pipeline.
|
||||
/// Expectation: get correct number of data.
|
||||
TEST_F(MindDataTestPipeline, TestManifestBasicWithPipeline) {
|
||||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestManifestBasicWithPipeline.";
|
||||
|
||||
|
|
@ -106,6 +112,65 @@ TEST_F(MindDataTestPipeline, TestManifestBasicWithPipeline) {
|
|||
iter->Stop();
|
||||
}
|
||||
|
||||
/// Feature: ManifestIteratorOneColumn.
|
||||
/// Description: test iterator of ManifestDataset with only the "image" column.
|
||||
/// Expectation: get correct data.
|
||||
TEST_F(MindDataTestPipeline, TestManifestIteratorOneColumn) {
|
||||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestManifestIteratorOneColumn.";
|
||||
std::string file_path = datasets_root_path_ + "/testManifestData/cpp.json";
|
||||
// Create a Manifest Dataset
|
||||
std::shared_ptr<Dataset> ds = Manifest(file_path);
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create a Batch operation on ds
|
||||
int32_t batch_size = 1;
|
||||
ds = ds->Batch(batch_size);
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create an iterator over the result of the above dataset
|
||||
// Only select "image" column and drop others
|
||||
std::vector<std::string> columns = {"image"};
|
||||
std::shared_ptr<Iterator> iter = ds->CreateIterator(columns, -1);
|
||||
EXPECT_NE(iter, nullptr);
|
||||
|
||||
// Iterate the dataset and get each row
|
||||
std::vector<mindspore::MSTensor> row;
|
||||
ASSERT_OK(iter->GetNextRow(&row));
|
||||
|
||||
uint64_t i = 0;
|
||||
while (row.size() != 0) {
|
||||
for (auto &v : row) {
|
||||
MS_LOG(INFO) << "image shape:" << v.Shape();
|
||||
}
|
||||
ASSERT_OK(iter->GetNextRow(&row));
|
||||
i++;
|
||||
}
|
||||
|
||||
EXPECT_EQ(i, 2);
|
||||
|
||||
// Manually terminate the pipeline
|
||||
iter->Stop();
|
||||
}
|
||||
|
||||
/// Feature: ManifestIteratorWrongColumn.
|
||||
/// Description: test iterator of ManifestDataset with wrong column.
|
||||
/// Expectation: get none piece of data.
|
||||
TEST_F(MindDataTestPipeline, TestManifestIteratorWrongColumn) {
|
||||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestManifestIteratorWrongColumn.";
|
||||
std::string file_path = datasets_root_path_ + "/testManifestData/cpp.json";
|
||||
// Create a Manifest Dataset
|
||||
std::shared_ptr<Dataset> ds = Manifest(file_path);
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Pass wrong column name
|
||||
std::vector<std::string> columns = {"digital"};
|
||||
std::shared_ptr<Iterator> iter = ds->CreateIterator(columns);
|
||||
EXPECT_EQ(iter, nullptr);
|
||||
}
|
||||
|
||||
/// Feature: ManifestGetters.
|
||||
/// Description: test usage of getters ManifestDataset.
|
||||
/// Expectation: get correct number of data and correct tensor shape.
|
||||
TEST_F(MindDataTestPipeline, TestManifestGetters) {
|
||||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestManifestGetters.";
|
||||
|
||||
|
|
@ -142,6 +207,9 @@ TEST_F(MindDataTestPipeline, TestManifestGetters) {
|
|||
EXPECT_EQ(class_index2[2].second[0], 2);
|
||||
}
|
||||
|
||||
/// Feature: ManifestDecode.
|
||||
/// Description: test usage of ManifestDecode.
|
||||
/// Expectation: get correct number of data.
|
||||
TEST_F(MindDataTestPipeline, TestManifestDecode) {
|
||||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestManifestDecode.";
|
||||
|
||||
|
|
@ -176,6 +244,9 @@ TEST_F(MindDataTestPipeline, TestManifestDecode) {
|
|||
iter->Stop();
|
||||
}
|
||||
|
||||
/// Feature: ManifestEval.
|
||||
/// Description: test usage of ManifestEval.
|
||||
/// Expectation: get correct number of data.
|
||||
TEST_F(MindDataTestPipeline, TestManifestEval) {
|
||||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestManifestEval.";
|
||||
|
||||
|
|
@ -207,6 +278,9 @@ TEST_F(MindDataTestPipeline, TestManifestEval) {
|
|||
iter->Stop();
|
||||
}
|
||||
|
||||
/// Feature: ManifestClassIndex.
|
||||
/// Description: test usage of ManifestClassIndex.
|
||||
/// Expectation: get correct number of data.
|
||||
TEST_F(MindDataTestPipeline, TestManifestClassIndex) {
|
||||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestManifestClassIndex.";
|
||||
|
||||
|
|
@ -261,6 +335,9 @@ TEST_F(MindDataTestPipeline, TestManifestClassIndex) {
|
|||
iter->Stop();
|
||||
}
|
||||
|
||||
/// Feature: ManifestNumSamplers.
|
||||
/// Description: test usage of ManifestDataset with num sampler.
|
||||
/// Expectation: get correct piece of data.
|
||||
TEST_F(MindDataTestPipeline, TestManifestNumSamplers) {
|
||||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestManifestNumSamplers.";
|
||||
|
||||
|
|
@ -292,6 +369,9 @@ TEST_F(MindDataTestPipeline, TestManifestNumSamplers) {
|
|||
iter->Stop();
|
||||
}
|
||||
|
||||
/// Feature: ManifestError.
|
||||
/// Description: test failure of Manifest Dataset.
|
||||
/// Expectation: get none piece of data.
|
||||
TEST_F(MindDataTestPipeline, TestManifestError) {
|
||||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestManifestError.";
|
||||
|
||||
|
|
@ -324,6 +404,9 @@ TEST_F(MindDataTestPipeline, TestManifestError) {
|
|||
EXPECT_EQ(iter2, nullptr);
|
||||
}
|
||||
|
||||
/// Feature: ManifestWithNullSamplerError.
|
||||
/// Description: test failure of ManifestDataset with null sampler.
|
||||
/// Expectation: get none piece of data.
|
||||
TEST_F(MindDataTestPipeline, TestManifestWithNullSamplerError) {
|
||||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestManifestWithNullSamplerError.";
|
||||
std::string file_path = datasets_root_path_ + "/testManifestData/cpp.json";
|
||||
|
|
|
|||
|
|
@ -26,6 +26,9 @@ class MindDataTestPipeline : public UT::DatasetOpTesting {
|
|||
protected:
|
||||
};
|
||||
|
||||
/// Feature: SBUDataset.
|
||||
/// Description: test basic usage of SBUDataset.
|
||||
/// Expectation: get correct number of data.
|
||||
TEST_F(MindDataTestPipeline, TestSBUDataset) {
|
||||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSBUDataset.";
|
||||
|
||||
|
|
@ -60,6 +63,9 @@ TEST_F(MindDataTestPipeline, TestSBUDataset) {
|
|||
iter->Stop();
|
||||
}
|
||||
|
||||
/// Feature: SBUDatasetWithPipeline.
|
||||
/// Description: test usage of SBUDataset with pipeline.
|
||||
/// Expectation: get correct number of data.
|
||||
TEST_F(MindDataTestPipeline, TestSBUDatasetWithPipeline) {
|
||||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSBUDatasetWithPipeline.";
|
||||
|
||||
|
|
@ -115,6 +121,65 @@ TEST_F(MindDataTestPipeline, TestSBUDatasetWithPipeline) {
|
|||
iter->Stop();
|
||||
}
|
||||
|
||||
/// Feature: SBUIteratorOneColumn.
|
||||
/// Description: test iterator of SBUDataset with only the "image" column.
|
||||
/// Expectation: get correct data.
|
||||
TEST_F(MindDataTestPipeline, TestSBUIteratorOneColumn) {
|
||||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSBUIteratorOneColumn.";
|
||||
// Create a SBU Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testSBUDataset/";
|
||||
std::shared_ptr<Dataset> ds = SBU(folder_path, true, std::make_shared<RandomSampler>(false, 5));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create a Batch operation on ds
|
||||
int32_t batch_size = 1;
|
||||
ds = ds->Batch(batch_size);
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create an iterator over the result of the above dataset
|
||||
// Only select "image" column and drop others
|
||||
std::vector<std::string> columns = {"image"};
|
||||
std::shared_ptr<Iterator> iter = ds->CreateIterator(columns, -1);
|
||||
EXPECT_NE(iter, nullptr);
|
||||
|
||||
// Iterate the dataset and get each row
|
||||
std::vector<mindspore::MSTensor> row;
|
||||
ASSERT_OK(iter->GetNextRow(&row));
|
||||
|
||||
uint64_t i = 0;
|
||||
while (row.size() != 0) {
|
||||
for (auto &v : row) {
|
||||
MS_LOG(INFO) << "image shape:" << v.Shape();
|
||||
}
|
||||
ASSERT_OK(iter->GetNextRow(&row));
|
||||
i++;
|
||||
}
|
||||
|
||||
EXPECT_EQ(i, 5);
|
||||
|
||||
// Manually terminate the pipeline
|
||||
iter->Stop();
|
||||
}
|
||||
|
||||
/// Feature: SBUIteratorWrongColumn.
|
||||
/// Description: test iterator of SBUtDataset with wrong column.
|
||||
/// Expectation: get none piece of data.
|
||||
TEST_F(MindDataTestPipeline, TestSBUIteratorWrongColumn) {
|
||||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSBUIteratorWrongColumn.";
|
||||
// Create a SBU Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testSBUDataset/";
|
||||
std::shared_ptr<Dataset> ds = SBU(folder_path, true, std::make_shared<RandomSampler>(false, 5));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Pass wrong column name
|
||||
std::vector<std::string> columns = {"digital"};
|
||||
std::shared_ptr<Iterator> iter = ds->CreateIterator(columns);
|
||||
EXPECT_EQ(iter, nullptr);
|
||||
}
|
||||
|
||||
/// Feature: SBUDatasetSize.
|
||||
/// Description: test usage of get the size of SBUDataset.
|
||||
/// Expectation: get correct number of data.
|
||||
TEST_F(MindDataTestPipeline, TestGetSBUDatasetSize) {
|
||||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestGetSBUDatasetSize.";
|
||||
|
||||
|
|
@ -126,6 +191,9 @@ TEST_F(MindDataTestPipeline, TestGetSBUDatasetSize) {
|
|||
EXPECT_EQ(ds->GetDatasetSize(), 5);
|
||||
}
|
||||
|
||||
/// Feature: SBUDatasetGetters.
|
||||
/// Description: test usage of getters SBUDataset.
|
||||
/// Expectation: get correct number of data and correct tensor shape.
|
||||
TEST_F(MindDataTestPipeline, TestSBUDatasetGetters) {
|
||||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSBUDatasetGetters.";
|
||||
|
||||
|
|
@ -160,6 +228,9 @@ TEST_F(MindDataTestPipeline, TestSBUDatasetGetters) {
|
|||
EXPECT_EQ(ds->GetDatasetSize(), 5);
|
||||
}
|
||||
|
||||
/// Feature: SBUDataFail.
|
||||
/// Description: test failure of SBUDataset.
|
||||
/// Expectation: get none piece of data.
|
||||
TEST_F(MindDataTestPipeline, TestSBUDatasetFail) {
|
||||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSBUDatasetFail.";
|
||||
|
||||
|
|
@ -173,6 +244,9 @@ TEST_F(MindDataTestPipeline, TestSBUDatasetFail) {
|
|||
EXPECT_EQ(iter, nullptr);
|
||||
}
|
||||
|
||||
/// Feature: SBUDataWithNullSamplerFail.
|
||||
/// Description: test failure of SBUDataset with null sampler.
|
||||
/// Expectation: get none piece of data.
|
||||
TEST_F(MindDataTestPipeline, TestSBUDatasetWithNullSamplerFail) {
|
||||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSBUDatasetWithNullSamplerFail.";
|
||||
|
||||
|
|
|
|||
|
|
@ -25,6 +25,9 @@ class MindDataTestPipeline : public UT::DatasetOpTesting {
|
|||
protected:
|
||||
};
|
||||
|
||||
/// Feature: USPSTrainDataset.
|
||||
/// Description: test basic usage of USPSTrainDataset.
|
||||
/// Expectation: get correct number of data.
|
||||
TEST_F(MindDataTestPipeline, TestUSPSTrainDataset) {
|
||||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestUSPSTrainDataset.";
|
||||
|
||||
|
|
@ -59,6 +62,9 @@ TEST_F(MindDataTestPipeline, TestUSPSTrainDataset) {
|
|||
iter->Stop();
|
||||
}
|
||||
|
||||
/// Feature: USPSTestDataset.
|
||||
/// Description: test basic usage of USPSTestDataset.
|
||||
/// Expectation: get correct number of data.
|
||||
TEST_F(MindDataTestPipeline, TestUSPSTestDataset) {
|
||||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestUSPSTestDataset.";
|
||||
|
||||
|
|
@ -93,6 +99,9 @@ TEST_F(MindDataTestPipeline, TestUSPSTestDataset) {
|
|||
iter->Stop();
|
||||
}
|
||||
|
||||
/// Feature: USPSAllDataset.
|
||||
/// Description: test basic usage of USPSAllDataset.
|
||||
/// Expectation: get correct number of data.
|
||||
TEST_F(MindDataTestPipeline, TestUSPSAllDataset) {
|
||||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestUSPSAllDataset.";
|
||||
|
||||
|
|
@ -127,6 +136,9 @@ TEST_F(MindDataTestPipeline, TestUSPSAllDataset) {
|
|||
iter->Stop();
|
||||
}
|
||||
|
||||
/// Feature: USPSDatasetWithPipeline.
|
||||
/// Description: test usage of USPSDataset with pipeline.
|
||||
/// Expectation: get correct number of data.
|
||||
TEST_F(MindDataTestPipeline, TestUSPSDatasetWithPipeline) {
|
||||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestUSPSTrainDatasetWithPipeline.";
|
||||
|
||||
|
|
@ -182,6 +194,67 @@ TEST_F(MindDataTestPipeline, TestUSPSDatasetWithPipeline) {
|
|||
iter->Stop();
|
||||
}
|
||||
|
||||
/// Feature: USPSIteratorOneColumn.
|
||||
/// Description: test iterator of USPSDataset with only the "image" column.
|
||||
/// Expectation: get correct data.
|
||||
TEST_F(MindDataTestPipeline, TestUSPSIteratorOneColumn) {
|
||||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestUSPSIteratorOneColumn.";
|
||||
// Create a USPS Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testUSPSDataset/";
|
||||
std::shared_ptr<Dataset> ds = USPS(folder_path, "train");
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create a Batch operation on ds
|
||||
int32_t batch_size = 1;
|
||||
ds = ds->Batch(batch_size);
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create an iterator over the result of the above dataset
|
||||
// Only select "image" column and drop others
|
||||
std::vector<std::string> columns = {"image"};
|
||||
std::shared_ptr<Iterator> iter = ds->CreateIterator(columns, -1);
|
||||
EXPECT_NE(iter, nullptr);
|
||||
|
||||
// Iterate the dataset and get each row
|
||||
std::vector<mindspore::MSTensor> row;
|
||||
ASSERT_OK(iter->GetNextRow(&row));
|
||||
std::vector<int64_t> expect_image = {1, 16, 16, 1};
|
||||
|
||||
uint64_t i = 0;
|
||||
while (row.size() != 0) {
|
||||
for (auto &v : row) {
|
||||
MS_LOG(INFO) << "image shape:" << v.Shape();
|
||||
EXPECT_EQ(expect_image, v.Shape());
|
||||
}
|
||||
ASSERT_OK(iter->GetNextRow(&row));
|
||||
i++;
|
||||
}
|
||||
|
||||
EXPECT_EQ(i, 3);
|
||||
|
||||
// Manually terminate the pipeline
|
||||
iter->Stop();
|
||||
}
|
||||
|
||||
/// Feature: USPSIteratorWrongColumn.
|
||||
/// Description: test iterator of USPSDataset with wrong column.
|
||||
/// Expectation: get none piece of data.
|
||||
TEST_F(MindDataTestPipeline, TestUSPSIteratorWrongColumn) {
|
||||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestUSPSIteratorWrongColumn.";
|
||||
// Create a USPS Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testUSPSDataset/";
|
||||
std::shared_ptr<Dataset> ds = USPS(folder_path, "train");
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Pass wrong column name
|
||||
std::vector<std::string> columns = {"digital"};
|
||||
std::shared_ptr<Iterator> iter = ds->CreateIterator(columns);
|
||||
EXPECT_EQ(iter, nullptr);
|
||||
}
|
||||
|
||||
/// Feature: GetUSPSDatasetSize.
|
||||
/// Description: test usage of get the size of USPSDataset.
|
||||
/// Expectation: get correct number of data.
|
||||
TEST_F(MindDataTestPipeline, TestGetUSPSDatasetSize) {
|
||||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestGetUSPSTrainDatasetSize.";
|
||||
|
||||
|
|
@ -193,6 +266,9 @@ TEST_F(MindDataTestPipeline, TestGetUSPSDatasetSize) {
|
|||
EXPECT_EQ(ds->GetDatasetSize(), 3);
|
||||
}
|
||||
|
||||
/// Feature: USPSDatasetGetters.
|
||||
/// Description: test usage of getters USPSDataset.
|
||||
/// Expectation: get correct number of data and correct tensor shape.
|
||||
TEST_F(MindDataTestPipeline, TestUSPSDatasetGetters) {
|
||||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestUSPSTrainDatasetGetters.";
|
||||
|
||||
|
|
@ -227,6 +303,9 @@ TEST_F(MindDataTestPipeline, TestUSPSDatasetGetters) {
|
|||
EXPECT_EQ(ds->GetDatasetSize(), 3);
|
||||
}
|
||||
|
||||
/// Feature: USPSDataFail.
|
||||
/// Description: test failure of USPSDataset.
|
||||
/// Expectation: get none piece of data.
|
||||
TEST_F(MindDataTestPipeline, TestUSPSDatasetFail) {
|
||||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestUSPSDatasetFail.";
|
||||
|
||||
|
|
@ -240,6 +319,9 @@ TEST_F(MindDataTestPipeline, TestUSPSDatasetFail) {
|
|||
EXPECT_EQ(iter, nullptr);
|
||||
}
|
||||
|
||||
/// Feature: USPSDatasetWithInvalidUsageFail.
|
||||
/// Description: test failure of USPSDataset with invalid usage.
|
||||
/// Expectation: get none piece of data.
|
||||
TEST_F(MindDataTestPipeline, TestUSPSDatasetWithInvalidUsageFail) {
|
||||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestUSPSDatasetWithInvalidUsageFail.";
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue