diff --git a/mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/text/bindings.cc b/mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/text/bindings.cc index 6391741a787..ce4e64b2e1a 100644 --- a/mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/text/bindings.cc +++ b/mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/text/bindings.cc @@ -20,6 +20,7 @@ #include "minddata/dataset/api/python/pybind_register.h" #include "minddata/dataset/include/dataset/constants.h" #include "minddata/dataset/text/fast_text.h" +#include "minddata/dataset/text/glove.h" #include "minddata/dataset/text/sentence_piece_vocab.h" #include "minddata/dataset/text/vectors.h" #include "minddata/dataset/text/vocab.h" @@ -99,6 +100,16 @@ PYBIND_REGISTER(FastText, 1, ([](const py::module *m) { }); })); +PYBIND_REGISTER(GloVe, 1, ([](const py::module *m) { + (void)py::class_>(*m, "GloVe") + .def(py::init<>()) + .def_static("from_file", [](const std::string &path, int32_t max_vectors) { + std::shared_ptr glove; + THROW_IF_ERROR(GloVe::BuildFromFile(&glove, path, max_vectors)); + return glove; + }); + })); + PYBIND_REGISTER(Vectors, 0, ([](const py::module *m) { (void)py::class_>(*m, "Vectors") .def(py::init<>()) diff --git a/mindspore/ccsrc/minddata/dataset/text/CMakeLists.txt b/mindspore/ccsrc/minddata/dataset/text/CMakeLists.txt index 517188690c6..912ca83feef 100644 --- a/mindspore/ccsrc/minddata/dataset/text/CMakeLists.txt +++ b/mindspore/ccsrc/minddata/dataset/text/CMakeLists.txt @@ -5,6 +5,7 @@ file(GLOB _CURRENT_SRC_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.cc") set_property(SOURCE ${_CURRENT_SRC_FILES} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_MD) add_library(text OBJECT fast_text.cc + glove.cc sentence_piece_vocab.cc vectors.cc vocab.cc diff --git a/mindspore/ccsrc/minddata/dataset/text/glove.cc b/mindspore/ccsrc/minddata/dataset/text/glove.cc new file mode 100644 index 00000000000..278291e9581 --- /dev/null +++ b/mindspore/ccsrc/minddata/dataset/text/glove.cc @@ -0,0 +1,56 @@ +/** + * Copyright 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "minddata/dataset/text/glove.h" + +#include "utils/file_utils.h" + +namespace mindspore { +namespace dataset { +GloVe::GloVe(const std::unordered_map> &map, int dim) : Vectors(map, dim) {} + +Status CheckGloVe(const std::string &file_path) { + Path path = Path(file_path); + if (path.Exists() && !path.IsDirectory()) { + std::string basename = path.Basename(); + size_t dot = basename.rfind('.'); + std::string suffix = basename.substr(dot + 1); + std::string sub_name = basename.substr(0, dot); + dot = sub_name.rfind('.'); + std::string glove_name = sub_name.substr(0, dot); + dot = glove_name.rfind('.'); + std::string infix = glove_name.substr(dot + 1); + std::string prefix = glove_name.substr(0, dot); + if (suffix != "txt" || infix != "6B" || prefix != "glove") { + RETURN_STATUS_UNEXPECTED("GloVe: invalid file, can not find file 'glove.6B.*.txt', but got: " + file_path); + } + return Status::OK(); + } else { + RETURN_STATUS_UNEXPECTED("GloVe: invalid file, failed to open GloVe file."); + } +} + +Status GloVe::BuildFromFile(std::shared_ptr *glove, const std::string &path, int32_t max_vectors) { + RETURN_UNEXPECTED_IF_NULL(glove); + RETURN_IF_NOT_OK(CheckGloVe(path)); + std::unordered_map> map; + int vector_dim = -1; + RETURN_IF_NOT_OK(Load(path, max_vectors, &map, &vector_dim)); + *glove = std::make_shared(std::move(map), vector_dim); + return Status::OK(); +} +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/minddata/dataset/text/glove.h b/mindspore/ccsrc/minddata/dataset/text/glove.h new file mode 100644 index 00000000000..31f9d02e217 --- /dev/null +++ b/mindspore/ccsrc/minddata/dataset/text/glove.h @@ -0,0 +1,55 @@ +/** + * Copyright 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_MINDDATA_DATASET_TEXT_GLOVE_H_ +#define MINDSPORE_CCSRC_MINDDATA_DATASET_TEXT_GLOVE_H_ + +#include +#include +#include +#include +#include + +#include "minddata/dataset/core/tensor.h" +#include "minddata/dataset/include/dataset/iterator.h" +#include "minddata/dataset/text/vectors.h" +#include "minddata/dataset/util/path.h" + +namespace mindspore { +namespace dataset { +/// \brief Pre-train word vectors. +class GloVe : public Vectors { + public: + /// Constructor. + GloVe() = default; + + /// Constructor. + /// \param[in] map A map between string and vector. + /// \param[in] dim Dimension of the vectors. + GloVe(const std::unordered_map> &map, int dim); + + /// Destructor. + ~GloVe() = default; + + /// \brief Build Vectors from reading a pre-train vector file. + /// \param[out] glove GloVe object which contains the pre-train vectors. + /// \param[in] path Path to the pre-trained word vector file. + /// \param[in] max_vectors This can be used to limit the number of pre-trained vectors loaded (default=0, no limit). + static Status BuildFromFile(std::shared_ptr *glove, const std::string &path, int32_t max_vectors = 0); +}; +} // namespace dataset +} // namespace mindspore +#endif // MINDSPORE_CCSRC_MINDDATA_DATASET_TEXT_GLOVE_H_ diff --git a/mindspore/ccsrc/minddata/dataset/text/ir/kernels/text_ir.h b/mindspore/ccsrc/minddata/dataset/text/ir/kernels/text_ir.h index f07ea2d5f59..6ea80915ce2 100644 --- a/mindspore/ccsrc/minddata/dataset/text/ir/kernels/text_ir.h +++ b/mindspore/ccsrc/minddata/dataset/text/ir/kernels/text_ir.h @@ -27,8 +27,6 @@ namespace mindspore { namespace dataset { -class Vectors; -class Vocab; class SentencePieceVocab; class Vectors; class Vocab; diff --git a/mindspore/ccsrc/minddata/dataset/text/vectors.cc b/mindspore/ccsrc/minddata/dataset/text/vectors.cc index 41b3a5c4e37..2b58efd8248 100644 --- a/mindspore/ccsrc/minddata/dataset/text/vectors.cc +++ b/mindspore/ccsrc/minddata/dataset/text/vectors.cc @@ -113,6 +113,7 @@ Vectors::Vectors(const std::unordered_map> &map, } Status Vectors::BuildFromFile(std::shared_ptr *vectors, const std::string &path, int32_t max_vectors) { + RETURN_UNEXPECTED_IF_NULL(vectors); std::unordered_map> map; int vector_dim = -1; RETURN_IF_NOT_OK(Load(path, max_vectors, &map, &vector_dim)); diff --git a/mindspore/dataset/text/__init__.py b/mindspore/dataset/text/__init__.py index 96309507f3e..888c83f569b 100644 --- a/mindspore/dataset/text/__init__.py +++ b/mindspore/dataset/text/__init__.py @@ -28,13 +28,14 @@ import platform from .transforms import Lookup, JiebaTokenizer, UnicodeCharTokenizer, Ngram, WordpieceTokenizer, \ TruncateSequencePair, ToNumber, SlidingWindow, SentencePieceTokenizer, PythonTokenizer, ToVectors from .utils import to_str, to_bytes, JiebaMode, Vocab, NormalizeForm, SentencePieceVocab, SentencePieceModel, \ - SPieceTokenizerOutType, SPieceTokenizerLoadType, Vectors, FastText + SPieceTokenizerOutType, SPieceTokenizerLoadType, Vectors, FastText, GloVe __all__ = [ "Lookup", "JiebaTokenizer", "UnicodeCharTokenizer", "Ngram", "to_str", "to_bytes", "Vocab", "WordpieceTokenizer", "TruncateSequencePair", "ToNumber", "PythonTokenizer", "SlidingWindow", "SentencePieceVocab", "SentencePieceTokenizer", "SPieceTokenizerOutType", - "SentencePieceModel", "SPieceTokenizerLoadType", "JiebaMode", "NormalizeForm", "Vectors", "ToVectors", "FastText" + "SentencePieceModel", "SPieceTokenizerLoadType", "JiebaMode", "NormalizeForm", "Vectors", "ToVectors", "FastText", + "GloVe" ] if platform.system().lower() != 'windows': diff --git a/mindspore/dataset/text/utils.py b/mindspore/dataset/text/utils.py index 1c934b9b506..df5161a289f 100644 --- a/mindspore/dataset/text/utils.py +++ b/mindspore/dataset/text/utils.py @@ -27,7 +27,7 @@ from .validators import check_from_file, check_from_list, check_from_dict, check check_from_file_vectors __all__ = [ - "Vocab", "SentencePieceVocab", "to_str", "to_bytes", "Vectors", "FastText" + "Vocab", "SentencePieceVocab", "to_str", "to_bytes", "Vectors", "FastText", "GloVe" ] @@ -438,3 +438,30 @@ class FastText(cde.FastText): max_vectors = max_vectors if max_vectors is not None else 0 return super().from_file(file_path, max_vectors) + + +class GloVe(cde.GloVe): + """ + GloVe object that is used to map tokens into vectors. + """ + + @classmethod + @check_from_file_vectors + def from_file(cls, file_path, max_vectors=None): + """ + Build a GloVe vector from a file. + + Args: + file_path (str): Path of the file that contains the vectors. The format of pre-trained vector sets + must be `glove.6B.*.txt`. + max_vectors (int, optional): This can be used to limit the number of pre-trained vectors loaded. + Most pre-trained vector sets are sorted in the descending order of word frequency. Thus, in + situations where the entire set doesn’t fit in memory, or is not needed for another reason, + passing max_vectors can limit the size of the loaded set (default=None, no limit). + + Examples: + >>> glove = text.GloVe.from_file("/path/to/glove/file", max_vectors=None) + """ + + max_vectors = max_vectors if max_vectors is not None else 0 + return super().from_file(file_path, max_vectors) diff --git a/tests/ut/cpp/dataset/c_api_text_test.cc b/tests/ut/cpp/dataset/c_api_text_test.cc index 218839e1db5..d818cc917d4 100644 --- a/tests/ut/cpp/dataset/c_api_text_test.cc +++ b/tests/ut/cpp/dataset/c_api_text_test.cc @@ -24,12 +24,14 @@ #include "minddata/dataset/include/dataset/text.h" #include "minddata/dataset/include/dataset/transforms.h" #include "minddata/dataset/text/fast_text.h" +#include "minddata/dataset/text/glove.h" #include "minddata/dataset/text/vectors.h" #include "minddata/dataset/text/vocab.h" using namespace mindspore::dataset; using mindspore::Status; using mindspore::dataset::FastText; +using mindspore::dataset::GloVe; using mindspore::dataset::ShuffleMode; using mindspore::dataset::Tensor; using mindspore::dataset::Vectors; @@ -4299,3 +4301,365 @@ TEST_F(MindDataTestPipeline, TestFastTextWithWrongSuffix) { Status s = FastText::BuildFromFile(&fast_text, vectors_dir); EXPECT_NE(s, Status::OK()); } + +/// Feature: GloVe +/// Description: test with default parameter in function BuildFromFile and function Lookup +/// Expectation: return correct MSTensor which is equal to the expected +TEST_F(MindDataTestPipeline, TestGloVeDefaultParam) { + // Test with default parameter. + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestGloVeDefaultParam."; + + // Create a TextFile dataset + std::string data_file = datasets_root_path_ + "/testGloVe/words.txt"; + std::shared_ptr ds = TextFile({data_file}, 0, ShuffleMode::kFalse); + EXPECT_NE(ds, nullptr); + + std::string vectors_dir = datasets_root_path_ + "/testGloVe/glove.6B.test.txt"; + std::shared_ptr glove; + Status s = GloVe::BuildFromFile(&glove, vectors_dir); + EXPECT_EQ(s, Status::OK()); + + std::shared_ptr lookup = + std::make_shared(glove); + EXPECT_NE(lookup, nullptr); + + // Create Map operation on ds + ds = ds->Map({lookup}, {"text"}); + EXPECT_NE(ds, nullptr); + + // Create an iterator over the result of the above dataset + std::shared_ptr iter = ds->CreateIterator(); + EXPECT_NE(iter, nullptr); + + // Iterate the dataset and get each row + std::unordered_map row; + ASSERT_OK(iter->GetNextRow(&row)); + + uint64_t i = 0; + std::vector> expected = { + {0.418, 0.24968, -0.41242, 0.1217, 0.34527, -0.04445718411}, + {0, 0, 0, 0, 0, 0}, + {0.15164, 0.30177, -0.16763, 0.17684, 0.31719, 0.33973}, + {0.70853, 0.57088, -0.4716, 0.18048, 0.54449, 0.72603}, + {0.68047, -0.039263, 0.30186, -0.17792, 0.42962, 0.032246}, + {0.26818, 0.14346, -0.27877, 0.016257, 0.11384, 0.69923}, + {0, 0, 0, 0, 0, 0}}; + while (row.size() != 0) { + auto ind = row["text"]; + MS_LOG(INFO) << ind.Shape(); + TEST_MS_LOG_MSTENSOR(INFO, "ind: ", ind); + TensorPtr de_expected_item; + dsize_t dim = 6; + ASSERT_OK(Tensor::CreateFromVector(expected[i], TensorShape({dim}), &de_expected_item)); + mindspore::MSTensor ms_expected_item = + mindspore::MSTensor(std::make_shared(de_expected_item)); + EXPECT_MSTENSOR_EQ(ind, ms_expected_item); + + ASSERT_OK(iter->GetNextRow(&row)); + i++; + } + + EXPECT_EQ(i, 7); + + // Manually terminate the pipeline + iter->Stop(); +} + +/// Feature: GloVe +/// Description: test with all parameters which include `path` and `max_vector` in function BuildFromFile +/// Expectation: return correct MSTensor which is equal to the expected +TEST_F(MindDataTestPipeline, TestGloVeAllBuildfromfileParams) { + // Test with two parameters. + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestGloVeAllBuildfromfileParams."; + + // Create a TextFile dataset + std::string data_file = datasets_root_path_ + "/testGloVe/words.txt"; + std::shared_ptr ds = TextFile({data_file}, 0, ShuffleMode::kFalse); + EXPECT_NE(ds, nullptr); + + std::string vectors_dir = datasets_root_path_ + "/testGloVe/glove.6B.test.txt"; + std::shared_ptr glove; + Status s = GloVe::BuildFromFile(&glove, vectors_dir, 100); + EXPECT_EQ(s, Status::OK()); + + std::shared_ptr lookup = + std::make_shared(glove); + EXPECT_NE(lookup, nullptr); + + // Create Map operation on ds + ds = ds->Map({lookup}, {"text"}); + EXPECT_NE(ds, nullptr); + + // Create an iterator over the result of the above dataset + std::shared_ptr iter = ds->CreateIterator(); + EXPECT_NE(iter, nullptr); + + // Iterate the dataset and get each row + std::unordered_map row; + ASSERT_OK(iter->GetNextRow(&row)); + + uint64_t i = 0; + std::vector> expected = { + {0.418, 0.24968, -0.41242, 0.1217, 0.34527, -0.04445718411}, + {0, 0, 0, 0, 0, 0}, + {0.15164, 0.30177, -0.16763, 0.17684, 0.31719, 0.33973}, + {0.70853, 0.57088, -0.4716, 0.18048, 0.54449, 0.72603}, + {0.68047, -0.039263, 0.30186, -0.17792, 0.42962, 0.032246}, + {0.26818, 0.14346, -0.27877, 0.016257, 0.11384, 0.69923}, + {0, 0, 0, 0, 0, 0}}; + while (row.size() != 0) { + auto ind = row["text"]; + MS_LOG(INFO) << ind.Shape(); + TEST_MS_LOG_MSTENSOR(INFO, "ind: ", ind); + TensorPtr de_expected_item; + dsize_t dim = 6; + ASSERT_OK(Tensor::CreateFromVector(expected[i], TensorShape({dim}), &de_expected_item)); + mindspore::MSTensor ms_expected_item = + mindspore::MSTensor(std::make_shared(de_expected_item)); + EXPECT_MSTENSOR_EQ(ind, ms_expected_item); + + ASSERT_OK(iter->GetNextRow(&row)); + i++; + } + + EXPECT_EQ(i, 7); + + // Manually terminate the pipeline + iter->Stop(); +} + +/// Feature: GloVe +/// Description: test with all parameters in function BuildFromFile and `unknown_init` in function Lookup +/// Expectation: return correct MSTensor which is equal to the expected +TEST_F(MindDataTestPipeline, TestGloVeUnknownInit) { + // Test with two parameters. + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestGloVeUnknownInit."; + + // Create a TextFile dataset + std::string data_file = datasets_root_path_ + "/testGloVe/words.txt"; + std::shared_ptr ds = TextFile({data_file}, 0, ShuffleMode::kFalse); + EXPECT_NE(ds, nullptr); + + std::string vectors_dir = datasets_root_path_ + "/testGloVe/glove.6B.test.txt"; + std::shared_ptr glove; + Status s = GloVe::BuildFromFile(&glove, vectors_dir, 100); + EXPECT_EQ(s, Status::OK()); + + std::vector unknown_init = {-1, -1, -1, -1, -1, -1}; + std::shared_ptr lookup = + std::make_shared(glove, unknown_init); + EXPECT_NE(lookup, nullptr); + + // Create Map operation on ds + ds = ds->Map({lookup}, {"text"}); + EXPECT_NE(ds, nullptr); + + // Create an iterator over the result of the above dataset + std::shared_ptr iter = ds->CreateIterator(); + EXPECT_NE(iter, nullptr); + + // Iterate the dataset and get each row + std::unordered_map row; + ASSERT_OK(iter->GetNextRow(&row)); + + uint64_t i = 0; + std::vector> expected = { + {0.418, 0.24968, -0.41242, 0.1217, 0.34527, -0.04445718411}, + {-1, -1, -1, -1, -1, -1}, + {0.15164, 0.30177, -0.16763, 0.17684, 0.31719, 0.33973}, + {0.70853, 0.57088, -0.4716, 0.18048, 0.54449, 0.72603}, + {0.68047, -0.039263, 0.30186, -0.17792, 0.42962, 0.032246}, + {0.26818, 0.14346, -0.27877, 0.016257, 0.11384, 0.69923}, + {-1, -1, -1, -1, -1, -1}}; + while (row.size() != 0) { + auto ind = row["text"]; + MS_LOG(INFO) << ind.Shape(); + TEST_MS_LOG_MSTENSOR(INFO, "ind: ", ind); + TensorPtr de_expected_item; + dsize_t dim = 6; + ASSERT_OK(Tensor::CreateFromVector(expected[i], TensorShape({dim}), &de_expected_item)); + mindspore::MSTensor ms_expected_item = + mindspore::MSTensor(std::make_shared(de_expected_item)); + EXPECT_MSTENSOR_EQ(ind, ms_expected_item); + + ASSERT_OK(iter->GetNextRow(&row)); + i++; + } + + EXPECT_EQ(i, 7); + + // Manually terminate the pipeline + iter->Stop(); +} + +/// Feature: GloVe +/// Description: test with all parameters which include `path` and `max_vectors` in function BuildFromFile and `token`, +/// `unknown_init` and `lower_case_backup` in function Lookup. But some tokens have some big letters +/// Expectation: return correct MSTensor which is equal to the expected +TEST_F(MindDataTestPipeline, TestGloVeAllParams) { + // Test with all parameters. + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestGloVeAllParams."; + // Create a TextFile dataset + std::string data_file = datasets_root_path_ + "/testGloVe/words.txt"; + std::shared_ptr ds = TextFile({data_file}, 0, ShuffleMode::kFalse); + EXPECT_NE(ds, nullptr); + + std::string vectors_dir = datasets_root_path_ + "/testGloVe/glove.6B.test.txt"; + std::shared_ptr glove; + Status s = GloVe::BuildFromFile(&glove, vectors_dir); + EXPECT_EQ(s, Status::OK()); + + std::vector unknown_init = {-1, -1, -1, -1, -1, -1}; + std::shared_ptr lookup = + std::make_shared(glove, unknown_init, true); + EXPECT_NE(lookup, nullptr); + + // Create Map operation on ds + ds = ds->Map({lookup}, {"text"}); + EXPECT_NE(ds, nullptr); + + // Create an iterator over the result of the above dataset + std::shared_ptr iter = ds->CreateIterator(); + EXPECT_NE(iter, nullptr); + + // Iterate the dataset and get each row + std::unordered_map row; + ASSERT_OK(iter->GetNextRow(&row)); + + uint64_t i = 0; + std::vector> expected = { + {0.418, 0.24968, -0.41242, 0.1217, 0.34527, -0.04445718411}, + {-1, -1, -1, -1, -1, -1}, + {0.15164, 0.30177, -0.16763, 0.17684, 0.31719, 0.33973}, + {0.70853, 0.57088, -0.4716, 0.18048, 0.54449, 0.72603}, + {0.68047, -0.039263, 0.30186, -0.17792, 0.42962, 0.032246}, + {0.26818, 0.14346, -0.27877, 0.016257, 0.11384, 0.69923}, + {-1, -1, -1, -1, -1, -1}}; + while (row.size() != 0) { + auto ind = row["text"]; + MS_LOG(INFO) << ind.Shape(); + TEST_MS_LOG_MSTENSOR(INFO, "ind: ", ind); + TensorPtr de_expected_item; + dsize_t dim = 6; + ASSERT_OK(Tensor::CreateFromVector(expected[i], TensorShape({dim}), &de_expected_item)); + mindspore::MSTensor ms_expected_item = + mindspore::MSTensor(std::make_shared(de_expected_item)); + EXPECT_MSTENSOR_EQ(ind, ms_expected_item); + + ASSERT_OK(iter->GetNextRow(&row)); + i++; + } + + EXPECT_EQ(i, 7); + + // Manually terminate the pipeline + iter->Stop(); +} + +/// Feature: GloVe +/// Description: test with pre-vectors set that have the different dimension +/// Expectation: throw correct error and message +TEST_F(MindDataTestPipeline, TestGloVeDifferentDimension) { + // Tokens don't have the same number of glove. + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestGloVeDifferentDimension."; + + // Create a TextFile dataset + std::string data_file = datasets_root_path_ + "/testGloVe/words.txt"; + std::shared_ptr ds = TextFile({data_file}, 0, ShuffleMode::kFalse); + EXPECT_NE(ds, nullptr); + + std::string vectors_dir = datasets_root_path_ + "/testGloVe/glove.6B.dim_different.txt"; + std::shared_ptr glove; + Status s = GloVe::BuildFromFile(&glove, vectors_dir, 100); + EXPECT_NE(s, Status::OK()); +} + +/// Feature: GloVe +/// Description: test with the parameter max_vectors that is <= 0 +/// Expectation: throw correct error and message +TEST_F(MindDataTestPipeline, TestGloVeMaxVectorsLessThanZero) { + // Test with max_vectors <= 0. + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestGloVeMaxVectorsLessThanZero."; + + // Create a TextFile dataset + std::string data_file = datasets_root_path_ + "/testGloVe/words.txt"; + std::shared_ptr ds = TextFile({data_file}, 0, ShuffleMode::kFalse); + EXPECT_NE(ds, nullptr); + + std::string vectors_dir = datasets_root_path_ + "/testGloVe/glove.6B.test.txt"; + std::shared_ptr glove; + Status s = GloVe::BuildFromFile(&glove, vectors_dir, -1); + EXPECT_NE(s, Status::OK()); +} + +/// Feature: GloVe +/// Description: test with the pre-vectors file that is empty +/// Expectation: throw correct error and message +TEST_F(MindDataTestPipeline, TestGloVeWithEmptyFile) { + // Read empty file. + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestGloVeWithEmptyFile."; + + // Create a TextFile dataset + std::string data_file = datasets_root_path_ + "/testGloVe/words.txt"; + std::shared_ptr ds = TextFile({data_file}, 0, ShuffleMode::kFalse); + EXPECT_NE(ds, nullptr); + + std::string vectors_dir = datasets_root_path_ + "/testGloVe/glove.6B.empty.txt"; + std::shared_ptr glove; + Status s = GloVe::BuildFromFile(&glove, vectors_dir); + EXPECT_NE(s, Status::OK()); +} + +/// Feature: GloVe +/// Description: test with the pre-vectors file that is not exist +/// Expectation: throw correct error and message +TEST_F(MindDataTestPipeline, TestGloVeWithNotExistFile) { + // Test with not exist file. + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestGloVeWithNotExistFile."; + + // Create a TextFile dataset + std::string data_file = datasets_root_path_ + "/testGloVe/words.txt"; + std::shared_ptr ds = TextFile({data_file}, 0, ShuffleMode::kFalse); + EXPECT_NE(ds, nullptr); + + std::string vectors_dir = datasets_root_path_ + "/testGloVe/glove.6B.empty.txt"; + std::shared_ptr glove; + Status s = GloVe::BuildFromFile(&glove, vectors_dir); + EXPECT_NE(s, Status::OK()); +} + +/// Feature: GloVe +/// Description: test with the pre-vectors set that has a situation that info-head is not the first line in the set +/// Expectation: throw correct error and message +TEST_F(MindDataTestPipeline, TestGloVeWithWrongInfoFile) { + // wrong info. + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestGloVeWithWrongInfoFile."; + + // Create a TextFile dataset + std::string data_file = datasets_root_path_ + "/testGloVe/words.txt"; + std::shared_ptr ds = TextFile({data_file}, 0, ShuffleMode::kFalse); + EXPECT_NE(ds, nullptr); + + std::string vectors_dir = datasets_root_path_ + "/testGloVe/glove.6B.with_wrong_info.txt"; + std::shared_ptr glove; + Status s = GloVe::BuildFromFile(&glove, vectors_dir); + EXPECT_NE(s, Status::OK()); +} + +/// Feature: GloVe +/// Description: test with the pre-vectors set that has a wrong format +/// Expectation: throw correct error and message +TEST_F(MindDataTestPipeline, TestGloVeWithWrongFormat) { + // wrong info. + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestGloVeWithWrongFormat."; + + // Create a TextFile dataset + std::string data_file = datasets_root_path_ + "/testGloVe/words.txt"; + std::shared_ptr ds = TextFile({data_file}, 0, ShuffleMode::kFalse); + EXPECT_NE(ds, nullptr); + + std::string vectors_dir = datasets_root_path_ + "/testGloVe/glove.6B.tests.vec"; + std::shared_ptr glove; + Status s = GloVe::BuildFromFile(&glove, vectors_dir); + EXPECT_NE(s, Status::OK()); +} diff --git a/tests/ut/cpp/dataset/execute_test.cc b/tests/ut/cpp/dataset/execute_test.cc index f7cdf25382b..44270d488d5 100644 --- a/tests/ut/cpp/dataset/execute_test.cc +++ b/tests/ut/cpp/dataset/execute_test.cc @@ -24,12 +24,14 @@ #include "minddata/dataset/include/dataset/audio.h" #include "minddata/dataset/include/dataset/text.h" #include "minddata/dataset/text/fast_text.h" +#include "minddata/dataset/text/glove.h" #include "minddata/dataset/text/vectors.h" #include "utils/log_adapter.h" using namespace mindspore::dataset; using mindspore::LogStream; using mindspore::dataset::FastText; +using mindspore::dataset::GloVe; using mindspore::dataset::Vectors; using mindspore::ExceptionType::NoExceptionType; using mindspore::MsLogLevel::INFO; @@ -1801,6 +1803,140 @@ TEST_F(MindDataTestExecute, TestToVectorsWithInvalidParamForFastText) { EXPECT_FALSE(status02.IsOk()); } +/// Feature: GloVe +/// Description: test basic usage of GloVe and the ToVectors with default parameter +/// Expectation: get correct MSTensor +TEST_F(MindDataTestExecute, TestGloVeParam) { + MS_LOG(INFO) << "Doing MindDataTestExecute-TestGloVeParam."; + std::shared_ptr de_tensor; + Tensor::CreateScalar("ok", &de_tensor); + auto token = mindspore::MSTensor(std::make_shared(de_tensor)); + mindspore::MSTensor lookup_result; + + // Create expected output. + std::shared_ptr de_expected; + std::vector expected = {0.418, 0.24968, -0.41242, 0.1217, 0.34527, -0.04445718411}; + dsize_t dim = 6; + ASSERT_OK(Tensor::CreateFromVector(expected, TensorShape({dim}), &de_expected)); + auto ms_expected = mindspore::MSTensor(std::make_shared(de_expected)); + + // Transform params. + std::string vectors_dir = "data/dataset/testGloVe/glove.6B.test.txt"; + std::shared_ptr glove01; + Status s01 = GloVe::BuildFromFile(&glove01, vectors_dir); + EXPECT_EQ(s01, Status::OK()); + std::shared_ptr to_vectors01 = std::make_shared(glove01); + auto transform01 = Execute({to_vectors01}); + Status status01 = transform01(token, &lookup_result); + EXPECT_MSTENSOR_EQ(lookup_result, ms_expected); + EXPECT_TRUE(status01.IsOk()); + + std::shared_ptr glove02; + Status s02 = GloVe::BuildFromFile(&glove02, vectors_dir, 100); + EXPECT_EQ(s02, Status::OK()); + std::shared_ptr to_vectors02 = std::make_shared(glove02); + auto transform02 = Execute({to_vectors02}); + Status status02 = transform02(token, &lookup_result); + EXPECT_MSTENSOR_EQ(lookup_result, ms_expected); + EXPECT_TRUE(status02.IsOk()); + + std::shared_ptr glove03; + Status s03 = GloVe::BuildFromFile(&glove03, vectors_dir, 3); + EXPECT_EQ(s03, Status::OK()); + std::shared_ptr to_vectors03 = std::make_shared(glove03); + auto transform03 = Execute({to_vectors03}); + Status status03 = transform03(token, &lookup_result); + EXPECT_MSTENSOR_EQ(lookup_result, ms_expected); + EXPECT_TRUE(status03.IsOk()); +} + +/// Feature: ToVectors +/// Description: test basic usage of ToVectors and the GloVe with default parameter +/// Expectation: get correct MSTensor +TEST_F(MindDataTestExecute, TestToVectorsParamForGloVe) { + MS_LOG(INFO) << "Doing MindDataTestExecute-TestToVectorsParamForGloVe."; + std::shared_ptr de_tensor01; + Tensor::CreateScalar("none", &de_tensor01); + auto token01 = mindspore::MSTensor(std::make_shared(de_tensor01)); + std::shared_ptr de_tensor02; + Tensor::CreateScalar("ok", &de_tensor02); + auto token02 = mindspore::MSTensor(std::make_shared(de_tensor02)); + std::shared_ptr de_tensor03; + Tensor::CreateScalar("OK", &de_tensor03); + auto token03 = mindspore::MSTensor(std::make_shared(de_tensor03)); + mindspore::MSTensor lookup_result; + + // Create expected output. + dsize_t dim = 6; + std::shared_ptr de_expected01; + std::vector expected01 = {0, 0, 0, 0, 0, 0}; + ASSERT_OK(Tensor::CreateFromVector(expected01, TensorShape({dim}), &de_expected01)); + auto ms_expected01 = mindspore::MSTensor(std::make_shared(de_expected01)); + std::shared_ptr de_expected02; + std::vector expected02 = {-1, -1, -1, -1, -1, -1}; + ASSERT_OK(Tensor::CreateFromVector(expected02, TensorShape({dim}), &de_expected02)); + auto ms_expected02 = mindspore::MSTensor(std::make_shared(de_expected02)); + std::shared_ptr de_expected03; + std::vector expected03 = {0.418, 0.24968, -0.41242, 0.1217, 0.34527, -0.04445718411}; + ASSERT_OK(Tensor::CreateFromVector(expected03, TensorShape({dim}), &de_expected03)); + auto ms_expected03 = mindspore::MSTensor(std::make_shared(de_expected03)); + + // Transform params. + std::string vectors_dir = "data/dataset/testGloVe/glove.6B.test.txt"; + std::shared_ptr glove; + Status s = GloVe::BuildFromFile(&glove, vectors_dir); + EXPECT_EQ(s, Status::OK()); + + std::shared_ptr to_vectors01 = std::make_shared(glove); + auto transform01 = Execute({to_vectors01}); + Status status01 = transform01(token01, &lookup_result); + EXPECT_MSTENSOR_EQ(lookup_result, ms_expected01); + EXPECT_TRUE(status01.IsOk()); + std::vector unknown_init = {-1, -1, -1, -1, -1, -1}; + std::shared_ptr to_vectors02 = std::make_shared(glove, unknown_init); + auto transform02 = Execute({to_vectors02}); + Status status02 = transform02(token01, &lookup_result); + EXPECT_MSTENSOR_EQ(lookup_result, ms_expected02); + EXPECT_TRUE(status02.IsOk()); + std::shared_ptr to_vectors03 = std::make_shared(glove, unknown_init); + auto transform03 = Execute({to_vectors03}); + Status status03 = transform03(token02, &lookup_result); + EXPECT_MSTENSOR_EQ(lookup_result, ms_expected03); + EXPECT_TRUE(status03.IsOk()); + std::shared_ptr to_vectors04 = std::make_shared(glove, unknown_init, true); + auto transform04 = Execute({to_vectors04}); + Status status04 = transform04(token03, &lookup_result); + EXPECT_MSTENSOR_EQ(lookup_result, ms_expected03); + EXPECT_TRUE(status04.IsOk()); +} + +/// Feature: ToVectors +/// Description: test invalid parameter of ToVectors for GloVe +/// Expectation: throw exception correctly +TEST_F(MindDataTestExecute, TestToVectorsWithInvalidParamForGloVe) { + MS_LOG(INFO) << "Doing MindDataTestExecute-TestToVectorsWithInvalidParamForGloVe."; + std::shared_ptr de_tensor; + Tensor::CreateScalar("none", &de_tensor); + auto token = mindspore::MSTensor(std::make_shared(de_tensor)); + mindspore::MSTensor lookup_result; + + // Transform params. + std::string vectors_dir = "data/dataset/testGloVe/glove.6B.test.txt"; + std::shared_ptr glove01; + Status s = GloVe::BuildFromFile(&glove01, vectors_dir); + EXPECT_EQ(s, Status::OK()); + std::vector unknown_init = {-1, -1, -1, -1}; + std::shared_ptr to_vectors01 = std::make_shared(glove01, unknown_init); + auto transform01 = Execute({to_vectors01}); + Status status01 = transform01(token, &lookup_result); + EXPECT_FALSE(status01.IsOk()); + std::shared_ptr glove02 = nullptr; + std::shared_ptr to_vectors02 = std::make_shared(glove02); + auto transform02 = Execute({to_vectors02}); + Status status02 = transform02(token, &lookup_result); + EXPECT_FALSE(status02.IsOk()); +} + // Feature: DBToAmplitude // Description: test DBToAmplitude in eager mode // Expectation: the data is processed successfully diff --git a/tests/ut/data/dataset/testGloVe/glove.6B.dim_different.txt b/tests/ut/data/dataset/testGloVe/glove.6B.dim_different.txt new file mode 100644 index 00000000000..65830c6aaf0 --- /dev/null +++ b/tests/ut/data/dataset/testGloVe/glove.6B.dim_different.txt @@ -0,0 +1,6 @@ +ok 0.418 0.24968 -0.41242 0.1217 0.34527 -0.04445718411 +! 0.013441 0.23682 -0.16899 0.40951 0.63812 0.47709 +this 0.15164 0.30177 -0.16763 0.17684 0.31719 +is 0.70853 0.57088 -0.4716 0.18048 0.54449 0.72603 +my 0.68047 -0.039263 0.30186 -0.17792 0.42962 0.032246 +home 0.26818 0.14346 -0.27877 0.016257 0.11384 0.69923 \ No newline at end of file diff --git a/tests/ut/data/dataset/testGloVe/glove.6B.empty.txt b/tests/ut/data/dataset/testGloVe/glove.6B.empty.txt new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/ut/data/dataset/testGloVe/glove.6B.test.txt b/tests/ut/data/dataset/testGloVe/glove.6B.test.txt new file mode 100644 index 00000000000..dc5c942ba1d --- /dev/null +++ b/tests/ut/data/dataset/testGloVe/glove.6B.test.txt @@ -0,0 +1,6 @@ +ok 0.418 0.24968 -0.41242 0.1217 0.34527 -0.04445718411 +! 0.013441 0.23682 -0.16899 0.40951 0.63812 0.47709 +this 0.15164 0.30177 -0.16763 0.17684 0.31719 0.33973 +is 0.70853 0.57088 -0.4716 0.18048 0.54449 0.72603 +my 0.68047 -0.039263 0.30186 -0.17792 0.42962 0.032246 +home 0.26818 0.14346 -0.27877 0.016257 0.11384 0.69923 diff --git a/tests/ut/data/dataset/testGloVe/glove.6B.test.vec b/tests/ut/data/dataset/testGloVe/glove.6B.test.vec new file mode 100644 index 00000000000..dc5c942ba1d --- /dev/null +++ b/tests/ut/data/dataset/testGloVe/glove.6B.test.vec @@ -0,0 +1,6 @@ +ok 0.418 0.24968 -0.41242 0.1217 0.34527 -0.04445718411 +! 0.013441 0.23682 -0.16899 0.40951 0.63812 0.47709 +this 0.15164 0.30177 -0.16763 0.17684 0.31719 0.33973 +is 0.70853 0.57088 -0.4716 0.18048 0.54449 0.72603 +my 0.68047 -0.039263 0.30186 -0.17792 0.42962 0.032246 +home 0.26818 0.14346 -0.27877 0.016257 0.11384 0.69923 diff --git a/tests/ut/data/dataset/testGloVe/glove.6B.with_info.txt b/tests/ut/data/dataset/testGloVe/glove.6B.with_info.txt new file mode 100644 index 00000000000..74d030d01e3 --- /dev/null +++ b/tests/ut/data/dataset/testGloVe/glove.6B.with_info.txt @@ -0,0 +1,7 @@ +6 6 +ok 0.418 0.24968 -0.41242 0.1217 0.34527 -0.04445718411 +! 0.013441 0.23682 -0.16899 0.40951 0.63812 0.47709 +this 0.15164 0.30177 -0.16763 0.17684 0.31719 0.33973 +is 0.70853 0.57088 -0.4716 0.18048 0.54449 0.72603 +my 0.68047 -0.039263 0.30186 -0.17792 0.42962 0.032246 +home 0.26818 0.14346 -0.27877 0.016257 0.11384 0.69923 diff --git a/tests/ut/data/dataset/testGloVe/glove.6B.with_wrong_info.txt b/tests/ut/data/dataset/testGloVe/glove.6B.with_wrong_info.txt new file mode 100644 index 00000000000..86d3cc3952f --- /dev/null +++ b/tests/ut/data/dataset/testGloVe/glove.6B.with_wrong_info.txt @@ -0,0 +1,7 @@ +the 0.418 0.24968 -0.41242 0.1217 0.34527 -0.04445718411 +, 0.013441 0.23682 -0.16899 0.40951 0.63812 0.47709 +. 0.15164 0.30177 -0.16763 0.17684 0.31719 0.33973 +6 6 +of 0.70853 0.57088 -0.4716 0.18048 0.54449 0.72603 +to 0.68047 -0.039263 0.30186 -0.17792 0.42962 0.032246 +and 0.26818 0.14346 -0.27877 0.016257 0.11384 0.69923 \ No newline at end of file diff --git a/tests/ut/data/dataset/testGloVe/words.txt b/tests/ut/data/dataset/testGloVe/words.txt new file mode 100644 index 00000000000..87e004ad8f1 --- /dev/null +++ b/tests/ut/data/dataset/testGloVe/words.txt @@ -0,0 +1,7 @@ +ok +. +this +is +my +home +. diff --git a/tests/ut/data/dataset/testGloVe/words_with_big_letter.txt b/tests/ut/data/dataset/testGloVe/words_with_big_letter.txt new file mode 100644 index 00000000000..efa25a4b390 --- /dev/null +++ b/tests/ut/data/dataset/testGloVe/words_with_big_letter.txt @@ -0,0 +1,7 @@ +ok +! +This +iS +my +HOME +. diff --git a/tests/ut/python/dataset/test_glove.py b/tests/ut/python/dataset/test_glove.py new file mode 100644 index 00000000000..664a96ab532 --- /dev/null +++ b/tests/ut/python/dataset/test_glove.py @@ -0,0 +1,238 @@ +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +import numpy as np +import pytest + +from mindspore import log +import mindspore.dataset as ds +import mindspore.dataset.text as text +import mindspore.dataset.text.transforms as T + +DATASET_ROOT_PATH = "../data/dataset/testGloVe/" + + +def test_glove_all_build_from_file_params(): + """ + Feature: GloVe + Description: test with all parameters which include `path` and `max_vector` in function BuildFromFile + Expectation: output is equal to the expected value + """ + vectors = text.GloVe.from_file(DATASET_ROOT_PATH + "glove.6B.test.txt", max_vectors=100) + to_vectors = text.ToVectors(vectors) + data = ds.TextFileDataset(DATASET_ROOT_PATH + "words.txt", shuffle=False) + data = data.map(operations=to_vectors, input_columns=["text"]) + ind = 0 + res = [[0.418, 0.24968, -0.41242, 0.1217, 0.34527, -0.04445718411], + [0, 0, 0, 0, 0, 0], + [0.15164, 0.30177, -0.16763, 0.17684, 0.31719, 0.33973], + [0.70853, 0.57088, -0.4716, 0.18048, 0.54449, 0.72603], + [0.68047, -0.039263, 0.30186, -0.17792, 0.42962, 0.032246], + [0.26818, 0.14346, -0.27877, 0.016257, 0.11384, 0.69923], + [0, 0, 0, 0, 0, 0]] + print(data) + for d in data.create_dict_iterator(num_epochs=1, output_numpy=True): + res_array = np.array(res[ind], dtype=np.float32) + assert np.array_equal(res_array, d["text"]), ind + ind += 1 + + +def test_glove_all_build_from_file_params_eager(): + """ + Feature: GloVe + Description: test with all parameters which include `path` and `max_vector` in function BuildFromFile in eager mode + Expectation: output is equal to the expected value + """ + vectors = text.GloVe.from_file(DATASET_ROOT_PATH + "glove.6B.test.txt", max_vectors=4) + to_vectors = T.ToVectors(vectors) + result1 = to_vectors("ok") + result2 = to_vectors("!") + result3 = to_vectors("this") + result4 = to_vectors("is") + result5 = to_vectors("my") + result6 = to_vectors("home") + result7 = to_vectors("none") + res = [[0.418, 0.24968, -0.41242, 0.1217, 0.34527, -0.04445718411], + [0.013441, 0.23682, -0.16899, 0.40951, 0.63812, 0.47709], + [0.15164, 0.30177, -0.16763, 0.17684, 0.31719, 0.33973], + [0.70853, 0.57088, -0.4716, 0.18048, 0.54449, 0.72603], + [0, 0, 0, 0, 0, 0], + [0, 0, 0, 0, 0, 0], + [0, 0, 0, 0, 0, 0]] + res_array = np.array(res, dtype=np.float32) + + assert np.array_equal(result1, res_array[0]) + assert np.array_equal(result2, res_array[1]) + assert np.array_equal(result3, res_array[2]) + assert np.array_equal(result4, res_array[3]) + assert np.array_equal(result5, res_array[4]) + assert np.array_equal(result6, res_array[5]) + assert np.array_equal(result7, res_array[6]) + + +def test_glove_all_to_vectors_params_eager(): + """ + Feature: GloVe + Description: test with all parameters which include `unk_init` and `lower_case_backup` in function ToVectors + in eager mode + Expectation: output is equal to the expected value + """ + vectors = text.GloVe.from_file(DATASET_ROOT_PATH + "glove.6B.test.txt", max_vectors=4) + my_unk = [-1, -1, -1, -1, -1, -1] + to_vectors = T.ToVectors(vectors, unk_init=my_unk, lower_case_backup=True) + result1 = to_vectors("Ok") + result2 = to_vectors("!") + result3 = to_vectors("This") + result4 = to_vectors("is") + result5 = to_vectors("my") + result6 = to_vectors("home") + result7 = to_vectors("none") + res = [[0.418, 0.24968, -0.41242, 0.1217, 0.34527, -0.04445718411], + [0.013441, 0.23682, -0.16899, 0.40951, 0.63812, 0.47709], + [0.15164, 0.30177, -0.16763, 0.17684, 0.31719, 0.33973], + [0.70853, 0.57088, -0.4716, 0.18048, 0.54449, 0.72603], + [-1, -1, -1, -1, -1, -1], + [-1, -1, -1, -1, -1, -1], + [-1, -1, -1, -1, -1, -1]] + res_array = np.array(res, dtype=np.float32) + + assert np.array_equal(result1, res_array[0]) + assert np.array_equal(result2, res_array[1]) + assert np.array_equal(result3, res_array[2]) + assert np.array_equal(result4, res_array[3]) + assert np.array_equal(result5, res_array[4]) + assert np.array_equal(result6, res_array[5]) + assert np.array_equal(result7, res_array[6]) + + +def test_glove_build_from_file(): + """ + Feature: GloVe + Description: test with only default parameter + Expectation: output is equal to the expected value + """ + vectors = text.GloVe.from_file(DATASET_ROOT_PATH + "glove.6B.test.txt") + to_vectors = text.ToVectors(vectors) + data = ds.TextFileDataset(DATASET_ROOT_PATH + "words.txt", shuffle=False) + data = data.map(operations=to_vectors, input_columns=["text"]) + ind = 0 + res = [[0.418, 0.24968, -0.41242, 0.1217, 0.34527, -0.04445718411], + [0, 0, 0, 0, 0, 0], + [0.15164, 0.30177, -0.16763, 0.17684, 0.31719, 0.33973], + [0.70853, 0.57088, -0.4716, 0.18048, 0.54449, 0.72603], + [0.68047, -0.039263, 0.30186, -0.17792, 0.42962, 0.032246], + [0.26818, 0.14346, -0.27877, 0.016257, 0.11384, 0.69923], + [0, 0, 0, 0, 0, 0]] + print(data) + for d in data.create_dict_iterator(num_epochs=1, output_numpy=True): + res_array = np.array(res[ind], dtype=np.float32) + assert np.array_equal(res_array, d["text"]), ind + ind += 1 + + +def test_glove_build_from_file_eager(): + """ + Feature: GloVe + Description: test with only default parameter in eager mode + Expectation: output is equal to the expected value + """ + vectors = text.GloVe.from_file(DATASET_ROOT_PATH + "glove.6B.test.txt") + to_vectors = T.ToVectors(vectors) + result1 = to_vectors("ok") + result2 = to_vectors("!") + result3 = to_vectors("this") + result4 = to_vectors("is") + result5 = to_vectors("my") + result6 = to_vectors("home") + result7 = to_vectors("none") + res = [[0.418, 0.24968, -0.41242, 0.1217, 0.34527, -0.04445718411], + [0.013441, 0.23682, -0.16899, 0.40951, 0.63812, 0.47709], + [0.15164, 0.30177, -0.16763, 0.17684, 0.31719, 0.33973], + [0.70853, 0.57088, -0.4716, 0.18048, 0.54449, 0.72603], + [0.68047, -0.039263, 0.30186, -0.17792, 0.42962, 0.032246], + [0.26818, 0.14346, -0.27877, 0.016257, 0.11384, 0.69923], + [0, 0, 0, 0, 0, 0]] + res_array = np.array(res, dtype=np.float32) + + assert np.array_equal(result1, res_array[0]) + assert np.array_equal(result2, res_array[1]) + assert np.array_equal(result3, res_array[2]) + assert np.array_equal(result4, res_array[3]) + assert np.array_equal(result5, res_array[4]) + assert np.array_equal(result6, res_array[5]) + assert np.array_equal(result7, res_array[6]) + + +def test_glove_invalid_input(): + """ + Feature: GloVe + Description: test the validate function with invalid parameters + Expectation: output is equal to the expected error + """ + def test_invalid_input(test_name, file_path, error, error_msg, max_vectors=None, unk_init=None, + lower_case_backup=False, token="ok"): + log.info("Test Vectors with wrong input: {0}".format(test_name)) + with pytest.raises(error) as error_info: + vectors = text.GloVe.from_file(file_path, max_vectors=max_vectors) + to_vectors = T.ToVectors(vectors, unk_init=unk_init, lower_case_backup=lower_case_backup) + to_vectors(token) + assert error_msg in str(error_info.value) + + test_invalid_input("Not all vectors have the same number of dimensions", + DATASET_ROOT_PATH + "glove.6B.dim_different.txt", error=RuntimeError, + error_msg="all vectors must have the same number of dimensions, " \ + "but got dim 5 while expecting 6") + test_invalid_input("the file is empty.", DATASET_ROOT_PATH + "glove.6B.empty.txt", + error=RuntimeError, error_msg="invalid file, file is empty.") + test_invalid_input("the count of `unknown_init`'s element is different with word vector.", + DATASET_ROOT_PATH + "glove.6B.test.txt", + error=RuntimeError, + error_msg="unk_init must be the same length as vectors, but got unk_init", + unk_init=[-1, -1]) + test_invalid_input("The file not exist", DATASET_ROOT_PATH + "not_exist.txt", RuntimeError, + error_msg="GloVe: invalid file") + test_invalid_input("The token is 1-dimensional", DATASET_ROOT_PATH + "glove.6B.with_wrong_info.txt", + error=RuntimeError, error_msg="token with 1-dimensional vector.") + test_invalid_input("max_vectors parameter must be greater than 0", DATASET_ROOT_PATH + "glove.6B.test.txt", + error=ValueError, error_msg="Input max_vectors is not within the required interval", + max_vectors=-1) + test_invalid_input("invalid max_vectors parameter type as a float", DATASET_ROOT_PATH + "glove.6B.test.txt", + error=TypeError, error_msg="Argument max_vectors with value 1.0 is not of type []," + " but got .", max_vectors=1.0) + test_invalid_input("invalid max_vectors parameter type as a string", DATASET_ROOT_PATH + "glove.6B.test.txt", + error=TypeError, error_msg="Argument max_vectors with value 1 is not of type []," + " but got .", max_vectors="1") + test_invalid_input("invalid token parameter type as a float", DATASET_ROOT_PATH + "glove.6B.test.txt", + error=RuntimeError, error_msg="input tensor type should be string.", token=1.0) + test_invalid_input("invalid lower_case_backup parameter type as a string", DATASET_ROOT_PATH + "glove.6B.test.txt", + error=TypeError, error_msg="Argument lower_case_backup with value True is " \ + "not of type []," + " but got .", lower_case_backup="True") + test_invalid_input("invalid lower_case_backup parameter type as a string", DATASET_ROOT_PATH + "glove.6B.test.txt", + error=TypeError, error_msg="Argument lower_case_backup with value True is " \ + "not of type []," + " but got .", lower_case_backup="True") + test_invalid_input("not right glove dataset. The formal must be `glove.6B.*.txt`", DATASET_ROOT_PATH + + "glove.6B.test.vec", error=RuntimeError, error_msg="GloVe: invalid file, can not " \ + "find file 'glove.6B.*.txt'") + + +if __name__ == '__main__': + test_glove_all_build_from_file_params() + test_glove_all_build_from_file_params_eager() + test_glove_all_to_vectors_params_eager() + test_glove_build_from_file() + test_glove_build_from_file_eager() + test_glove_invalid_input()