From 797cd52c967bea85ab53b24d73071cc7043285ea Mon Sep 17 00:00:00 2001 From: tophand <1297030928@qq.com> Date: Fri, 28 Jan 2022 02:26:20 -0800 Subject: [PATCH] [feat][assistant][I40GXK]add new data operator Filter_Wikipedia_XML --- .../dataset/text/kernels/ir/bindings.cc | 11 ++ mindspore/ccsrc/minddata/dataset/api/text.cc | 5 + .../minddata/dataset/include/dataset/text.h | 15 +++ .../minddata/dataset/kernels/tensor_op.h | 1 + .../dataset/text/ir/kernels/text_ir.cc | 11 ++ .../dataset/text/ir/kernels/text_ir.h | 14 +++ .../dataset/text/kernels/CMakeLists.txt | 1 + .../text/kernels/filter_wikipedia_xml_op.cc | 117 ++++++++++++++++++ .../text/kernels/filter_wikipedia_xml_op.h | 49 ++++++++ .../mindspore/dataset/text/transforms.py | 16 +++ tests/ut/cpp/dataset/c_api_text_test.cc | 49 ++++++++ tests/ut/cpp/dataset/execute_test.cc | 18 +++ tests/ut/data/dataset/testTokenizerData/2.txt | 3 + .../dataset/test_filter_wikipedia_xml.py | 71 +++++++++++ 14 files changed, 381 insertions(+) create mode 100644 mindspore/ccsrc/minddata/dataset/text/kernels/filter_wikipedia_xml_op.cc create mode 100644 mindspore/ccsrc/minddata/dataset/text/kernels/filter_wikipedia_xml_op.h create mode 100644 tests/ut/data/dataset/testTokenizerData/2.txt create mode 100644 tests/ut/python/dataset/test_filter_wikipedia_xml.py diff --git a/mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/text/kernels/ir/bindings.cc b/mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/text/kernels/ir/bindings.cc index f3962ad34ae..7768cbbe599 100644 --- a/mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/text/kernels/ir/bindings.cc +++ b/mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/text/kernels/ir/bindings.cc @@ -65,6 +65,17 @@ PYBIND_REGISTER(CaseFoldOperation, 1, ([](const py::module *m) { })); })); +PYBIND_REGISTER(FilterWikipediaXMLOperation, 1, ([](const py::module *m) { + (void)py::class_>(*m, + "FilterWikipediaXMLOperation") + .def(py::init([]() { + auto filter_wikipedia_xml = std::make_shared(); + THROW_IF_ERROR(filter_wikipedia_xml->ValidateParams()); + return filter_wikipedia_xml; + })); + })); + PYBIND_REGISTER( NormalizeUTF8Operation, 1, ([](const py::module *m) { (void)py::class_>( diff --git a/mindspore/ccsrc/minddata/dataset/api/text.cc b/mindspore/ccsrc/minddata/dataset/api/text.cc index 0dfb7970409..9169d36c88a 100644 --- a/mindspore/ccsrc/minddata/dataset/api/text.cc +++ b/mindspore/ccsrc/minddata/dataset/api/text.cc @@ -105,6 +105,11 @@ std::shared_ptr BertTokenizer::Parse() { CaseFold::CaseFold() = default; std::shared_ptr CaseFold::Parse() { return std::make_shared(); } + +// FilterWikipediaXML +FilterWikipediaXML::FilterWikipediaXML() {} + +std::shared_ptr FilterWikipediaXML::Parse() { return std::make_shared(); } #endif // JiebaTokenizer diff --git a/mindspore/ccsrc/minddata/dataset/include/dataset/text.h b/mindspore/ccsrc/minddata/dataset/include/dataset/text.h index a5e46c840d1..644045517cf 100644 --- a/mindspore/ccsrc/minddata/dataset/include/dataset/text.h +++ b/mindspore/ccsrc/minddata/dataset/include/dataset/text.h @@ -173,6 +173,21 @@ class MS_API CaseFold final : public TensorTransform { /// \return Shared pointer to the TensorOperation object. std::shared_ptr Parse() override; }; + +/// \brief Filter wikipedia xml lines. +class FilterWikipediaXML final : public TensorTransform { + public: + /// \brief Constructor. + FilterWikipediaXML(); + + /// \brief Destructor + ~FilterWikipediaXML() = default; + + protected: + /// \brief The function to convert a TensorTransform object into a TensorOperation object. + /// \return Shared pointer to the TensorOperation object. + std::shared_ptr Parse() override; +}; #endif /// \brief Tokenize a Chinese string into words based on the dictionary. diff --git a/mindspore/ccsrc/minddata/dataset/kernels/tensor_op.h b/mindspore/ccsrc/minddata/dataset/kernels/tensor_op.h index ca606f0ad72..453f7692a49 100644 --- a/mindspore/ccsrc/minddata/dataset/kernels/tensor_op.h +++ b/mindspore/ccsrc/minddata/dataset/kernels/tensor_op.h @@ -125,6 +125,7 @@ constexpr char kVerticalFlipOp[] = "VerticalFlipOp"; constexpr char kBasicTokenizerOp[] = "BasicTokenizerOp"; constexpr char kBertTokenizerOp[] = "BertTokenizerOp"; constexpr char kCaseFoldOp[] = "CaseFoldOp"; +constexpr char kFilterWikipediaXMLOp[] = "FilterWikipediaXMLOp"; constexpr char kJiebaTokenizerOp[] = "JiebaTokenizerOp"; constexpr char kLookupOp[] = "LookupOp"; constexpr char kNgramOp[] = "NgramOp"; diff --git a/mindspore/ccsrc/minddata/dataset/text/ir/kernels/text_ir.cc b/mindspore/ccsrc/minddata/dataset/text/ir/kernels/text_ir.cc index 6e6d6937bbf..c4f98942230 100644 --- a/mindspore/ccsrc/minddata/dataset/text/ir/kernels/text_ir.cc +++ b/mindspore/ccsrc/minddata/dataset/text/ir/kernels/text_ir.cc @@ -21,6 +21,7 @@ #include "minddata/dataset/text/kernels/basic_tokenizer_op.h" #include "minddata/dataset/text/kernels/bert_tokenizer_op.h" #include "minddata/dataset/text/kernels/case_fold_op.h" +#include "minddata/dataset/text/kernels/filter_wikipedia_xml_op.h" #endif #include "minddata/dataset/text/kernels/jieba_tokenizer_op.h" #include "minddata/dataset/text/kernels/lookup_op.h" @@ -137,6 +138,16 @@ std::shared_ptr CaseFoldOperation::Build() { std::shared_ptr tensor_op = std::make_shared(); return tensor_op; } + +// FilterWikipediaXMLOperation +FilterWikipediaXMLOperation::FilterWikipediaXMLOperation() {} + +Status FilterWikipediaXMLOperation::ValidateParams() { return Status::OK(); } + +std::shared_ptr FilterWikipediaXMLOperation::Build() { + std::shared_ptr tensor_op = std::make_shared(); + return tensor_op; +} #endif // JiebaTokenizerOperation diff --git a/mindspore/ccsrc/minddata/dataset/text/ir/kernels/text_ir.h b/mindspore/ccsrc/minddata/dataset/text/ir/kernels/text_ir.h index 6ea80915ce2..04a3bfaa26d 100644 --- a/mindspore/ccsrc/minddata/dataset/text/ir/kernels/text_ir.h +++ b/mindspore/ccsrc/minddata/dataset/text/ir/kernels/text_ir.h @@ -37,6 +37,7 @@ namespace text { constexpr char kBasicTokenizerOperation[] = "BasicTokenizer"; constexpr char kBertTokenizerOperation[] = "BertTokenizer"; constexpr char kCaseFoldOperation[] = "CaseFold"; +constexpr char kFilterWikipediaXMLOperation[] = "FilterWikipediaXML"; constexpr char kJiebaTokenizerOperation[] = "JiebaTokenizer"; constexpr char kLookupOperation[] = "Lookup"; constexpr char kNgramOperation[] = "Ngram"; @@ -116,6 +117,19 @@ class CaseFoldOperation : public TensorOperation { std::string Name() const override { return kCaseFoldOperation; } }; + +class FilterWikipediaXMLOperation : public TensorOperation { + public: + FilterWikipediaXMLOperation(); + + ~FilterWikipediaXMLOperation() = default; + + std::shared_ptr Build() override; + + Status ValidateParams() override; + + std::string Name() const override { return kFilterWikipediaXMLOperation; } +}; #endif class JiebaTokenizerOperation : public TensorOperation { diff --git a/mindspore/ccsrc/minddata/dataset/text/kernels/CMakeLists.txt b/mindspore/ccsrc/minddata/dataset/text/kernels/CMakeLists.txt index 756ac2b1790..1bfb90caa83 100644 --- a/mindspore/ccsrc/minddata/dataset/text/kernels/CMakeLists.txt +++ b/mindspore/ccsrc/minddata/dataset/text/kernels/CMakeLists.txt @@ -5,6 +5,7 @@ if(NOT (CMAKE_SYSTEM_NAME MATCHES "Windows")) basic_tokenizer_op.cc bert_tokenizer_op.cc case_fold_op.cc + filter_wikipedia_xml_op.cc normalize_utf8_op.cc regex_replace_op.cc regex_tokenizer_op.cc diff --git a/mindspore/ccsrc/minddata/dataset/text/kernels/filter_wikipedia_xml_op.cc b/mindspore/ccsrc/minddata/dataset/text/kernels/filter_wikipedia_xml_op.cc new file mode 100644 index 00000000000..bb30659c94f --- /dev/null +++ b/mindspore/ccsrc/minddata/dataset/text/kernels/filter_wikipedia_xml_op.cc @@ -0,0 +1,117 @@ +/** + * Copyright 2022 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "minddata/dataset/text/kernels/filter_wikipedia_xml_op.h" + +#include +#include +#include + +namespace mindspore { +namespace dataset { +std::map patterns = {{R"(<.*>)", ""}, + {R"(&)", "&"}, + {"<", "<"}, + {">", ">"}, + {R"()", ""}, + {"<[^>]*>", ""}, + {R"(\[http:[^] ]*)", "["}, + {R"(\|thumb)", ""}, + {R"(\|left)", ""}, + {R"(\|right)", ""}, + {R"(\|\d+px)", ""}, + {R"(\[\[image:[^\[\]]*\|)", ""}, + {R"(\[\[category:([^|\]]*)[^]]*\]\])", "[[$1]]"}, + {R"(\[\[[a-z\-]*:[^\]]*\]\])", ""}, + {R"(\[\[[^\|\]]*\|)", "[["}, + {R"(\{\{[^\}]*\}\})", ""}, + {R"(\{[^\}]*\})", ""}, + {R"(\[)", ""}, + {R"(\])", ""}, + {"&[^;]*;", " "}, + {"A", "a"}, + {"B", "b"}, + {"C", "c"}, + {"D", "d"}, + {"E", "e"}, + {"F", "f"}, + {"G", "g"}, + {"H", "h"}, + {"I", "i"}, + {"J", "j"}, + {"K", "k"}, + {"L", "l"}, + {"M", "m"}, + {"N", "n"}, + {"O", "o"}, + {"P", "p"}, + {"Q", "q"}, + {"R", ""}, + {"S", "s"}, + {"T", "t"}, + {"U", "u"}, + {"V", "v"}, + {"W", "w"}, + {"X", "x"}, + {"Y", "y"}, + {"Z", "z"}, + {"0", " zero "}, + {"1", " one "}, + {"2", " two "}, + {"3", " three "}, + {"4", " four "}, + {"5", " five "}, + {"6", " six "}, + {"7", " seven "}, + {"8", " eight "}, + {"9", " nine "}, + {R"([^a-z\n]+)", " "}, + {R"(\n )", ""}, + {R"(\s+)", " "}, + {R"(\n\s*\n)", R"(\n)"}}; + +Status FilterWikipediaXMLOp::FilterWikipediaXML(const std::string_view &text, std::string *out) const { + if (((text).find("#redirect") == -1) && ((text).find("#REDIRECT") == -1)) { + (*out) = text; + UErrorCode icu_error = U_ZERO_ERROR; + for (auto pattern_iter = patterns.begin(); pattern_iter != patterns.end(); pattern_iter++) { + icu::RegexMatcher matcher(pattern_iter->first, 0, icu_error); + CHECK_FAIL_RETURN_UNEXPECTED(U_SUCCESS(icu_error), + "RegexReplace: create icu RegexMatcher failed, you may input an error pattern."); + CHECK_FAIL_RETURN_UNEXPECTED((out != nullptr), "FilterWikipediaXML: icu init failed."); + icu::UnicodeString unicode_text = icu::UnicodeString::fromUTF8(*out); + matcher.reset(unicode_text); + icu::UnicodeString unicode_out = matcher.replaceAll(pattern_iter->second, icu_error); + CHECK_FAIL_RETURN_UNEXPECTED(U_SUCCESS(icu_error), "FilterWikipediaXML: FilterWikipediaXML failed."); + (*out) = ""; + unicode_out.trim().toUTF8String(*out); + } + } else { + (*out) = ""; + } + return Status::OK(); +} + +Status FilterWikipediaXMLOp::Compute(const std::shared_ptr &input, std::shared_ptr *output) { + IO_CHECK(input, output); + CHECK_FAIL_RETURN_UNEXPECTED(input->type() == DataType::DE_STRING, "RegexReplace: input is not of type string."); + std::vector strs(input->Size()); + auto iter = input->begin(); + RETURN_IF_NOT_OK(FilterWikipediaXML(*iter, &strs[0])); + RETURN_IF_NOT_OK(Tensor::CreateFromVector(strs, input->shape(), output)); + return Status::OK(); +} +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/minddata/dataset/text/kernels/filter_wikipedia_xml_op.h b/mindspore/ccsrc/minddata/dataset/text/kernels/filter_wikipedia_xml_op.h new file mode 100644 index 00000000000..3c71f8ed989 --- /dev/null +++ b/mindspore/ccsrc/minddata/dataset/text/kernels/filter_wikipedia_xml_op.h @@ -0,0 +1,49 @@ +/** + * Copyright 2022 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_MINDDATA_DATASET_TEXT_KERNELS_FILTER_WIKIPEDIA_XML_OP_H_ +#define MINDSPORE_CCSRC_MINDDATA_DATASET_TEXT_KERNELS_FILTER_WIKIPEDIA_XML_OP_H_ + +#include +#include +#include + +#include "unicode/errorcode.h" +#include "unicode/regex.h" +#include "unicode/utypes.h" + +#include "minddata/dataset/core/tensor.h" +#include "minddata/dataset/kernels/tensor_op.h" +#include "minddata/dataset/text/kernels/whitespace_tokenizer_op.h" +#include "minddata/dataset/util/status.h" + +namespace mindspore { +namespace dataset { +class FilterWikipediaXMLOp : public TensorOp { + public: + FilterWikipediaXMLOp() {} + + ~FilterWikipediaXMLOp() override = default; + + Status Compute(const std::shared_ptr &input, std::shared_ptr *output) override; + + std::string Name() const override { return kFilterWikipediaXMLOp; } + + private: + Status FilterWikipediaXML(const std::string_view &text, std::string *out) const; +}; +} // namespace dataset +} // namespace mindspore +#endif // MINDSPORE_CCSRC_MINDDATA_DATASET_TEXT_KERNELS_FILTER_WIKIPEDIA_XML_OP_H_ diff --git a/mindspore/python/mindspore/dataset/text/transforms.py b/mindspore/python/mindspore/dataset/text/transforms.py index 032f4494d57..c618cd9cc00 100644 --- a/mindspore/python/mindspore/dataset/text/transforms.py +++ b/mindspore/python/mindspore/dataset/text/transforms.py @@ -758,6 +758,22 @@ if platform.system().lower() != 'windows': return cde.CaseFoldOperation() + class FilterWikipediaXML(TextTensorOperation): + """ + Filter Wikipedia XML dumps to "clean" text consisting only of lowercase letters (a-z, converted from A-Z), + and spaces (never consecutive). + + Note: + FilterWikipediaXML is not supported on Windows platform yet. + + Examples: + >>> replace_op = text.FilterWikipediaXML() + >>> text_file_dataset = text_file_dataset.map(operations=replace_op) + """ + def parse(self): + return cde.FilterWikipediaXMLOperation() + + class NormalizeUTF8(TextTensorOperation): """ Apply normalize operation on UTF-8 string tensor. diff --git a/tests/ut/cpp/dataset/c_api_text_test.cc b/tests/ut/cpp/dataset/c_api_text_test.cc index 8396032a89e..ce6a51bccef 100644 --- a/tests/ut/cpp/dataset/c_api_text_test.cc +++ b/tests/ut/cpp/dataset/c_api_text_test.cc @@ -769,6 +769,55 @@ TEST_F(MindDataTestPipeline, TestCaseFoldSuccess) { iter->Stop(); } +/// Feature: FilterWikipediaXML +/// Description: test FilterWikipediaXML in pipeline mode +/// Expectation: the data is processed successfully +TEST_F(MindDataTestPipeline, TestFilterWikipediaXMLSuccess) { + // Testing the parameter of FilterWikipediaXML interface . + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestFilterWikipediaXMLSuccess."; + + // Create a TextFile dataset + std::string data_file = datasets_root_path_ + "/testTokenizerData/2.txt"; + std::shared_ptr ds = TextFile({data_file}, 0, ShuffleMode::kFalse); + EXPECT_NE(ds, nullptr); + + // Create filter_wikipedia_xml operation on ds + std::shared_ptr filter_wikipedia_xml = std::make_shared(); + EXPECT_NE(filter_wikipedia_xml, nullptr); + + // Create Map operation on ds + ds = ds->Map({filter_wikipedia_xml}, {"text"}); + EXPECT_NE(ds, nullptr); + + // Create an iterator over the result of the above dataset + // This will trigger the creation of the Execution Tree and launch it. + std::shared_ptr iter = ds->CreateIterator(); + EXPECT_NE(iter, nullptr); + + // Iterate the dataset and get each row + std::unordered_map row; + ASSERT_OK(iter->GetNextRow(&row)); + std::vector expected = {"welcome to beijing","",""}; + + uint64_t i = 0; + + while (row.size() != 0) { + auto ind = row["text"]; + std::shared_ptr de_expected_tensor; + ASSERT_OK(Tensor::CreateScalar(expected[i], &de_expected_tensor)); + mindspore::MSTensor ms_expected_tensor = + mindspore::MSTensor(std::make_shared(de_expected_tensor)); + EXPECT_MSTENSOR_EQ(ind, ms_expected_tensor); + ASSERT_OK(iter->GetNextRow(&row)); + i++; + } + + EXPECT_EQ(i, 3); + + // Manually terminate the pipeline + iter->Stop(); +} + TEST_F(MindDataTestPipeline, TestJiebaTokenizerSuccess) { // Testing the parameter of JiebaTokenizer interface when the mode is JiebaMode::kMp and the with_offsets is false. MS_LOG(INFO) << "Doing MindDataTestPipeline-TestJiebaTokenizerSuccess."; diff --git a/tests/ut/cpp/dataset/execute_test.cc b/tests/ut/cpp/dataset/execute_test.cc index e521eb0eff7..afeb3a6c2f9 100644 --- a/tests/ut/cpp/dataset/execute_test.cc +++ b/tests/ut/cpp/dataset/execute_test.cc @@ -254,6 +254,24 @@ TEST_F(MindDataTestExecute, TestCrop) { EXPECT_EQ(image.Shape()[1], 15); } +/// Feature: FilterWikipediaXMLEager +/// Description: Test FilterWikipediaXML's Eager mode +/// Expectation: Run successfully +TEST_F(MindDataTestExecute, TestFilterWikipediaXMLEager) { + // Test FilterWikipediaXML's Eager mode + MS_LOG(INFO) << "Doing MindDataTestExecute-TestFilterWikipediaXMLEager."; + std::vector origin = {"中国","Wcdma","Pang","Yuchao"}; + TensorShape input_shape({2, 2}); + std::shared_ptr de_tensor; + Tensor::CreateFromVector(origin, input_shape, &de_tensor); + std::shared_ptr filter = std::make_shared(); + auto input = mindspore::MSTensor(std::make_shared(de_tensor)); + mindspore::dataset::Execute Transform({filter}); + Status s = Transform(input, &input); + + ASSERT_TRUE(s.IsOk()); +} + TEST_F(MindDataTestExecute, TestFrequencyMasking) { MS_LOG(INFO) << "Doing MindDataTestExecute-TestFrequencyMasking."; std::shared_ptr input_tensor_; diff --git a/tests/ut/data/dataset/testTokenizerData/2.txt b/tests/ut/data/dataset/testTokenizerData/2.txt new file mode 100644 index 00000000000..0cf242595d5 --- /dev/null +++ b/tests/ut/data/dataset/testTokenizerData/2.txt @@ -0,0 +1,3 @@ +Welcome to Beijing! +中国 +#redirect \ No newline at end of file diff --git a/tests/ut/python/dataset/test_filter_wikipedia_xml.py b/tests/ut/python/dataset/test_filter_wikipedia_xml.py new file mode 100644 index 00000000000..156fbdb2d1f --- /dev/null +++ b/tests/ut/python/dataset/test_filter_wikipedia_xml.py @@ -0,0 +1,71 @@ +# Copyright 2022 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +""" +Testing FilterWikipediaXML op +""" +import numpy as np + +import mindspore.dataset as ds +import mindspore.dataset.text.transforms as a_c_trans +from mindspore import log as logger + + +def count_unequal_element(data_expected, data_me): + assert data_expected.shape == data_me.shape + assert data_expected == data_me + + +def test_filter_wikipedia_xml_eager(): + """ + Feature: FilterWikipediaXML + Description: test FilterWikipediaXML in eager mode + Expectation: the data is processed successfully + """ + logger.info("test FilterWikipediaXML in eager mode") + + # Original text + input01 = np.array(["Welcome to China"], dtype=np.unicode_) + # Expect text + expected = np.array(["welcome to china"], dtype=np.unicode_) + filter_wikipedia_xml_op = a_c_trans.FilterWikipediaXML() + output = filter_wikipedia_xml_op(input01) + count_unequal_element(expected, output) + + +def test_filter_wikipedia_xml_pipeline(): + """ + Feature: FilterWikipediaXML + Description: test FilterWikipediaXML in pipeline mode + Expectation: the data is processed successfully + """ + logger.info("test FilterWikipediaXML in pipeline mode") + + # Original text + input02 = np.array(["Welcome to China", "中国", "ABC"], dtype=np.unicode_) + # Expect text + expected = np.array(["welcome to china", "", "abc"], dtype=np.string_) + dataset = ds.NumpySlicesDataset(input02, ["text"], shuffle=False) + filter_wikipedia_xml_op = a_c_trans.FilterWikipediaXML() + # Filtered waveform by filter_wikipedia_xml + dataset = dataset.map(input_columns=["text"], operations=filter_wikipedia_xml_op, num_parallel_workers=8) + i = 0 + for data in dataset.create_dict_iterator(output_numpy=True): + count_unequal_element(np.array(expected[i]), data['text']) + i += 1 + + +if __name__ == "__main__": + test_filter_wikipedia_xml_eager() + test_filter_wikipedia_xml_pipeline()