natas_hw
/
cassandra
mirror of https://github.com/apache/cassandra
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Correct english word stemming test and add a test for french 

patch by doanduyhai; reviewed by xedin for CASSANDRA-12078
This commit is contained in:
Pavel Yaskevich 2016-06-26 01:48:23 -07:00
parent f1cabcade8
commit eb82861c8d
3 changed files with 34 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/java/org/apache/cassandra/index/sasi/analyzer/filter/StemmingFilters.java
View File

@ -37,7 +37,7 @@ public class StemmingFilters
public String process(String input) throws Exception
{
if (stemmer == null)
if (input == null || stemmer == null)
return input;
stemmer.setCurrent(input);
return (stemmer.stem()) ? stemmer.getCurrent() : input;

1
test/resources/tokenization/french_skip_stop_words_before_stemming.txt Normal file
View File

@ -0,0 +1 @@
"La danse sous la pluie" est une chanson connue

33
test/unit/org/apache/cassandra/index/sasi/analyzer/StandardAnalyzerTest.java
View File

@ -25,6 +25,8 @@ import java.util.Locale;
import org.junit.Test;
import org.apache.cassandra.serializers.UTF8Serializer;
import static org.junit.Assert.assertEquals;
public class StandardAnalyzerTest
@ -151,7 +153,36 @@ public class StandardAnalyzerTest
while (tokenizer.hasNext())
tokens.add(tokenizer.next());
assertEquals(40249, tokens.size());
assertEquals(37739, tokens.size());
}
@Test
public void testSkipStopWordBeforeStemmingFrench() throws Exception
{
InputStream is = StandardAnalyzerTest.class.getClassLoader()
.getResourceAsStream("tokenization/french_skip_stop_words_before_stemming.txt");
StandardTokenizerOptions options = new StandardTokenizerOptions.OptionsBuilder().stemTerms(true)
.ignoreStopTerms(true).useLocale(Locale.FRENCH)
.alwaysLowerCaseTerms(true).build();
StandardAnalyzer tokenizer = new StandardAnalyzer();
tokenizer.init(options);
List<ByteBuffer> tokens = new ArrayList<>();
List<String> words = new ArrayList<>();
tokenizer.reset(is);
while (tokenizer.hasNext())
{
final ByteBuffer nextToken = tokenizer.next();
tokens.add(nextToken);
words.add(UTF8Serializer.instance.deserialize(nextToken.duplicate()));
}
assertEquals(4, tokens.size());
assertEquals("dans", words.get(0));
assertEquals("plui", words.get(1));
assertEquals("chanson", words.get(2));
assertEquals("connu", words.get(3));
}
@Test