natas_hw
/
cassandra
mirror of https://github.com/apache/cassandra
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'cassandra-3.11' into trunk

This commit is contained in:
Mick Semb Wever 2018-03-15 17:42:31 +11:00
parent d4dfbb5c67 4c6a10af8b
commit 92cde9b27c
No known key found for this signature in database
GPG Key ID: E91335D77E3E87CB
6 changed files with 479 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGES.txt
View File

@ -201,6 +201,7 @@
3.11.3
* SASI tokenizer for simple delimiter based entries (CASSANDRA-14247)
* Fix Loss of digits when doing CAST from varint/bigint to decimal (CASSANDRA-14170)
* RateBasedBackPressure unnecessarily invokes a lock on the Guava RateLimiter (CASSANDRA-14163)
* Fix wildcard GROUP BY queries (CASSANDRA-14209)

21
doc/SASI.md
View File

@ -247,6 +247,27 @@ cqlsh:demo> SELECT * FROM sasi WHERE last_name LIKE '%a%' AND height >= 175 ALLO
(4 rows)
```
#### Delimiter based Tokenization Analysis
A simple text analysis provided is delimiter based tokenization. This provides an alternative to indexing collections,
as delimiter separated text can be indexed without the overhead of `CONTAINS` mode nor using `PREFIX` or `SUFFIX` queries.
```
cqlsh:demo> ALTER TABLE sasi ADD aliases text;
cqlsh:demo> CREATE CUSTOM INDEX on sasi (aliases) USING 'org.apache.cassandra.index.sasi.SASIIndex'
... WITH OPTIONS = {
... 'analyzer_class': 'org.apache.cassandra.index.sasi.analyzer.DelimiterAnalyzer',
... 'delimiter': ',',
... 'mode': 'prefix',
... 'analyzed': 'true'};
cqlsh:demo> UPDATE sasi SET aliases = 'Mike,Mick,Mikey,Mickey' WHERE id = f5dfcabe-de96-4148-9b80-a1c41ed276b4;
cqlsh:demo> SELECT * FROM sasi WHERE aliases LIKE 'Mikey' ALLOW FILTERING;
id | age | aliases | created_at | first_name | height | last_name
--------------------------------------+-----+------------------------+---------------+------------+--------+-----------
f5dfcabe-de96-4148-9b80-a1c41ed276b4 | 26 | Mike,Mick,Mikey,Mickey | 1442959315021 | Michael | 180 | Kjellman
```
#### Text Analysis (Tokenization and Stemming)
Lastly, to demonstrate text analysis an additional column is needed on

103
src/java/org/apache/cassandra/index/sasi/analyzer/DelimiterAnalyzer.java Normal file
View File

@ -0,0 +1,103 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.index.sasi.analyzer;
import java.nio.CharBuffer;
import java.nio.ByteBuffer;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import com.google.common.annotations.Beta;
import com.google.common.base.Preconditions;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.db.marshal.AsciiType;
import org.apache.cassandra.db.marshal.UTF8Type;
import org.apache.cassandra.utils.AbstractIterator;
@Beta
public class DelimiterAnalyzer extends AbstractAnalyzer
{
private static final Map<AbstractType<?>,Charset> VALID_ANALYZABLE_TYPES = new HashMap<AbstractType<?>,Charset>()
{{
put(UTF8Type.instance, StandardCharsets.UTF_8);
put(AsciiType.instance, StandardCharsets.US_ASCII);
}};
private char delimiter;
private Charset charset;
private Iterator<ByteBuffer> iter;
public DelimiterAnalyzer()
{
}
public ByteBuffer next()
{
return iter.next();
}
public void init(Map<String, String> options, AbstractType validator)
{
DelimiterTokenizingOptions tokenizingOptions = DelimiterTokenizingOptions.buildFromMap(options);
delimiter = tokenizingOptions.getDelimiter();
if (!VALID_ANALYZABLE_TYPES.containsKey(validator))
throw new IllegalArgumentException(String.format("Only text types supported, got %s", validator));
charset = VALID_ANALYZABLE_TYPES.get(validator);
}
public boolean hasNext()
{
return iter.hasNext();
}
public void reset(ByteBuffer input)
{
Preconditions.checkNotNull(input);
final CharBuffer cb = charset.decode(input);
this.iter = new AbstractIterator<ByteBuffer>() {
protected ByteBuffer computeNext() {
if (!cb.hasRemaining())
return endOfData();
CharBuffer readahead = cb.duplicate();
// loop until we see the next delimiter character, or reach end of data
while (readahead.hasRemaining() && readahead.get() != delimiter);
char[] chars = new char[readahead.position() - cb.position() - (readahead.hasRemaining() ? 1 : 0)];
cb.get(chars);
Preconditions.checkState(!cb.hasRemaining() || cb.get() == delimiter);
return charset.encode(CharBuffer.wrap(chars));
}
};
}
public boolean isTokenizing()
{
return true;
}
}

71
src/java/org/apache/cassandra/index/sasi/analyzer/DelimiterTokenizingOptions.java Normal file
View File

@ -0,0 +1,71 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.index.sasi.analyzer;
import java.util.Map;
/** Simple tokenizer based on a specified delimiter (rather than whitespace).
*/
public class DelimiterTokenizingOptions
{
public static final String DELIMITER = "delimiter";
private final char delimiter;
private DelimiterTokenizingOptions(char delimiter)
{
this.delimiter = delimiter;
}
char getDelimiter()
{
return delimiter;
}
private static class OptionsBuilder
{
private char delimiter = ',';
public DelimiterTokenizingOptions build()
{
return new DelimiterTokenizingOptions(delimiter);
}
}
static DelimiterTokenizingOptions buildFromMap(Map<String, String> optionsMap)
{
OptionsBuilder optionsBuilder = new OptionsBuilder();
for (Map.Entry<String, String> entry : optionsMap.entrySet())
{
switch (entry.getKey())
{
case DELIMITER:
{
String value = entry.getValue();
if (1 != value.length())
throw new IllegalArgumentException(String.format("Only single character delimiters supported, was %s", value));
optionsBuilder.delimiter = entry.getValue().charAt(0);
break;
}
}
}
return optionsBuilder.build();
}
}

1
test/resources/tokenization/world_cities_a.csv Normal file
View File

File diff suppressed because one or more lines are too long

282
test/unit/org/apache/cassandra/index/sasi/analyzer/DelimiterAnalyzerTest.java Normal file
View File

@ -0,0 +1,282 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.index.sasi.analyzer;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import org.apache.cassandra.db.marshal.Int32Type;
import org.apache.cassandra.db.marshal.UTF8Type;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.commons.io.IOUtils;
import org.junit.Assert;
import org.junit.Test;
import static org.junit.Assert.assertEquals;
public class DelimiterAnalyzerTest
{
@Test
public void caseSensitiveAnalizer() throws Exception
{
DelimiterAnalyzer analyzer = new DelimiterAnalyzer();
analyzer.init(
new HashMap()
{{
put(DelimiterTokenizingOptions.DELIMITER, " ");
}},
UTF8Type.instance);
String testString = "Nip it in the bud";
ByteBuffer toAnalyze = ByteBuffer.wrap(testString.getBytes());
analyzer.reset(toAnalyze);
StringBuilder output = new StringBuilder();
while (analyzer.hasNext())
output.append(ByteBufferUtil.string(analyzer.next()) + (analyzer.hasNext() ? ' ' : ""));
Assert.assertTrue(testString.equals(output.toString()));
Assert.assertFalse(testString.toLowerCase().equals(output.toString()));
}
@Test(expected = IllegalArgumentException.class)
public void ensureIncompatibleInputSkipped() throws Exception
{
new DelimiterAnalyzer().init(new HashMap(), Int32Type.instance);
}
@Test
public void testTokenizationLoremIpsum() throws Exception
{
ByteBuffer bb = ByteBuffer.wrap(IOUtils.toByteArray(
DelimiterAnalyzerTest.class.getClassLoader().getResourceAsStream("tokenization/lorem_ipsum.txt")));
DelimiterAnalyzer tokenizer = new DelimiterAnalyzer();
tokenizer.init(
new HashMap()
{{
put(DelimiterTokenizingOptions.DELIMITER, " ");
}},
UTF8Type.instance);
List<ByteBuffer> tokens = new ArrayList<>();
tokenizer.reset(bb);
while (tokenizer.hasNext())
tokens.add(tokenizer.next());
assertEquals(69, tokens.size());
}
@Test
public void testTokenizationJaJp1() throws Exception
{
ByteBuffer bb = ByteBuffer.wrap(IOUtils.toByteArray(
DelimiterAnalyzerTest.class.getClassLoader().getResourceAsStream("tokenization/ja_jp_1.txt")));
DelimiterAnalyzer tokenizer = new DelimiterAnalyzer();
tokenizer.init(
new HashMap()
{{
put(DelimiterTokenizingOptions.DELIMITER, "。");
}},
UTF8Type.instance);
tokenizer.reset(bb);
List<ByteBuffer> tokens = new ArrayList<>();
while (tokenizer.hasNext())
tokens.add(tokenizer.next());
assertEquals(4, tokens.size());
}
@Test
public void testTokenizationJaJp2() throws Exception
{
ByteBuffer bb = ByteBuffer.wrap(IOUtils.toByteArray(
DelimiterAnalyzerTest.class.getClassLoader().getResourceAsStream("tokenization/ja_jp_2.txt")));
DelimiterAnalyzer tokenizer = new DelimiterAnalyzer();
tokenizer.init(
new HashMap()
{{
put(DelimiterTokenizingOptions.DELIMITER, "。");
}},
UTF8Type.instance);
tokenizer.reset(bb);
List<ByteBuffer> tokens = new ArrayList<>();
while (tokenizer.hasNext())
tokens.add(tokenizer.next());
assertEquals(2, tokens.size());
}
@Test
public void testTokenizationRuRu1() throws Exception
{
ByteBuffer bb = ByteBuffer.wrap(IOUtils.toByteArray(
DelimiterAnalyzerTest.class.getClassLoader().getResourceAsStream("tokenization/ru_ru_1.txt")));
DelimiterAnalyzer tokenizer = new DelimiterAnalyzer();
tokenizer.init(
new HashMap()
{{
put(DelimiterTokenizingOptions.DELIMITER, " ");
}},
UTF8Type.instance);
List<ByteBuffer> tokens = new ArrayList<>();
tokenizer.reset(bb);
while (tokenizer.hasNext())
tokens.add(tokenizer.next());
assertEquals(447, tokens.size());
}
@Test
public void testTokenizationZnTw1() throws Exception
{
ByteBuffer bb = ByteBuffer.wrap(IOUtils.toByteArray(
DelimiterAnalyzerTest.class.getClassLoader().getResourceAsStream("tokenization/zn_tw_1.txt")));
DelimiterAnalyzer tokenizer = new DelimiterAnalyzer();
tokenizer.init(
new HashMap()
{{
put(DelimiterTokenizingOptions.DELIMITER, " ");
}},
UTF8Type.instance);
List<ByteBuffer> tokens = new ArrayList<>();
tokenizer.reset(bb);
while (tokenizer.hasNext())
tokens.add(tokenizer.next());
assertEquals(403, tokens.size());
}
@Test
public void testTokenizationAdventuresOfHuckFinn() throws Exception
{
ByteBuffer bb = ByteBuffer.wrap(IOUtils.toByteArray(
DelimiterAnalyzerTest.class.getClassLoader().getResourceAsStream("tokenization/adventures_of_huckleberry_finn_mark_twain.txt")));
DelimiterAnalyzer tokenizer = new DelimiterAnalyzer();
tokenizer.init(
new HashMap()
{{
put(DelimiterTokenizingOptions.DELIMITER, " ");
}},
UTF8Type.instance);
List<ByteBuffer> tokens = new ArrayList<>();
tokenizer.reset(bb);
while (tokenizer.hasNext())
tokens.add(tokenizer.next());
assertEquals(104594, tokens.size());
}
@Test
public void testWorldCities() throws Exception
{
ByteBuffer bb = ByteBuffer.wrap(IOUtils.toByteArray(
DelimiterAnalyzerTest.class.getClassLoader().getResourceAsStream("tokenization/world_cities_a.csv")));
DelimiterAnalyzer tokenizer = new DelimiterAnalyzer();
tokenizer.init(
new HashMap()
{{
put(DelimiterTokenizingOptions.DELIMITER, ",");
}},
UTF8Type.instance);
List<ByteBuffer> tokens = new ArrayList<>();
tokenizer.reset(bb);
while (tokenizer.hasNext())
tokens.add(tokenizer.next());
assertEquals(122265, tokens.size());
}
@Test
public void tokenizeDomainNamesAndUrls() throws Exception
{
ByteBuffer bb = ByteBuffer.wrap(IOUtils.toByteArray(
DelimiterAnalyzerTest.class.getClassLoader().getResourceAsStream("tokenization/top_visited_domains.txt")));
DelimiterAnalyzer tokenizer = new DelimiterAnalyzer();
tokenizer.init(
new HashMap()
{{
put(DelimiterTokenizingOptions.DELIMITER, " ");
}},
UTF8Type.instance);
tokenizer.reset(bb);
List<ByteBuffer> tokens = new ArrayList<>();
while (tokenizer.hasNext())
tokens.add(tokenizer.next());
assertEquals(12, tokens.size());
}
@Test
public void testReuseAndResetTokenizerInstance() throws Exception
{
List<ByteBuffer> bbToTokenize = new ArrayList<>();
bbToTokenize.add(ByteBuffer.wrap("Nip it in the bud".getBytes()));
bbToTokenize.add(ByteBuffer.wrap("I couldn’t care less".getBytes()));
bbToTokenize.add(ByteBuffer.wrap("One and the same".getBytes()));
bbToTokenize.add(ByteBuffer.wrap("The squeaky wheel gets the grease.".getBytes()));
bbToTokenize.add(ByteBuffer.wrap("The pen is mightier than the sword.".getBytes()));
DelimiterAnalyzer tokenizer = new DelimiterAnalyzer();
tokenizer.init(
new HashMap()
{{
put(DelimiterTokenizingOptions.DELIMITER, " ");
}},
UTF8Type.instance);
List<ByteBuffer> tokens = new ArrayList<>();
for (ByteBuffer bb : bbToTokenize)
{
tokenizer.reset(bb);
while (tokenizer.hasNext())
tokens.add(tokenizer.next());
}
assertEquals(26, tokens.size());
}
}