natas_hw
/
cassandra
mirror of https://github.com/apache/cassandra
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add basic text analysis to SAI, including "case_sensitive", "normalize", and "ascii" modes 

patch by Mike Adamson; reviewed by Caleb Rackliffe and Andres de la Peña for CASSANDRA-18479
This commit is contained in:
Mike Adamson 2023-05-09 12:29:01 +01:00 committed by Caleb Rackliffe
parent cba3e19ccd
commit eb208d3561
22 changed files with 904 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
.build/cassandra-deps-template.xml
View File

@ -324,10 +324,6 @@
<groupId>org.hdrhistogram</groupId>
<artifactId>HdrHistogram</artifactId>
</dependency>
<dependency>
<groupId>com.github.rholder</groupId>
<artifactId>snowball-stemmer</artifactId>
</dependency>
<dependency>
<groupId>com.googlecode.concurrent-trees</groupId>
<artifactId>concurrent-trees</artifactId>
@ -376,5 +372,9 @@
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-core</artifactId>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-analysis-common</artifactId>
</dependency>
</dependencies>
</project>

10
.build/parent-pom-template.xml
View File

@ -882,11 +882,6 @@
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.github.rholder</groupId>
<artifactId>snowball-stemmer</artifactId>
<version>1.3.0.581.1</version>
</dependency>
<dependency>
<groupId>com.googlecode.concurrent-trees</groupId>
<artifactId>concurrent-trees</artifactId>
@ -1055,6 +1050,11 @@
<artifactId>lucene-core</artifactId>
<version>9.7.0</version>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-analysis-common</artifactId>
<version>9.7.0</version>
</dependency>
<dependency>
<groupId>com.carrotsearch.randomizedtesting</groupId>
<artifactId>randomizedtesting-runner</artifactId>

31
src/java/org/apache/cassandra/index/sai/IndexContext.java
View File

@ -90,8 +90,7 @@ public class IndexContext
private final IndexViewManager viewManager;
private final IndexMetrics indexMetrics;
private final ColumnQueryMetrics columnQueryMetrics;
private final AbstractAnalyzer.AnalyzerFactory indexAnalyzerFactory;
private final AbstractAnalyzer.AnalyzerFactory queryAnalyzerFactory;
private final AbstractAnalyzer.AnalyzerFactory analyzerFactory;
private final PrimaryKey.Factory primaryKeyFactory;
public IndexContext(String keyspace,
@ -119,9 +118,8 @@ public class IndexContext
this.columnQueryMetrics = isLiteral() ? new ColumnQueryMetrics.TrieIndexMetrics(this)
: new ColumnQueryMetrics.BalancedTreeIndexMetrics(this);
// We currently only support the NoOpAnalyzer
this.indexAnalyzerFactory = AbstractAnalyzer.fromOptions(getValidator(), Collections.emptyMap());
this.queryAnalyzerFactory = AbstractAnalyzer.fromOptions(getValidator(), Collections.emptyMap());
this.analyzerFactory = indexMetadata == null ? AbstractAnalyzer.fromOptions(getValidator(), Collections.emptyMap())
: AbstractAnalyzer.fromOptions(getValidator(), indexMetadata.options);
}
public AbstractType<?> keyValidator()
@ -199,23 +197,12 @@ public class IndexContext
}
/**
* Returns an {@code AnalyzerFactory} for use by the write path to transform incoming literal
* during indexing. The analyzers can be tokenising or non-tokenising. Tokenising analyzers
* will split the incoming terms into multiple terms in the index while non-tokenising analyzers
* will not split the incoming term but will transform the term (e.g. case-insensitive)
* Returns an {@link AbstractAnalyzer.AnalyzerFactory} for use by write and query paths to transform
* literal values.
*/
public AbstractAnalyzer.AnalyzerFactory getIndexAnalyzerFactory()
public AbstractAnalyzer.AnalyzerFactory getAnalyzerFactory()
{
return indexAnalyzerFactory;
}
/**
* Return an {@code AnalyzerFactory} for use by the query path to transform query terms before
* searching for them in the index. This can be the same as the indexAnalyzerFactory.
*/
public AbstractAnalyzer.AnalyzerFactory getQueryAnalyzerFactory()
{
return queryAnalyzerFactory;
return analyzerFactory;
}
public View getView()
@ -256,9 +243,7 @@ public class IndexContext
public void invalidate()
{
viewManager.invalidate();
indexAnalyzerFactory.close();
if (queryAnalyzerFactory != indexAnalyzerFactory)
queryAnalyzerFactory.close();
analyzerFactory.close();
if (memtableIndexManager != null)
memtableIndexManager.invalidate();
if (indexMetrics != null)

6
src/java/org/apache/cassandra/index/sai/StorageAttachedIndex.java
View File

@ -74,6 +74,7 @@ import org.apache.cassandra.index.IndexRegistry;
import org.apache.cassandra.index.SecondaryIndexBuilder;
import org.apache.cassandra.index.TargetParser;
import org.apache.cassandra.index.sai.analyzer.AbstractAnalyzer;
import org.apache.cassandra.index.sai.analyzer.NonTokenizingOptions;
import org.apache.cassandra.index.sai.disk.SSTableIndex;
import org.apache.cassandra.index.sai.disk.format.IndexDescriptor;
import org.apache.cassandra.index.sai.disk.format.Version;
@ -139,7 +140,10 @@ public class StorageAttachedIndex implements Index
private static final StorageAttachedIndexBuildingSupport INDEX_BUILDER_SUPPORT = new StorageAttachedIndexBuildingSupport();
private static final Set<String> VALID_OPTIONS = ImmutableSet.of(IndexTarget.TARGET_OPTION_NAME,
IndexTarget.CUSTOM_INDEX_OPTION_NAME);
IndexTarget.CUSTOM_INDEX_OPTION_NAME,
NonTokenizingOptions.CASE_SENSITIVE,
NonTokenizingOptions.NORMALIZE,
NonTokenizingOptions.ASCII);
public static final Set<CQL3Type> SUPPORTED_TYPES = ImmutableSet.of(CQL3Type.Native.ASCII, CQL3Type.Native.BIGINT, CQL3Type.Native.DATE,
CQL3Type.Native.DOUBLE, CQL3Type.Native.FLOAT, CQL3Type.Native.INT,

26
src/java/org/apache/cassandra/index/sai/analyzer/AbstractAnalyzer.java
View File

@ -24,10 +24,13 @@ import java.util.Map;
import java.util.NoSuchElementException;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.db.marshal.StringType;
import org.apache.cassandra.exceptions.InvalidRequestException;
public abstract class AbstractAnalyzer implements Iterator<ByteBuffer>
{
protected ByteBuffer next = null;
protected String nextLiteral = null;
/**
* @return true if index value is transformed, e.g. normalized or lower-cased or tokenized.
@ -65,6 +68,7 @@ public abstract class AbstractAnalyzer implements Iterator<ByteBuffer>
public void reset(ByteBuffer input)
{
this.next = null;
this.nextLiteral = null;
resetInternal(input);
}
@ -80,6 +84,28 @@ public abstract class AbstractAnalyzer implements Iterator<ByteBuffer>
public static AnalyzerFactory fromOptions(AbstractType<?> type, Map<String, String> options)
{
if (hasNonTokenizingOptions(options))
{
if (type instanceof StringType)
{
// validate options
NonTokenizingOptions.fromMap(options);
return () -> new NonTokenizingAnalyzer(type, options);
}
else
{
throw new InvalidRequestException("CQL type " + type.asCQL3Type() + " cannot be analyzed.");
}
}
return NoOpAnalyzer::new;
}
private static boolean hasNonTokenizingOptions(Map<String, String> options)
{
return options.get(NonTokenizingOptions.ASCII) != null ||
options.containsKey(NonTokenizingOptions.CASE_SENSITIVE) ||
options.containsKey(NonTokenizingOptions.NORMALIZE);
}
}

147
src/java/org/apache/cassandra/index/sai/analyzer/NonTokenizingAnalyzer.java Normal file
View File

@ -0,0 +1,147 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.index.sai.analyzer;
import java.nio.ByteBuffer;
import java.util.Map;
import com.google.common.base.MoreObjects;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.db.marshal.StringType;
import org.apache.cassandra.index.sai.analyzer.filter.BasicFilters;
import org.apache.cassandra.index.sai.analyzer.filter.FilterPipeline;
import org.apache.cassandra.index.sai.analyzer.filter.FilterPipelineExecutor;
import org.apache.cassandra.serializers.MarshalException;
import org.apache.cassandra.utils.ByteBufferUtil;
/**
* Analyzer that does *not* tokenize the input. Optionally will
* apply filters for the input based on {@link NonTokenizingOptions}.
*/
public class NonTokenizingAnalyzer extends AbstractAnalyzer
{
private static final Logger logger = LoggerFactory.getLogger(NonTokenizingAnalyzer.class);
private final AbstractType<?> type;
private final NonTokenizingOptions options;
private final FilterPipeline filterPipeline;
private ByteBuffer input;
private boolean hasNext = false;
NonTokenizingAnalyzer(AbstractType<?> type, Map<String, String> options)
{
this(type, NonTokenizingOptions.fromMap(options));
}
NonTokenizingAnalyzer(AbstractType<?> type, NonTokenizingOptions tokenizerOptions)
{
this.type = type;
this.options = tokenizerOptions;
this.filterPipeline = getFilterPipeline();
}
@Override
public boolean hasNext()
{
// check that we know how to handle the input, otherwise bail
if (!(type instanceof StringType)) return false;
if (hasNext)
{
try
{
String input = type.getString(this.input);
if (input == null)
{
throw new MarshalException(String.format("'null' deserialized value for %s with %s",
ByteBufferUtil.bytesToHex(this.input), type));
}
String result = FilterPipelineExecutor.execute(filterPipeline, input);
if (result == null)
{
nextLiteral = null;
next = null;
return false;
}
nextLiteral = result;
next = type.fromString(result);
return true;
}
catch (MarshalException e)
{
logger.error("Failed to deserialize value with " + type, e);
return false;
}
finally
{
hasNext = false;
}
}
return false;
}
@Override
public boolean transformValue()
{
return !options.isCaseSensitive() || options.isNormalized() || options.isAscii();
}
@Override
protected void resetInternal(ByteBuffer input)
{
this.input = input;
this.hasNext = true;
}
private FilterPipeline getFilterPipeline()
{
FilterPipeline builder = new FilterPipeline(new BasicFilters.NoOperation());
if (!options.isCaseSensitive())
builder = builder.add("to_lower", new BasicFilters.LowerCase());
if (options.isNormalized())
builder = builder.add("normalize", new BasicFilters.Normalize());
if (options.isAscii())
builder = builder.add("ascii", new BasicFilters.Ascii());
return builder;
}
@Override
public String toString()
{
return MoreObjects.toStringHelper(this)
.add("caseSensitive", options.isCaseSensitive())
.add("normalized", options.isNormalized())
.add("ascii", options.isAscii())
.toString();
}
}

156
src/java/org/apache/cassandra/index/sai/analyzer/NonTokenizingOptions.java Normal file
View File

@ -0,0 +1,156 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.index.sai.analyzer;
import java.util.HashMap;
import java.util.Map;
import com.google.common.base.Strings;
import org.apache.cassandra.exceptions.InvalidRequestException;
public class NonTokenizingOptions
{
public static final String NORMALIZE = "normalize";
public static final String CASE_SENSITIVE = "case_sensitive";
public static final String ASCII = "ascii";
private boolean caseSensitive;
private boolean normalized;
private boolean ascii;
boolean isCaseSensitive()
{
return caseSensitive;
}
void setCaseSensitive(boolean caseSensitive)
{
this.caseSensitive = caseSensitive;
}
boolean isNormalized()
{
return this.normalized;
}
void setAscii(boolean ascii)
{
this.ascii = ascii;
}
boolean isAscii()
{
return this.ascii;
}
void setNormalized(boolean normalized)
{
this.normalized = normalized;
}
public static class OptionsBuilder
{
private boolean caseSensitive = true;
private boolean normalized = false;
private boolean ascii = false;
OptionsBuilder() {}
OptionsBuilder caseSensitive(boolean caseSensitive)
{
this.caseSensitive = caseSensitive;
return this;
}
OptionsBuilder ascii(boolean ascii)
{
this.ascii = ascii;
return this;
}
OptionsBuilder normalized(boolean normalized)
{
this.normalized = normalized;
return this;
}
public NonTokenizingOptions build()
{
NonTokenizingOptions options = new NonTokenizingOptions();
options.setCaseSensitive(caseSensitive);
options.setNormalized(normalized);
options.setAscii(ascii);
return options;
}
}
public static NonTokenizingOptions getDefaultOptions()
{
return fromMap(new HashMap<>(1));
}
public static NonTokenizingOptions fromMap(Map<String, String> options)
{
OptionsBuilder builder = new OptionsBuilder();
for (Map.Entry<String, String> entry : options.entrySet())
{
switch (entry.getKey())
{
case CASE_SENSITIVE:
{
boolean boolValue = validateBoolean(entry.getValue(), CASE_SENSITIVE);
builder = builder.caseSensitive(boolValue);
break;
}
case NORMALIZE:
{
boolean boolValue = validateBoolean(entry.getValue(), NORMALIZE);
builder = builder.normalized(boolValue);
break;
}
case ASCII:
{
boolean boolValue = validateBoolean(entry.getValue(), ASCII);
builder = builder.ascii(boolValue);
break;
}
}
}
return builder.build();
}
private static boolean validateBoolean(String value, String option)
{
if (Strings.isNullOrEmpty(value))
{
throw new InvalidRequestException("Empty value for boolean option '" + option + '\'');
}
if (!value.equalsIgnoreCase(Boolean.TRUE.toString()) && !value.equalsIgnoreCase(Boolean.FALSE.toString()))
{
throw new InvalidRequestException("Illegal value for boolean option '" + option + "': " + value);
}
return Boolean.parseBoolean(value);
}
}

82
src/java/org/apache/cassandra/index/sai/analyzer/filter/BasicFilters.java Normal file
View File

@ -0,0 +1,82 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.index.sai.analyzer.filter;
import java.text.Normalizer;
import java.util.Locale;
import org.apache.lucene.analysis.miscellaneous.ASCIIFoldingFilter;
public class BasicFilters
{
private static final Locale DEFAULT_LOCALE = Locale.getDefault();
public static class LowerCase extends FilterPipeline.Task
{
private final Locale locale;
public LowerCase()
{
this.locale = DEFAULT_LOCALE;
}
@Override
public String process(String input)
{
return input.toLowerCase(locale);
}
}
public static class Normalize extends FilterPipeline.Task
{
public Normalize() { }
@Override
public String process(String input)
{
if (input == null) return null;
return Normalizer.isNormalized(input, Normalizer.Form.NFC) ? input : Normalizer.normalize(input, Normalizer.Form.NFC);
}
}
public static class Ascii extends FilterPipeline.Task
{
public Ascii() { }
@Override
public String process(String input)
{
if (input == null) return null;
char[] inputChars = input.toCharArray();
// The output can (potentially) be 4 times the size of the input
char[] outputChars = new char[inputChars.length * 4];
int outputSize = ASCIIFoldingFilter.foldToASCII(inputChars, 0, outputChars, 0, inputChars.length);
return new String(outputChars, 0, outputSize);
}
}
public static class NoOperation extends FilterPipeline.Task
{
@Override
public String process(String input)
{
return input;
}
}
}

69
src/java/org/apache/cassandra/index/sai/analyzer/filter/FilterPipeline.java Normal file
View File

@ -0,0 +1,69 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.index.sai.analyzer.filter;
import javax.annotation.concurrent.NotThreadSafe;
import com.google.common.base.Preconditions;
/**
* A linked list of {@link Task} objects. Used to apply a sequence of filtering tasks
* to provided textual input in a guaranteed order.
*/
@NotThreadSafe
public class FilterPipeline
{
private final Task head;
private Task tail;
public FilterPipeline(Task first)
{
this(first, first);
}
private FilterPipeline(Task first, Task tail)
{
this.head = first;
this.tail = tail;
}
public FilterPipeline add(String name, Task task)
{
Preconditions.checkArgument(task != this.tail, "Provided last task [" + task.name + "] cannot be set to itself");
this.tail.next = task;
this.tail.name = name;
this.tail = task;
return this;
}
public Task head()
{
return this.head;
}
public abstract static class Task
{
public String name;
public Task next;
public abstract String process(String input);
}
}

40
src/java/org/apache/cassandra/index/sai/analyzer/filter/FilterPipelineExecutor.java Normal file
View File

@ -0,0 +1,40 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.index.sai.analyzer.filter;
/**
* Executes all linked {@link FilterPipeline.Task}s serially on the provided input and returns a result
*/
public class FilterPipelineExecutor
{
public static String execute(FilterPipeline pipeline, String initialInput)
{
FilterPipeline.Task currentTask = pipeline.head();
String result = initialInput;
while (true)
{
result = currentTask.process(result);
currentTask = currentTask.next;
if (currentTask == null)
return result;
}
}
}

2
src/java/org/apache/cassandra/index/sai/disk/v1/SSTableIndexWriter.java
View File

@ -79,7 +79,7 @@ public class SSTableIndexWriter implements PerColumnIndexWriter
{
this.indexDescriptor = indexDescriptor;
this.indexContext = indexContext;
this.analyzer = indexContext.getIndexAnalyzerFactory().create();
this.analyzer = indexContext.getAnalyzerFactory().create();
this.limiter = limiter;
this.isIndexValid = isIndexValid;
this.maxTermSize = indexContext.isFrozen() ? MAX_FROZEN_TERM_SIZE : MAX_STRING_TERM_SIZE;

2
src/java/org/apache/cassandra/index/sai/memory/TrieMemoryIndex.java
View File

@ -77,7 +77,7 @@ public class TrieMemoryIndex
this.data = new InMemoryTrie<>(TrieMemtable.BUFFER_TYPE);
this.primaryKeysReducer = new PrimaryKeysReducer();
// The use of the analyzer is within a synchronized block so can be considered thread-safe
this.analyzerFactory = indexContext.getIndexAnalyzerFactory();
this.analyzerFactory = indexContext.getAnalyzerFactory();
this.validator = indexContext.getValidator();
this.isLiteral = TypeUtil.isLiteral(validator);
}

2
src/java/org/apache/cassandra/index/sai/plan/Expression.java
View File

@ -93,7 +93,7 @@ public class Expression
public Expression(IndexContext indexContext)
{
this.context = indexContext;
this.analyzerFactory = indexContext.getQueryAnalyzerFactory();
this.analyzerFactory = indexContext.getAnalyzerFactory();
this.validator = indexContext.getValidator();
}

2
src/java/org/apache/cassandra/index/sai/plan/Operation.java
View File

@ -76,7 +76,7 @@ public class Operation
IndexContext indexContext = controller.getContext(e);
List<Expression> perColumn = analyzed.get(e.column());
AbstractAnalyzer analyzer = indexContext.getQueryAnalyzerFactory().create();
AbstractAnalyzer analyzer = indexContext.getAnalyzerFactory().create();
try
{
analyzer.reset(e.getIndexValue().duplicate());

2
src/java/org/apache/cassandra/index/sai/plan/StorageAttachedIndexSearcher.java
View File

@ -86,7 +86,7 @@ public class StorageAttachedIndexSearcher implements Index.Searcher
{
for (RowFilter.Expression expression : queryController.filterOperation())
{
AbstractAnalyzer analyzer = queryController.getContext(expression).getIndexAnalyzerFactory().create();
AbstractAnalyzer analyzer = queryController.getContext(expression).getAnalyzerFactory().create();
try
{
if (analyzer.transformValue())

2
src/java/org/apache/cassandra/index/sai/virtual/ColumnIndexesSystemView.java
View File

@ -104,7 +104,7 @@ public class ColumnIndexesSystemView extends AbstractVirtualTable
.column(IS_QUERYABLE, manager.isIndexQueryable(index))
.column(IS_BUILDING, manager.isIndexBuilding(indexName))
.column(IS_STRING, context.isLiteral())
.column(ANALYZER, context.getIndexAnalyzerFactory().toString());
.column(ANALYZER, context.getAnalyzerFactory().toString());
}
}
}

55
src/java/org/apache/cassandra/index/sasi/analyzer/filter/StemmerFactory.java
View File

@ -22,16 +22,29 @@ import java.util.HashMap;
import java.util.Locale;
import java.util.Map;
import org.apache.cassandra.concurrent.ImmediateExecutor;
import org.tartarus.snowball.SnowballStemmer;
import org.tartarus.snowball.ext.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.github.benmanes.caffeine.cache.CacheLoader;
import com.github.benmanes.caffeine.cache.Caffeine;
import com.github.benmanes.caffeine.cache.LoadingCache;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.concurrent.ImmediateExecutor;
import org.tartarus.snowball.SnowballStemmer;
import org.tartarus.snowball.ext.DanishStemmer;
import org.tartarus.snowball.ext.DutchStemmer;
import org.tartarus.snowball.ext.EnglishStemmer;
import org.tartarus.snowball.ext.FinnishStemmer;
import org.tartarus.snowball.ext.FrenchStemmer;
import org.tartarus.snowball.ext.GermanStemmer;
import org.tartarus.snowball.ext.HungarianStemmer;
import org.tartarus.snowball.ext.ItalianStemmer;
import org.tartarus.snowball.ext.NorwegianStemmer;
import org.tartarus.snowball.ext.PortugueseStemmer;
import org.tartarus.snowball.ext.RomanianStemmer;
import org.tartarus.snowball.ext.RussianStemmer;
import org.tartarus.snowball.ext.SpanishStemmer;
import org.tartarus.snowball.ext.SwedishStemmer;
import org.tartarus.snowball.ext.TurkishStemmer;
/**
* Returns a SnowballStemmer instance appropriate for
@ -63,21 +76,21 @@ public class StemmerFactory
static
{
SUPPORTED_LANGUAGES = new HashMap<>();
SUPPORTED_LANGUAGES.put("de", germanStemmer.class);
SUPPORTED_LANGUAGES.put("da", danishStemmer.class);
SUPPORTED_LANGUAGES.put("es", spanishStemmer.class);
SUPPORTED_LANGUAGES.put("en", englishStemmer.class);
SUPPORTED_LANGUAGES.put("fl", finnishStemmer.class);
SUPPORTED_LANGUAGES.put("fr", frenchStemmer.class);
SUPPORTED_LANGUAGES.put("hu", hungarianStemmer.class);
SUPPORTED_LANGUAGES.put("it", italianStemmer.class);
SUPPORTED_LANGUAGES.put("nl", dutchStemmer.class);
SUPPORTED_LANGUAGES.put("no", norwegianStemmer.class);
SUPPORTED_LANGUAGES.put("pt", portugueseStemmer.class);
SUPPORTED_LANGUAGES.put("ro", romanianStemmer.class);
SUPPORTED_LANGUAGES.put("ru", russianStemmer.class);
SUPPORTED_LANGUAGES.put("sv", swedishStemmer.class);
SUPPORTED_LANGUAGES.put("tr", turkishStemmer.class);
SUPPORTED_LANGUAGES.put("de", GermanStemmer.class);
SUPPORTED_LANGUAGES.put("da", DanishStemmer.class);
SUPPORTED_LANGUAGES.put("es", SpanishStemmer.class);
SUPPORTED_LANGUAGES.put("en", EnglishStemmer.class);
SUPPORTED_LANGUAGES.put("fl", FinnishStemmer.class);
SUPPORTED_LANGUAGES.put("fr", FrenchStemmer.class);
SUPPORTED_LANGUAGES.put("hu", HungarianStemmer.class);
SUPPORTED_LANGUAGES.put("it", ItalianStemmer.class);
SUPPORTED_LANGUAGES.put("nl", DutchStemmer.class);
SUPPORTED_LANGUAGES.put("no", NorwegianStemmer.class);
SUPPORTED_LANGUAGES.put("pt", PortugueseStemmer.class);
SUPPORTED_LANGUAGES.put("ro", RomanianStemmer.class);
SUPPORTED_LANGUAGES.put("ru", RussianStemmer.class);
SUPPORTED_LANGUAGES.put("sv", SwedishStemmer.class);
SUPPORTED_LANGUAGES.put("tr", TurkishStemmer.class);
}
public static SnowballStemmer getStemmer(Locale locale)

67
test/distributed/org/apache/cassandra/distributed/test/sai/ReplicaFilteringProtectionTest.java Normal file
View File

@ -0,0 +1,67 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.distributed.test.sai;
import java.io.IOException;
import org.junit.Test;
import org.apache.cassandra.distributed.Cluster;
import org.apache.cassandra.distributed.api.SimpleQueryResult;
import org.apache.cassandra.distributed.test.TestBaseImpl;
import static org.apache.cassandra.distributed.api.ConsistencyLevel.ALL;
import static org.apache.cassandra.distributed.shared.AssertUtils.assertRows;
import static org.apache.cassandra.distributed.shared.AssertUtils.row;
public class ReplicaFilteringProtectionTest extends TestBaseImpl
{
private static final int REPLICAS = 2;
@Test
public void testRFPWithIndexTransformations() throws IOException
{
try (Cluster cluster = init(Cluster.build()
.withNodes(REPLICAS)
.withConfig(config -> config.set("hinted_handoff_enabled", false)
.set("commitlog_sync", "batch")).start()))
{
String tableName = "sai_rfp";
String fullTableName = KEYSPACE + '.' + tableName;
cluster.schemaChange("CREATE TABLE " + fullTableName + " (k int PRIMARY KEY, v text)");
cluster.schemaChange("CREATE CUSTOM INDEX ON " + fullTableName + "(v) USING 'StorageAttachedIndex' " +
"WITH OPTIONS = { 'case_sensitive' : false}");
// both nodes have the old value
cluster.coordinator(1).execute("INSERT INTO " + fullTableName + "(k, v) VALUES (0, 'OLD')", ALL);
String select = "SELECT * FROM " + fullTableName + " WHERE v = 'old'";
Object[][] initialRows = cluster.coordinator(1).execute(select, ALL);
assertRows(initialRows, row(0, "OLD"));
// only one node gets the new value
cluster.get(1).executeInternal("UPDATE " + fullTableName + " SET v = 'new' WHERE k = 0");
// querying by the old value shouldn't return the old surviving row
SimpleQueryResult oldResult = cluster.coordinator(1).executeWithResult(select, ALL);
assertRows(oldResult.toObjectArrays());
}
}
}

75
test/unit/org/apache/cassandra/index/sai/analyzer/NonTokenizingAnalyzerTest.java Normal file
View File

@ -0,0 +1,75 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.index.sai.analyzer;
import java.nio.ByteBuffer;
import org.junit.Test;
import org.apache.cassandra.db.marshal.UTF8Type;
import org.apache.cassandra.utils.ByteBufferUtil;
import static org.junit.Assert.assertEquals;
public class NonTokenizingAnalyzerTest
{
@Test
public void asciiAnalyzer() throws Exception
{
NonTokenizingOptions options = NonTokenizingOptions.getDefaultOptions();
options.setCaseSensitive(false);
options.setAscii(true);
assertEquals("eppinger", getAnalyzedString("Éppinger", options));
}
@Test
public void asciiAnalyzerFalse() throws Exception
{
NonTokenizingOptions options = NonTokenizingOptions.getDefaultOptions();
options.setCaseSensitive(true);
options.setAscii(false);
assertEquals("Éppinger", getAnalyzedString("Éppinger", options));
}
@Test
public void caseInsensitiveAnalyzer() throws Exception
{
NonTokenizingOptions options = NonTokenizingOptions.getDefaultOptions();
options.setCaseSensitive(false);
assertEquals("nip it in the bud", getAnalyzedString("Nip it in the bud", options));
}
@Test
public void caseSensitiveAnalyzer() throws Exception
{
NonTokenizingOptions options = NonTokenizingOptions.getDefaultOptions();
assertEquals("Nip it in the bud", getAnalyzedString("Nip it in the bud", options));
}
private String getAnalyzedString(String input, NonTokenizingOptions options) throws Exception
{
NonTokenizingAnalyzer analyzer = new NonTokenizingAnalyzer(UTF8Type.instance, options);
analyzer.reset(ByteBuffer.wrap(input.getBytes()));
return analyzer.hasNext() ? ByteBufferUtil.string(analyzer.next) : null;
}
}

73
test/unit/org/apache/cassandra/index/sai/analyzer/filter/BasicFiltersTest.java Normal file
View File

@ -0,0 +1,73 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.index.sai.analyzer.filter;
import java.text.Normalizer;
import org.junit.Test;
import org.apache.cassandra.index.sai.SAITester;
import org.apache.lucene.analysis.miscellaneous.ASCIIFoldingFilter;
import static org.junit.Assert.assertEquals;
public class BasicFiltersTest
{
@Test
public void testLowerCase()
{
BasicFilters.LowerCase lowerCase = new BasicFilters.LowerCase();
for (int count = 0; count < SAITester.getRandom().nextIntBetween(100, 1000); count++)
{
String actual = SAITester.getRandom().nextTextString(10, 50);
assertEquals(actual.toLowerCase(), lowerCase.process(actual));
}
}
@Test
public void testNormalize()
{
BasicFilters.Normalize normalize = new BasicFilters.Normalize();
for (int count = 0; count < SAITester.getRandom().nextIntBetween(100, 1000); count++)
{
String actual = SAITester.getRandom().nextTextString(10, 50);
assertEquals(Normalizer.normalize(actual, Normalizer.Form.NFC), normalize.process(actual));
}
}
@Test
public void testAscii()
{
BasicFilters.Ascii ascii = new BasicFilters.Ascii();
for (int count = 0; count < SAITester.getRandom().nextIntBetween(100, 1000); count++)
{
String actual = SAITester.getRandom().nextTextString(100, 5000);
char[] actualChars = actual.toCharArray();
char[] expectedChars = new char[actualChars.length * 4];
int expectedSize = ASCIIFoldingFilter.foldToASCII(actualChars, 0, expectedChars, 0, actualChars.length);
String expected = new String(expectedChars, 0, expectedSize);
assertEquals(expected, ascii.process(actual));
}
}
}

106
test/unit/org/apache/cassandra/index/sai/cql/StorageAttachedIndexDDLTest.java
View File

@ -264,6 +264,112 @@ public class StorageAttachedIndexDDLTest extends SAITester
assertEquals(1, saiCreationCounter.get());
}
@Test
public void shouldBeCaseSensitiveByDefault()
{
createTable("CREATE TABLE %s (id text PRIMARY KEY, val text)");
createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex'");
execute("INSERT INTO %s (id, val) VALUES ('1', 'Camel')");
assertEquals(1, execute("SELECT id FROM %s WHERE val = 'Camel'").size());
assertEquals(0, execute("SELECT id FROM %s WHERE val = 'camel'").size());
}
@Test
public void shouldEnableCaseSensitiveSearch()
{
createTable("CREATE TABLE %s (id text PRIMARY KEY, val text)");
createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex' WITH OPTIONS = { 'case_sensitive' : true }");
execute("INSERT INTO %s (id, val) VALUES ('1', 'Camel')");
assertEquals(1, execute("SELECT id FROM %s WHERE val = 'Camel'").size());
assertEquals(0, execute("SELECT id FROM %s WHERE val = 'camel'").size());
}
@Test
public void shouldEnableCaseInsensitiveSearch()
{
createTable("CREATE TABLE %s (id text PRIMARY KEY, val text)");
createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex' WITH OPTIONS = { 'case_sensitive' : false }");
execute("INSERT INTO %s (id, val) VALUES ('1', 'Camel')");
assertEquals(1, execute("SELECT id FROM %s WHERE val = 'camel'").size());
}
@Test
public void shouldBeNonNormalizedByDefault()
{
createTable("CREATE TABLE %s (id text PRIMARY KEY, val text)");
createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex'");
execute("INSERT INTO %s (id, val) VALUES ('1', 'Cam\u00E1l')");
assertEquals(1, execute("SELECT id FROM %s WHERE val = 'Cam\u00E1l'").size());
// Both \u00E1 and \u0061\u0301 are visible as the character á, but without NFC normalization, they won't match.
assertEquals(0, execute("SELECT id FROM %s WHERE val = 'Cam\u0061\u0301l'").size());
}
@Test
public void shouldEnableNonNormalizedSearch()
{
createTable("CREATE TABLE %s (id text PRIMARY KEY, val text)");
createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex' WITH OPTIONS = { 'normalize' : false }");
execute("INSERT INTO %s (id, val) VALUES ('1', 'Cam\u00E1l')");
assertEquals(1, execute("SELECT id FROM %s WHERE val = 'Cam\u00E1l'").size());
// Both \u00E1 and \u0061\u0301 are visible as the character á, but without NFC normalization, they won't match.
assertEquals(0, execute("SELECT id FROM %s WHERE val = 'Cam\u0061\u0301l'").size());
}
@Test
public void shouldEnableNormalizedSearch()
{
createTable("CREATE TABLE %s (id text PRIMARY KEY, val text)");
createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex' WITH OPTIONS = { 'normalize' : true }");
execute("INSERT INTO %s (id, val) VALUES ('1', 'Cam\u00E1l')");
assertEquals(1, execute("SELECT id FROM %s WHERE val = 'Cam\u0061\u0301l'").size());
}
@Test
public void shouldEnableNormalizedCaseInsensitiveSearch()
{
createTable("CREATE TABLE %s (id text PRIMARY KEY, val text)");
createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex' WITH OPTIONS = { 'normalize' : true, 'case_sensitive' : false}");
execute("INSERT INTO %s (id, val) VALUES ('1', 'Cam\u00E1l')");
assertEquals(1, execute("SELECT id FROM %s WHERE val = 'cam\u0061\u0301l'").size());
}
@Test
public void shouldEnableAsciiSearch()
{
createTable("CREATE TABLE %s (id text PRIMARY KEY, val text)");
createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex' WITH OPTIONS = { 'ascii' : true, 'case_sensitive' : false}");
execute("INSERT INTO %s (id, val) VALUES ('1', 'Éppinger')");
assertEquals(1, execute("SELECT id FROM %s WHERE val = 'eppinger'").size());
}
@Test
public void shouldCreateIndexOnReversedType() throws Throwable
{

2
test/unit/org/apache/cassandra/index/sai/virtual/IndexesSystemViewTest.java
View File

@ -129,6 +129,6 @@ public class IndexesSystemViewTest extends SAITester
isQueryable,
isBuilding,
isString,
context.getIndexAnalyzerFactory().toString());
context.getAnalyzerFactory().toString());
}
}