natas_hw
/
cassandra
mirror of https://github.com/apache/cassandra
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Improve code model around IndexContext 

- Replace IndexContext with IndexTermType and IndexDefinition
 - Move index specific managers, factories and metrics to StorageAttachedIndex
 - Refactor Expression to explicitly define indexed and unindexed expressions

 patch by Mike Adamson; reviewed by Andres de la Peña, Caleb Rackliffe for CASSANDRA-18166
This commit is contained in:
Mike Adamson 2023-11-10 14:49:41 +00:00 committed by Mick Semb Wever
parent 9c6e671434
commit 0e42b77c97
No known key found for this signature in database
GPG Key ID: E91335D77E3E87CB
108 changed files with 3440 additions and 3066 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGES.txt
View File

@ -1,4 +1,5 @@
5.0-beta1
* Improve SAI IndexContext handling of indexed and non-indexed columns in queries (CASSANDRA-18166)
* Fixed bug where UnifiedCompactionTask constructor was calling the wrong base constructor of CompactionTask (CASSANDRA-18757)
* Fix SAI unindexed contexts not considering CONTAINS KEY (CASSANDRA-19040)
* Ensure that empty SAI column indexes do not fail on validation after full-SSTable streaming (CASSANDRA-19017)

588
src/java/org/apache/cassandra/index/sai/IndexContext.java
View File

@ -1,588 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.index.sai;
import java.nio.ByteBuffer;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import javax.annotation.Nullable;
import com.google.common.base.MoreObjects;
import com.google.common.collect.ImmutableSet;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.cql3.Operator;
import org.apache.cassandra.cql3.statements.schema.IndexTarget;
import org.apache.cassandra.db.ClusteringComparator;
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.db.marshal.AsciiType;
import org.apache.cassandra.db.marshal.BooleanType;
import org.apache.cassandra.db.marshal.CompositeType;
import org.apache.cassandra.db.marshal.FloatType;
import org.apache.cassandra.db.marshal.UTF8Type;
import org.apache.cassandra.db.marshal.UUIDType;
import org.apache.cassandra.db.marshal.VectorType;
import org.apache.cassandra.db.rows.Cell;
import org.apache.cassandra.db.rows.Row;
import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.index.sai.analyzer.AbstractAnalyzer;
import org.apache.cassandra.index.sai.disk.SSTableIndex;
import org.apache.cassandra.index.sai.disk.format.Version;
import org.apache.cassandra.index.sai.disk.v1.IndexWriterConfig;
import org.apache.cassandra.index.sai.memory.MemtableIndexManager;
import org.apache.cassandra.index.sai.metrics.ColumnQueryMetrics;
import org.apache.cassandra.index.sai.metrics.IndexMetrics;
import org.apache.cassandra.index.sai.plan.Expression;
import org.apache.cassandra.index.sai.utils.PrimaryKey;
import org.apache.cassandra.index.sai.utils.TypeUtil;
import org.apache.cassandra.index.sai.view.IndexViewManager;
import org.apache.cassandra.index.sai.view.View;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.schema.IndexMetadata;
import org.apache.cassandra.service.ClientWarn;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.NoSpamLogger;
import org.apache.cassandra.utils.Pair;
import static org.apache.cassandra.config.CassandraRelevantProperties.SAI_MAX_FROZEN_TERM_SIZE;
import static org.apache.cassandra.config.CassandraRelevantProperties.SAI_MAX_STRING_TERM_SIZE;
import static org.apache.cassandra.config.CassandraRelevantProperties.SAI_MAX_VECTOR_TERM_SIZE;
/**
* Manages metadata for each column index.
*/
public class IndexContext
{
private static final Logger logger = LoggerFactory.getLogger(IndexContext.class);
private static final NoSpamLogger noSpamLogger = NoSpamLogger.getLogger(logger, 1, TimeUnit.MINUTES);
public static final long MAX_STRING_TERM_SIZE = SAI_MAX_STRING_TERM_SIZE.getSizeInBytes();
public static final long MAX_FROZEN_TERM_SIZE = SAI_MAX_FROZEN_TERM_SIZE.getSizeInBytes();
public static final long MAX_VECTOR_TERM_SIZE = SAI_MAX_VECTOR_TERM_SIZE.getSizeInBytes();
public static final String TERM_OVERSIZE_MESSAGE = "Can't add term of column %s to index for key: %s, term size %s " +
"max allowed size %s, use analyzed = true (if not yet set) for that column.";
private static final Set<AbstractType<?>> EQ_ONLY_TYPES = ImmutableSet.of(UTF8Type.instance,
AsciiType.instance,
BooleanType.instance,
UUIDType.instance);
private final AbstractType<?> partitionKeyType;
private final ClusteringComparator clusteringComparator;
private final String keyspace;
private final String table;
private final ColumnMetadata columnMetadata;
private final IndexTarget.Type indexType;
private final AbstractType<?> validator;
// Config can be null if the column context is "fake" (i.e. created for a filtering expression).
@Nullable
private final IndexMetadata indexMetadata;
private final MemtableIndexManager memtableIndexManager;
private final IndexViewManager viewManager;
private final IndexMetrics indexMetrics;
private final ColumnQueryMetrics columnQueryMetrics;
private final IndexWriterConfig indexWriterConfig;
private final AbstractAnalyzer.AnalyzerFactory analyzerFactory;
private final PrimaryKey.Factory primaryKeyFactory;
private final long maxTermSize;
public IndexContext(String keyspace,
String table,
AbstractType<?> partitionKeyType,
IPartitioner partitioner,
ClusteringComparator clusteringComparator,
ColumnMetadata columnMetadata,
IndexTarget.Type indexType,
@Nullable IndexMetadata indexMetadata)
{
this.keyspace = Objects.requireNonNull(keyspace);
this.table = Objects.requireNonNull(table);
this.partitionKeyType = Objects.requireNonNull(partitionKeyType);
this.clusteringComparator = Objects.requireNonNull(clusteringComparator);
this.columnMetadata = Objects.requireNonNull(columnMetadata);
this.indexType = Objects.requireNonNull(indexType);
this.validator = TypeUtil.cellValueType(columnMetadata, indexType);
this.primaryKeyFactory = new PrimaryKey.Factory(partitioner, clusteringComparator);
this.indexMetadata = indexMetadata;
this.memtableIndexManager = indexMetadata == null ? null : new MemtableIndexManager(this);
this.indexWriterConfig = indexMetadata == null ? IndexWriterConfig.emptyConfig()
: IndexWriterConfig.fromOptions(indexMetadata.name, validator, indexMetadata.options);
this.indexMetrics = indexMetadata == null ? null : new IndexMetrics(this);
this.viewManager = new IndexViewManager(this);
this.columnQueryMetrics = isLiteral() ? new ColumnQueryMetrics.TrieIndexMetrics(this)
: new ColumnQueryMetrics.BalancedTreeIndexMetrics(this);
this.analyzerFactory = indexMetadata == null ? AbstractAnalyzer.fromOptions(getValidator(), Collections.emptyMap())
: AbstractAnalyzer.fromOptions(getValidator(), indexMetadata.options);
maxTermSize = isVector() ? MAX_VECTOR_TERM_SIZE : (isFrozen() ? MAX_FROZEN_TERM_SIZE : MAX_STRING_TERM_SIZE);
}
public boolean hasClustering()
{
return clusteringComparator.size() > 0;
}
public AbstractType<?> keyValidator()
{
return partitionKeyType;
}
public PrimaryKey.Factory keyFactory()
{
return primaryKeyFactory;
}
public String getKeyspace()
{
return keyspace;
}
public IndexWriterConfig getIndexWriterConfig()
{
return indexWriterConfig;
}
public IndexMetrics getIndexMetrics()
{
return indexMetrics;
}
public ColumnQueryMetrics getColumnQueryMetrics()
{
return columnQueryMetrics;
}
public String getTable()
{
return table;
}
public IndexMetadata getIndexMetadata()
{
return indexMetadata;
}
/**
* @return A set of SSTables which have attached to them invalid index components.
*/
public Collection<SSTableContext> onSSTableChanged(Collection<SSTableReader> oldSSTables, Collection<SSTableContext> newSSTables, IndexValidation validation)
{
return viewManager.update(oldSSTables, newSSTables, validation);
}
public ColumnMetadata getDefinition()
{
return columnMetadata;
}
public AbstractType<?> getValidator()
{
return validator;
}
public boolean isNonFrozenCollection()
{
return TypeUtil.isNonFrozenCollection(columnMetadata.type);
}
public boolean isFrozen()
{
return TypeUtil.isFrozen(columnMetadata.type);
}
public String getColumnName()
{
return columnMetadata.name.toString();
}
@Nullable
public String getIndexName()
{
return indexMetadata == null ? null : indexMetadata.name;
}
/**
* Returns an {@link AbstractAnalyzer.AnalyzerFactory} for use by write and query paths to transform
* literal values.
*/
public AbstractAnalyzer.AnalyzerFactory getAnalyzerFactory()
{
return analyzerFactory;
}
public View getView()
{
return viewManager.getView();
}
public MemtableIndexManager getMemtableIndexManager()
{
assert memtableIndexManager != null : "Attempt to use memtable index manager on non-indexed context";
return memtableIndexManager;
}
/**
* @return total number of per-index open files
*/
public int openPerIndexFiles()
{
return viewManager.getView().size() * Version.LATEST.onDiskFormat().openFilesPerColumnIndex(this);
}
public void drop(Collection<SSTableReader> sstablesToRebuild)
{
viewManager.drop(sstablesToRebuild);
}
public boolean isNotIndexed()
{
return indexMetadata == null;
}
/**
* Called when index is dropped. Clear all live in-memory indexes and close
* analyzer factories. Mark all {@link SSTableIndex} as released and per-column index files
* will be removed when in-flight queries are completed.
*/
public void invalidate()
{
viewManager.invalidate();
analyzerFactory.close();
if (memtableIndexManager != null)
memtableIndexManager.invalidate();
if (indexMetrics != null)
indexMetrics.release();
if (columnQueryMetrics != null)
columnQueryMetrics.release();
}
public boolean supports(Operator op)
{
if (op == Operator.LIKE ||
op == Operator.LIKE_CONTAINS ||
op == Operator.LIKE_PREFIX ||
op == Operator.LIKE_MATCHES ||
op == Operator.LIKE_SUFFIX) return false;
// ANN is only supported against vectors, and vector indexes only support ANN
if (isVector())
return op == Operator.ANN;
if (op == Operator.ANN)
return false;
Expression.IndexOperator operator = Expression.IndexOperator.valueOf(op);
if (isNonFrozenCollection())
{
if (indexType == IndexTarget.Type.KEYS) return operator == Expression.IndexOperator.CONTAINS_KEY;
if (indexType == IndexTarget.Type.VALUES) return operator == Expression.IndexOperator.CONTAINS_VALUE;
return indexType == IndexTarget.Type.KEYS_AND_VALUES && operator == Expression.IndexOperator.EQ;
}
if (indexType == IndexTarget.Type.FULL)
return operator == Expression.IndexOperator.EQ;
AbstractType<?> validator = getValidator();
if (operator != Expression.IndexOperator.EQ && EQ_ONLY_TYPES.contains(validator)) return false;
// RANGE only applicable to non-literal indexes
return (operator != null) && !(TypeUtil.isLiteral(validator) && operator == Expression.IndexOperator.RANGE);
}
public ByteBuffer getValueOf(DecoratedKey key, Row row, long nowInSecs)
{
if (row == null)
return null;
switch (columnMetadata.kind)
{
case PARTITION_KEY:
return partitionKeyType instanceof CompositeType
? CompositeType.extractComponent(key.getKey(), columnMetadata.position())
: key.getKey();
case CLUSTERING:
// skip indexing of static clustering when regular column is indexed
return row.isStatic() ? null : row.clustering().bufferAt(columnMetadata.position());
// treat static cell retrieval the same was as regular
// only if row kind is STATIC otherwise return null
case STATIC:
if (!row.isStatic())
return null;
case REGULAR:
Cell<?> cell = row.getCell(columnMetadata);
return cell == null || !cell.isLive(nowInSecs) ? null : cell.buffer();
default:
return null;
}
}
public Iterator<ByteBuffer> getValuesOf(Row row, long nowInSecs)
{
if (row == null)
return null;
switch (columnMetadata.kind)
{
// treat static cell retrieval the same was as regular
// only if row kind is STATIC otherwise return null
case STATIC:
if (!row.isStatic())
return null;
case REGULAR:
return TypeUtil.collectionIterator(validator,
row.getComplexColumnData(columnMetadata),
columnMetadata,
indexType,
nowInSecs);
default:
return null;
}
}
@Override
public String toString()
{
return MoreObjects.toStringHelper(this)
.add("columnName", getColumnName())
.add("indexName", getIndexName())
.toString();
}
public boolean isLiteral()
{
return TypeUtil.isLiteral(getValidator());
}
public boolean isVector()
{
return getValidator().isVector() && ((VectorType<?>)getValidator()).elementType instanceof FloatType;
}
@Override
public boolean equals(Object obj)
{
if (obj == this)
return true;
if (!(obj instanceof IndexContext))
return false;
IndexContext other = (IndexContext) obj;
return Objects.equals(columnMetadata, other.columnMetadata) &&
(indexType == other.indexType) &&
Objects.equals(indexMetadata, other.indexMetadata) &&
Objects.equals(partitionKeyType, other.partitionKeyType) &&
Objects.equals(clusteringComparator, other.clusteringComparator);
}
@Override
public int hashCode()
{
return Objects.hash(columnMetadata, indexType, indexMetadata, partitionKeyType, clusteringComparator);
}
/**
* A helper method for constructing consistent log messages for specific column indexes.
* <p>
* Example: For the index "idx" in keyspace "ks" on table "tb", calling this method with the raw message
* "Flushing new index segment..." will produce...
* <p>
* "[ks.tb.idx] Flushing new index segment..."
*
* @param message The raw content of a logging message, without information identifying it with an index.
*
* @return A log message with the proper keyspace, table and index name prepended to it.
*/
public String logMessage(String message)
{
// Index names are unique only within a keyspace.
return String.format("[%s.%s.%s] %s", keyspace, table, indexMetadata == null ? "?" : indexMetadata.name, message);
}
/**
* @return the indexes that are built on the given SSTables on the left and corrupted indexes'
* corresponding contexts on the right
*/
public Pair<Collection<SSTableIndex>, Collection<SSTableContext>> getBuiltIndexes(Collection<SSTableContext> sstableContexts, IndexValidation validation)
{
Set<SSTableIndex> valid = new HashSet<>(sstableContexts.size());
Set<SSTableContext> invalid = new HashSet<>();
for (SSTableContext sstableContext : sstableContexts)
{
if (sstableContext.sstable.isMarkedCompacted())
continue;
if (!sstableContext.indexDescriptor.isPerColumnIndexBuildComplete(this))
{
logger.debug(logMessage("An on-disk index build for SSTable {} has not completed."), sstableContext.descriptor());
continue;
}
if (sstableContext.indexDescriptor.isIndexEmpty(this))
{
logger.debug(logMessage("No on-disk index was built for SSTable {} because the SSTable " +
"had no indexable rows for the index."), sstableContext.descriptor());
continue;
}
try
{
if (validation != IndexValidation.NONE)
{
if (!sstableContext.indexDescriptor.validatePerIndexComponents(this, validation))
{
invalid.add(sstableContext);
continue;
}
}
SSTableIndex index = sstableContext.newSSTableIndex(this);
logger.debug(logMessage("Successfully created index for SSTable {}."), sstableContext.descriptor());
// Try to add new index to the set, if set already has such index, we'll simply release and move on.
// This covers situation when SSTable collection has the same SSTable multiple
// times because we don't know what kind of collection it actually is.
if (!valid.add(index))
{
index.release();
}
}
catch (Throwable e)
{
logger.warn(logMessage("Failed to update per-column components for SSTable {}"), sstableContext.descriptor(), e);
invalid.add(sstableContext);
}
}
return Pair.create(valid, invalid);
}
/**
* @return the number of indexed rows in this index (aka. a pair of term and rowId)
*/
public long getCellCount()
{
return getView().getIndexes()
.stream()
.mapToLong(SSTableIndex::getRowCount)
.sum();
}
/**
* @return the total size (in bytes) of per-column index components
*/
public long diskUsage()
{
return getView().getIndexes()
.stream()
.mapToLong(SSTableIndex::sizeOfPerColumnComponents)
.sum();
}
/**
* @return the total memory usage (in bytes) of per-column index on-disk data structure
*/
public long indexFileCacheSize()
{
return getView().getIndexes()
.stream()
.mapToLong(SSTableIndex::indexFileCacheSize)
.sum();
}
/**
* Validate maximum term size for given row
*/
public void validateMaxTermSizeForRow(DecoratedKey key, Row row, boolean sendClientWarning)
{
AbstractAnalyzer analyzer = getAnalyzerFactory().create();
if (isNonFrozenCollection())
{
Iterator<ByteBuffer> bufferIterator = getValuesOf(row, FBUtilities.nowInSeconds());
while (bufferIterator != null && bufferIterator.hasNext())
validateMaxTermSizeForCell(analyzer, key, bufferIterator.next(), sendClientWarning);
}
else
{
ByteBuffer value = getValueOf(key, row, FBUtilities.nowInSeconds());
validateMaxTermSizeForCell(analyzer, key, value, sendClientWarning);
}
}
private void validateMaxTermSizeForCell(AbstractAnalyzer analyzer, DecoratedKey key, @Nullable ByteBuffer cellBuffer, boolean sendClientWarning)
{
if (cellBuffer == null || cellBuffer.remaining() == 0)
return;
// analyzer should not return terms that are larger than the origin value.
if (cellBuffer.remaining() <= maxTermSize)
return;
analyzer.reset(cellBuffer.duplicate());
while (analyzer.hasNext())
validateMaxTermSize(key, analyzer.next(), sendClientWarning);
}
/**
* Validate maximum term size for given term
* @return true if given term is valid; otherwise false.
*/
public boolean validateMaxTermSize(DecoratedKey key, ByteBuffer term, boolean sendClientWarning)
{
if (term.remaining() > maxTermSize)
{
String message = logMessage(String.format(TERM_OVERSIZE_MESSAGE,
getColumnName(),
keyValidator().getString(key.getKey()),
FBUtilities.prettyPrintMemory(term.remaining()),
FBUtilities.prettyPrintMemory(maxTermSize)));
if (sendClientWarning)
ClientWarn.instance.warn(message);
noSpamLogger.warn(message);
return false;
}
return true;
}
}

4
src/java/org/apache/cassandra/index/sai/SSTableContext.java
View File

@ -104,9 +104,9 @@ public class SSTableContext extends SharedCloseableImpl
/**
* Returns a new {@link SSTableIndex} for a per-column index
*/
public SSTableIndex newSSTableIndex(IndexContext indexContext)
public SSTableIndex newSSTableIndex(StorageAttachedIndex index)
{
return indexDescriptor.newSSTableIndex(this, indexContext);
return indexDescriptor.newSSTableIndex(this, index);
}
/**

865
src/java/org/apache/cassandra/index/sai/StorageAttachedIndex.java
View File

File diff suppressed because it is too large Load Diff

4
src/java/org/apache/cassandra/index/sai/StorageAttachedIndexBuilder.java
View File

@ -152,7 +152,7 @@ public class StorageAttachedIndexBuilder extends SecondaryIndexBuilder
boolean perIndexComponentsOnly = perSSTableFileLock == null;
// remove existing per column index files instead of overwriting
IndexDescriptor indexDescriptor = IndexDescriptor.create(sstable);
indexes.forEach(index -> indexDescriptor.deleteColumnIndex(index.getIndexContext()));
indexes.forEach(index -> indexDescriptor.deleteColumnIndex(index.termType(), index.identifier()));
indexWriter = StorageAttachedIndexWriter.createBuilderWriter(indexDescriptor, indexes, txn, perIndexComponentsOnly);
@ -342,7 +342,7 @@ public class StorageAttachedIndexBuilder extends SecondaryIndexBuilder
if (!dropped.isEmpty())
{
String droppedIndexes = dropped.stream().map(sai -> sai.getIndexContext().getIndexName()).collect(Collectors.toList()).toString();
String droppedIndexes = dropped.stream().map(sai -> sai.identifier().indexName).collect(Collectors.toList()).toString();
if (isFullRebuild)
throw new RuntimeException(logMessage(String.format("%s are dropped, will stop index build.", droppedIndexes)));
else

72
src/java/org/apache/cassandra/index/sai/StorageAttachedIndexBuildingSupport.java Normal file
View File

@ -0,0 +1,72 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.index.sai;
import java.util.Collection;
import java.util.Comparator;
import java.util.HashSet;
import java.util.NavigableMap;
import java.util.Set;
import java.util.TreeMap;
import java.util.stream.Collectors;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.index.Index;
import org.apache.cassandra.index.SecondaryIndexBuilder;
import org.apache.cassandra.index.sai.disk.format.IndexDescriptor;
import org.apache.cassandra.io.sstable.SSTableIdFactory;
import org.apache.cassandra.io.sstable.format.SSTableReader;
class StorageAttachedIndexBuildingSupport implements Index.IndexBuildingSupport
{
@Override
public SecondaryIndexBuilder getIndexBuildTask(ColumnFamilyStore cfs,
Set<Index> indexes,
Collection<SSTableReader> sstablesToRebuild,
boolean isFullRebuild)
{
NavigableMap<SSTableReader, Set<StorageAttachedIndex>> sstables = new TreeMap<>(Comparator.comparing(s -> s.descriptor.id, SSTableIdFactory.COMPARATOR));
StorageAttachedIndexGroup group = StorageAttachedIndexGroup.getIndexGroup(cfs);
assert group != null : "Index group does not exist for table " + cfs.keyspace + '.' + cfs.name;
indexes.stream()
.filter((i) -> i instanceof StorageAttachedIndex)
.forEach((i) ->
{
StorageAttachedIndex sai = (StorageAttachedIndex) i;
// If this is not a full manual index rebuild we can skip SSTables that already have an
// attached index. Otherwise, we override any pre-existent index.
Collection<SSTableReader> ss = sstablesToRebuild;
if (!isFullRebuild)
{
ss = sstablesToRebuild.stream()
.filter(s -> !IndexDescriptor.create(s).isPerColumnIndexBuildComplete(sai.identifier()))
.collect(Collectors.toList());
}
group.dropIndexSSTables(ss, sai);
ss.forEach(sstable -> sstables.computeIfAbsent(sstable, ignore -> new HashSet<>()).add(sai));
});
return new StorageAttachedIndexBuilder(group, sstables, isFullRebuild, false);
}
}

22
src/java/org/apache/cassandra/index/sai/StorageAttachedIndexGroup.java
View File

@ -244,7 +244,7 @@ public class StorageAttachedIndexGroup implements Index.Group, INotificationCons
{
IndexDescriptor indexDescriptor = IndexDescriptor.create(sstable);
Set<Component> components = indexDescriptor.getLivePerSSTableComponents();
indices.forEach(index -> components.addAll(indexDescriptor.getLivePerIndexComponents(index.getIndexContext())));
indices.forEach(index -> components.addAll(indexDescriptor.getLivePerIndexComponents(index.termType(), index.identifier())));
return components;
}
@ -269,11 +269,11 @@ public class StorageAttachedIndexGroup implements Index.Group, INotificationCons
}
else if (notification instanceof MemtableRenewedNotification)
{
indexes.forEach(index -> index.getIndexContext().getMemtableIndexManager().renewMemtable(((MemtableRenewedNotification) notification).renewed));
indexes.forEach(index -> index.memtableIndexManager().renewMemtable(((MemtableRenewedNotification) notification).renewed));
}
else if (notification instanceof MemtableDiscardedNotification)
{
indexes.forEach(index -> index.getIndexContext().getMemtableIndexManager().discardMemtable(((MemtableDiscardedNotification) notification).memtable));
indexes.forEach(index -> index.memtableIndexManager().discardMemtable(((MemtableDiscardedNotification) notification).memtable));
}
}
@ -287,7 +287,7 @@ public class StorageAttachedIndexGroup implements Index.Group, INotificationCons
{
try
{
index.getIndexContext().drop(ss);
index.drop(ss);
}
catch (Throwable t)
{
@ -317,7 +317,7 @@ public class StorageAttachedIndexGroup implements Index.Group, INotificationCons
// Column indexes are invalid if their SSTable-level components are corrupted so delete
// their associated index files and mark them non-queryable.
indexes.forEach(index -> {
indexDescriptor.deleteColumnIndex(index.getIndexContext());
indexDescriptor.deleteColumnIndex(index.termType(), index.identifier());
index.makeIndexNonQueryable();
});
});
@ -328,13 +328,13 @@ public class StorageAttachedIndexGroup implements Index.Group, INotificationCons
for (StorageAttachedIndex index : indexes)
{
Collection<SSTableContext> invalid = index.getIndexContext().onSSTableChanged(removed, results.left, validation);
Collection<SSTableContext> invalid = index.onSSTableChanged(removed, results.left, validation);
if (!invalid.isEmpty())
{
// Delete the index files and mark the index non-queryable, as its view may be compromised,
// and incomplete, for our callers:
invalid.forEach(context -> context.indexDescriptor.deleteColumnIndex(index.getIndexContext()));
invalid.forEach(context -> context.indexDescriptor.deleteColumnIndex(index.termType(), index.identifier()));
index.makeIndexNonQueryable();
incomplete.add(index);
}
@ -357,8 +357,8 @@ public class StorageAttachedIndexGroup implements Index.Group, INotificationCons
for (StorageAttachedIndex index : indexes)
{
if (indexDescriptor.isPerColumnIndexBuildComplete(index.getIndexContext()))
indexDescriptor.checksumPerIndexComponents(index.getIndexContext());
if (indexDescriptor.isPerColumnIndexBuildComplete(index.identifier()))
indexDescriptor.checksumPerIndexComponents(index.termType(), index.identifier());
else if (throwOnIncomplete)
throw new IllegalStateException(indexDescriptor.logMessage("Incomplete per-column index build"));
else
@ -386,7 +386,7 @@ public class StorageAttachedIndexGroup implements Index.Group, INotificationCons
*/
public int openIndexFiles()
{
return contextManager.openFiles() + indexes.stream().mapToInt(index -> index.getIndexContext().openPerIndexFiles()).sum();
return contextManager.openFiles() + indexes.stream().mapToInt(StorageAttachedIndex::openPerColumnIndexFiles).sum();
}
/**
@ -426,7 +426,7 @@ public class StorageAttachedIndexGroup implements Index.Group, INotificationCons
*/
public long totalDiskUsage()
{
return diskUsage() + indexes.stream().flatMap(index -> index.getIndexContext().getView().getIndexes().stream())
return diskUsage() + indexes.stream().flatMap(index -> index.view().getIndexes().stream())
.mapToLong(SSTableIndex::sizeOfPerColumnComponents).sum();
}

13
src/java/org/apache/cassandra/index/sai/analyzer/AbstractAnalyzer.java
View File

@ -24,9 +24,8 @@ import java.util.Map;
import java.util.NoSuchElementException;
import java.util.stream.Collectors;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.db.marshal.StringType;
import org.apache.cassandra.exceptions.InvalidRequestException;
import org.apache.cassandra.index.sai.utils.IndexTermType;
public abstract class AbstractAnalyzer implements Iterator<ByteBuffer>
{
@ -83,23 +82,23 @@ public abstract class AbstractAnalyzer implements Iterator<ByteBuffer>
}
}
public static AnalyzerFactory fromOptions(AbstractType<?> type, Map<String, String> options)
public static AnalyzerFactory fromOptions(IndexTermType indexTermType, Map<String, String> options)
{
if (hasNonTokenizingOptions(options))
{
if (type instanceof StringType)
if (indexTermType.isString())
{
// validate options
NonTokenizingOptions.fromMap(options);
return () -> new NonTokenizingAnalyzer(type, options);
return () -> new NonTokenizingAnalyzer(indexTermType, options);
}
else
{
throw new InvalidRequestException("CQL type " + type.asCQL3Type() + " cannot be analyzed.");
throw new InvalidRequestException("CQL type " + indexTermType.asCQL3Type() + " cannot be analyzed.");
}
}
return NoOpAnalyzer::new;
return null;
}
private static boolean hasNonTokenizingOptions(Map<String, String> options)

66
src/java/org/apache/cassandra/index/sai/analyzer/NoOpAnalyzer.java
View File

@ -1,66 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.index.sai.analyzer;
import java.nio.ByteBuffer;
import com.google.common.base.MoreObjects;
/**
* Default NoOp tokenizer. The iterator will iterate only once returning the unmodified input
*/
public class NoOpAnalyzer extends AbstractAnalyzer
{
private ByteBuffer input;
private boolean hasNext = false;
public NoOpAnalyzer() {}
@Override
public boolean hasNext()
{
if (hasNext)
{
this.next = input;
this.hasNext = false;
return true;
}
this.next = null;
return false;
}
@Override
protected void resetInternal(ByteBuffer input)
{
this.input = input;
this.hasNext = true;
}
@Override
public boolean transformValue()
{
return false;
}
@Override
public String toString()
{
return MoreObjects.toStringHelper(this).toString();
}
}

24
src/java/org/apache/cassandra/index/sai/analyzer/NonTokenizingAnalyzer.java
View File

@ -25,11 +25,10 @@ import com.google.common.base.MoreObjects;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.db.marshal.StringType;
import org.apache.cassandra.index.sai.analyzer.filter.BasicFilters;
import org.apache.cassandra.index.sai.analyzer.filter.FilterPipeline;
import org.apache.cassandra.index.sai.analyzer.filter.FilterPipelineExecutor;
import org.apache.cassandra.index.sai.utils.IndexTermType;
import org.apache.cassandra.serializers.MarshalException;
import org.apache.cassandra.utils.ByteBufferUtil;
@ -41,21 +40,21 @@ public class NonTokenizingAnalyzer extends AbstractAnalyzer
{
private static final Logger logger = LoggerFactory.getLogger(NonTokenizingAnalyzer.class);
private final AbstractType<?> type;
private final IndexTermType indexTermType;
private final NonTokenizingOptions options;
private final FilterPipeline filterPipeline;
private ByteBuffer input;
private boolean hasNext = false;
NonTokenizingAnalyzer(AbstractType<?> type, Map<String, String> options)
NonTokenizingAnalyzer(IndexTermType indexTermType, Map<String, String> options)
{
this(type, NonTokenizingOptions.fromMap(options));
this(indexTermType, NonTokenizingOptions.fromMap(options));
}
NonTokenizingAnalyzer(AbstractType<?> type, NonTokenizingOptions tokenizerOptions)
NonTokenizingAnalyzer(IndexTermType indexTermType, NonTokenizingOptions tokenizerOptions)
{
this.type = type;
this.indexTermType = indexTermType;
this.options = tokenizerOptions;
this.filterPipeline = getFilterPipeline();
}
@ -64,18 +63,19 @@ public class NonTokenizingAnalyzer extends AbstractAnalyzer
public boolean hasNext()
{
// check that we know how to handle the input, otherwise bail
if (!(type instanceof StringType)) return false;
if (!indexTermType.isString())
return false;
if (hasNext)
{
try
{
String input = type.getString(this.input);
String input = indexTermType.asString(this.input);
if (input == null)
{
throw new MarshalException(String.format("'null' deserialized value for %s with %s",
ByteBufferUtil.bytesToHex(this.input), type));
ByteBufferUtil.bytesToHex(this.input), indexTermType));
}
String result = FilterPipelineExecutor.execute(filterPipeline, input);
@ -88,13 +88,13 @@ public class NonTokenizingAnalyzer extends AbstractAnalyzer
}
nextLiteral = result;
next = type.fromString(result);
next = indexTermType.fromString(result);
return true;
}
catch (MarshalException e)
{
logger.error("Failed to deserialize value with " + type, e);
logger.error("Failed to deserialize value with " + indexTermType, e);
return false;
}
finally

8
src/java/org/apache/cassandra/index/sai/disk/IndexSearchResultIterator.java
View File

@ -65,7 +65,7 @@ public class IndexSearchResultIterator extends KeyRangeIterator
{
List<KeyRangeIterator> subIterators = new ArrayList<>(1 + sstableIndexes.size());
KeyRangeIterator memtableIterator = expression.context.getMemtableIndexManager().searchMemtableIndexes(queryContext, expression, keyRange);
KeyRangeIterator memtableIterator = expression.getIndex().memtableIndexManager().searchMemtableIndexes(queryContext, expression, keyRange);
if (memtableIterator != null)
subIterators.add(memtableIterator);
@ -77,7 +77,7 @@ public class IndexSearchResultIterator extends KeyRangeIterator
queryContext.sstablesHit++;
if (sstableIndex.isReleased())
throw new IllegalStateException(sstableIndex.getIndexContext().logMessage("Index was released from the view during the query"));
throw new IllegalStateException(sstableIndex.getIndexIdentifier().logMessage("Index was released from the view during the query"));
List<KeyRangeIterator> indexIterators = sstableIndex.search(expression, keyRange, queryContext);
@ -87,7 +87,7 @@ public class IndexSearchResultIterator extends KeyRangeIterator
catch (Throwable e)
{
if (!(e instanceof QueryCancelledException))
logger.debug(sstableIndex.getIndexContext().logMessage(String.format("Failed search an index %s, aborting query.", sstableIndex.getSSTable())), e);
logger.debug(sstableIndex.getIndexIdentifier().logMessage(String.format("Failed search an index %s, aborting query.", sstableIndex.getSSTable())), e);
throw Throwables.cleaned(e);
}
@ -152,7 +152,7 @@ public class IndexSearchResultIterator extends KeyRangeIterator
}
catch (Throwable e)
{
logger.error(index.getIndexContext().logMessage(String.format("Failed to release index on SSTable %s", index.getSSTable())), e);
logger.error(index.getIndexIdentifier().logMessage(String.format("Failed to release index on SSTable %s", index.getSSTable())), e);
}
}
}

41
src/java/org/apache/cassandra/index/sai/disk/SSTableIndex.java
View File

@ -33,13 +33,15 @@ import org.slf4j.LoggerFactory;
import org.apache.cassandra.db.PartitionPosition;
import org.apache.cassandra.db.virtual.SimpleDataSet;
import org.apache.cassandra.dht.AbstractBounds;
import org.apache.cassandra.index.sai.IndexContext;
import org.apache.cassandra.index.sai.QueryContext;
import org.apache.cassandra.index.sai.SSTableContext;
import org.apache.cassandra.index.sai.StorageAttachedIndex;
import org.apache.cassandra.index.sai.utils.IndexIdentifier;
import org.apache.cassandra.index.sai.disk.format.Version;
import org.apache.cassandra.index.sai.disk.v1.segment.SegmentOrdering;
import org.apache.cassandra.index.sai.iterators.KeyRangeIterator;
import org.apache.cassandra.index.sai.plan.Expression;
import org.apache.cassandra.index.sai.utils.IndexTermType;
import org.apache.cassandra.io.sstable.SSTableIdFactory;
import org.apache.cassandra.io.sstable.format.SSTableReader;
@ -61,17 +63,17 @@ public abstract class SSTableIndex implements SegmentOrdering
.thenComparing(s -> s.getSSTable().descriptor.id, SSTableIdFactory.COMPARATOR);
protected final SSTableContext sstableContext;
protected final IndexContext indexContext;
protected final IndexTermType indexTermType;
protected final IndexIdentifier indexIdentifier;
private final AtomicInteger references = new AtomicInteger(1);
private final AtomicBoolean obsolete = new AtomicBoolean(false);
public SSTableIndex(SSTableContext sstableContext, IndexContext indexContext)
public SSTableIndex(SSTableContext sstableContext, StorageAttachedIndex index)
{
assert indexContext.getValidator() != null;
this.sstableContext = sstableContext.sharedCopy(); // this line must not be before any code that may throw
this.indexContext = indexContext;
this.indexTermType = index.termType();
this.indexIdentifier = index.identifier();
}
/**
@ -152,12 +154,17 @@ public abstract class SSTableIndex implements SegmentOrdering
*/
public long sizeOfPerColumnComponents()
{
return sstableContext.indexDescriptor.sizeOnDiskOfPerIndexComponents(indexContext);
return sstableContext.indexDescriptor.sizeOnDiskOfPerIndexComponents(indexTermType, indexIdentifier);
}
public IndexContext getIndexContext()
public IndexTermType getIndexTermType()
{
return indexContext;
return indexTermType;
}
public IndexIdentifier getIndexIdentifier()
{
return indexIdentifier;
}
public SSTableContext getSSTableContext()
@ -202,7 +209,7 @@ public abstract class SSTableIndex implements SegmentOrdering
}
catch (Throwable e)
{
logger.error(getIndexContext().logMessage("Failed to release index on SSTable {}"), getSSTable().descriptor, e);
logger.error(indexIdentifier.logMessage("Failed to release index on SSTable {}"), getSSTable().descriptor, e);
}
}
@ -221,7 +228,7 @@ public abstract class SSTableIndex implements SegmentOrdering
*/
if (obsolete.get())
{
sstableContext.indexDescriptor.deleteColumnIndex(indexContext);
sstableContext.indexDescriptor.deleteColumnIndex(indexTermType, indexIdentifier);
}
}
}
@ -238,23 +245,25 @@ public abstract class SSTableIndex implements SegmentOrdering
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
SSTableIndex other = (SSTableIndex)o;
return Objects.equal(sstableContext, other.sstableContext) && Objects.equal(indexContext, other.indexContext);
return Objects.equal(sstableContext, other.sstableContext) &&
Objects.equal(indexTermType, other.indexTermType) &&
Objects.equal(indexIdentifier, other.indexIdentifier);
}
@Override
public int hashCode()
{
return Objects.hashCode(sstableContext, indexContext);
return Objects.hashCode(sstableContext, indexTermType, indexIdentifier);
}
@Override
public String toString()
{
return MoreObjects.toStringHelper(this)
.add("column", indexContext.getColumnName())
.add("column", indexTermType.columnName())
.add("sstable", sstableContext.sstable.descriptor)
.add("minTerm", indexContext.getValidator().getString(minTerm()))
.add("maxTerm", indexContext.getValidator().getString(maxTerm()))
.add("minTerm", indexTermType.asString(minTerm()))
.add("maxTerm", indexTermType.asString(maxTerm()))
.add("totalRows", sstableContext.sstable.getTotalRows())
.toString();
}

108
src/java/org/apache/cassandra/index/sai/disk/format/IndexDescriptor.java
View File

@ -32,10 +32,9 @@ import org.slf4j.LoggerFactory;
import org.apache.cassandra.db.ClusteringComparator;
import org.apache.cassandra.db.lifecycle.LifecycleNewTracker;
import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.index.sai.IndexContext;
import org.apache.cassandra.index.sai.StorageAttachedIndex;
import org.apache.cassandra.index.sai.IndexValidation;
import org.apache.cassandra.index.sai.SSTableContext;
import org.apache.cassandra.index.sai.StorageAttachedIndex;
import org.apache.cassandra.index.sai.disk.PerColumnIndexWriter;
import org.apache.cassandra.index.sai.disk.PerSSTableIndexWriter;
import org.apache.cassandra.index.sai.disk.PrimaryKeyMap;
@ -43,6 +42,8 @@ import org.apache.cassandra.index.sai.disk.RowMapping;
import org.apache.cassandra.index.sai.disk.SSTableIndex;
import org.apache.cassandra.index.sai.disk.io.IndexFileUtils;
import org.apache.cassandra.index.sai.disk.io.IndexOutputWriter;
import org.apache.cassandra.index.sai.utils.IndexIdentifier;
import org.apache.cassandra.index.sai.utils.IndexTermType;
import org.apache.cassandra.index.sai.utils.PrimaryKey;
import org.apache.cassandra.io.sstable.Component;
import org.apache.cassandra.io.sstable.Descriptor;
@ -123,9 +124,9 @@ public class IndexDescriptor
return version.onDiskFormat().newPrimaryKeyMapFactory(this, sstable);
}
public SSTableIndex newSSTableIndex(SSTableContext sstableContext, IndexContext indexContext)
public SSTableIndex newSSTableIndex(SSTableContext sstableContext, StorageAttachedIndex index)
{
return version.onDiskFormat().newSSTableIndex(sstableContext, indexContext);
return version.onDiskFormat().newSSTableIndex(sstableContext, index);
}
public PerSSTableIndexWriter newPerSSTableIndexWriter() throws IOException
@ -145,9 +146,9 @@ public class IndexDescriptor
return version.onDiskFormat().isPerSSTableIndexBuildComplete(this);
}
public boolean isPerColumnIndexBuildComplete(IndexContext indexContext)
public boolean isPerColumnIndexBuildComplete(IndexIdentifier indexIdentifier)
{
return version.onDiskFormat().isPerColumnIndexBuildComplete(this, indexContext);
return version.onDiskFormat().isPerColumnIndexBuildComplete(this, indexIdentifier);
}
public boolean hasComponent(IndexComponent indexComponent)
@ -155,9 +156,9 @@ public class IndexDescriptor
return fileFor(indexComponent).exists();
}
public boolean hasComponent(IndexComponent indexComponent, IndexContext indexContext)
public boolean hasComponent(IndexComponent indexComponent, IndexIdentifier indexIdentifier)
{
return fileFor(indexComponent, indexContext).exists();
return fileFor(indexComponent, indexIdentifier).exists();
}
public File fileFor(IndexComponent indexComponent)
@ -165,19 +166,19 @@ public class IndexDescriptor
return createFile(indexComponent, null);
}
public File fileFor(IndexComponent indexComponent, IndexContext indexContext)
public File fileFor(IndexComponent indexComponent, IndexIdentifier indexIdentifier)
{
return createFile(indexComponent, indexContext);
return createFile(indexComponent, indexIdentifier);
}
public boolean isIndexEmpty(IndexContext indexContext)
public boolean isIndexEmpty(IndexTermType indexTermType, IndexIdentifier indexIdentifier)
{
// The index is empty if the index build completed successfully in that both
// a GROUP_COMPLETION_MARKER companent and a COLUMN_COMPLETION_MARKER exist for
// the index and the number of per-index components is 1 indicating that only the
// COLUMN_COMPLETION_MARKER exists for the index, as this is the only file that
// will be written if the index is empty
return isPerColumnIndexBuildComplete(indexContext) && numberOfPerIndexComponents(indexContext) == 1;
return isPerColumnIndexBuildComplete(indexIdentifier) && numberOfPerIndexComponents(indexTermType, indexIdentifier) == 1;
}
public void createComponentOnDisk(IndexComponent component) throws IOException
@ -185,9 +186,9 @@ public class IndexDescriptor
Files.touch(fileFor(component).toJavaIOFile());
}
public void createComponentOnDisk(IndexComponent component, IndexContext indexContext) throws IOException
public void createComponentOnDisk(IndexComponent component, IndexIdentifier indexIdentifier) throws IOException
{
Files.touch(fileFor(component, indexContext).toJavaIOFile());
Files.touch(fileFor(component, indexIdentifier).toJavaIOFile());
}
public IndexInput openPerSSTableInput(IndexComponent indexComponent)
@ -201,9 +202,9 @@ public class IndexDescriptor
return IndexFileUtils.instance.openBlockingInput(file);
}
public IndexInput openPerIndexInput(IndexComponent indexComponent, IndexContext indexContext)
public IndexInput openPerIndexInput(IndexComponent indexComponent, IndexIdentifier indexIdentifier)
{
final File file = fileFor(indexComponent, indexContext);
final File file = fileFor(indexComponent, indexIdentifier);
if (logger.isTraceEnabled())
logger.trace(logMessage("Opening blocking index input for file {} ({})"),
file,
@ -236,19 +237,17 @@ public class IndexDescriptor
return writer;
}
public IndexOutputWriter openPerIndexOutput(IndexComponent indexComponent, IndexContext indexContext) throws IOException
public IndexOutputWriter openPerIndexOutput(IndexComponent indexComponent, IndexIdentifier indexIdentifier) throws IOException
{
return openPerIndexOutput(indexComponent, indexContext, false);
return openPerIndexOutput(indexComponent, indexIdentifier, false);
}
public IndexOutputWriter openPerIndexOutput(IndexComponent component, IndexContext indexContext, boolean append) throws IOException
public IndexOutputWriter openPerIndexOutput(IndexComponent component, IndexIdentifier indexIdentifier, boolean append) throws IOException
{
final File file = fileFor(component, indexContext);
final File file = fileFor(component, indexIdentifier);
if (logger.isTraceEnabled())
logger.trace(indexContext.logMessage("Creating sstable attached index output for component {} on file {}..."),
component,
file);
logger.trace(logMessage("Creating sstable attached index output for component {} on file {}..."), component, file);
IndexOutputWriter writer = IndexFileUtils.instance.openOutput(file);
@ -267,10 +266,7 @@ public class IndexDescriptor
final File file = fileFor(indexComponent);
if (logger.isTraceEnabled())
{
logger.trace(logMessage("Opening {} file handle for {} ({})"),
file, FBUtilities.prettyPrintMemory(file.length()));
}
logger.trace(logMessage("Opening file handle for {} ({})"), file, FBUtilities.prettyPrintMemory(file.length()));
return new FileHandle.Builder(file).mmapped(true).complete();
}
@ -280,17 +276,19 @@ public class IndexDescriptor
}
}
public FileHandle createPerIndexFileHandle(IndexComponent indexComponent, IndexContext indexContext, Throwables.DiscreteAction<?> cleanup)
public FileHandle createPerIndexFileHandle(IndexComponent indexComponent, IndexIdentifier indexIdentifier)
{
return createPerIndexFileHandle(indexComponent, indexIdentifier, null);
}
public FileHandle createPerIndexFileHandle(IndexComponent indexComponent, IndexIdentifier indexIdentifier, Throwables.DiscreteAction<?> cleanup)
{
try
{
final File file = fileFor(indexComponent, indexContext);
final File file = fileFor(indexComponent, indexIdentifier);
if (logger.isTraceEnabled())
{
logger.trace(indexContext.logMessage("Opening file handle for {} ({})"),
file, FBUtilities.prettyPrintMemory(file.length()));
}
logger.trace(logMessage("Opening file handle for {} ({})"), file, FBUtilities.prettyPrintMemory(file.length()));
return new FileHandle.Builder(file).mmapped(true).complete();
}
@ -326,13 +324,13 @@ public class IndexDescriptor
.collect(Collectors.toSet());
}
public Set<Component> getLivePerIndexComponents(IndexContext indexContext)
public Set<Component> getLivePerIndexComponents(IndexTermType indexTermType, IndexIdentifier indexIdentifier)
{
return version.onDiskFormat()
.perColumnIndexComponents(indexContext)
.perColumnIndexComponents(indexTermType)
.stream()
.filter(c -> fileFor(c, indexContext).exists())
.map(c -> version.makePerIndexComponent(c, indexContext))
.filter(c -> fileFor(c, indexIdentifier).exists())
.map(c -> version.makePerIndexComponent(c, indexIdentifier))
.collect(Collectors.toSet());
}
@ -347,36 +345,36 @@ public class IndexDescriptor
.sum();
}
public long sizeOnDiskOfPerIndexComponents(IndexContext indexContext)
public long sizeOnDiskOfPerIndexComponents(IndexTermType indexTermType, IndexIdentifier indexIdentifier)
{
return version.onDiskFormat()
.perColumnIndexComponents(indexContext)
.perColumnIndexComponents(indexTermType)
.stream()
.map(c -> fileFor(c, indexContext))
.map(c -> fileFor(c, indexIdentifier))
.filter(File::exists)
.mapToLong(File::length)
.sum();
}
@VisibleForTesting
public long sizeOnDiskOfPerIndexComponent(IndexComponent indexComponent, IndexContext indexContext)
public long sizeOnDiskOfPerIndexComponent(IndexComponent indexComponent, IndexIdentifier indexIdentifier)
{
File componentFile = fileFor(indexComponent, indexContext);
File componentFile = fileFor(indexComponent, indexIdentifier);
return componentFile.exists() ? componentFile.length() : 0;
}
@SuppressWarnings("BooleanMethodIsAlwaysInverted")
public boolean validatePerIndexComponents(IndexContext indexContext, IndexValidation validation)
public boolean validatePerIndexComponents(IndexTermType indexTermType, IndexIdentifier indexIdentifier, IndexValidation validation)
{
if (validation == IndexValidation.NONE)
return true;
logger.info(indexContext.logMessage("Validating per-column index components using mode " + validation));
logger.info(logMessage("Validating per-column index components for {} using mode {}"), indexIdentifier, validation);
boolean checksum = validation == IndexValidation.CHECKSUM;
try
{
version.onDiskFormat().validatePerColumnIndexComponents(this, indexContext, checksum);
version.onDiskFormat().validatePerColumnIndexComponents(this, indexTermType, indexIdentifier, checksum);
return true;
}
catch (UncheckedIOException e)
@ -405,9 +403,9 @@ public class IndexDescriptor
}
}
public void checksumPerIndexComponents(IndexContext indexContext)
public void checksumPerIndexComponents(IndexTermType indexTermType, IndexIdentifier indexIdentifier)
{
version.onDiskFormat().validatePerColumnIndexComponents(this, indexContext, true);
version.onDiskFormat().validatePerColumnIndexComponents(this, indexTermType, indexIdentifier, true);
}
@SuppressWarnings("BooleanMethodIsAlwaysInverted")
@ -426,12 +424,12 @@ public class IndexDescriptor
.forEach(this::deleteComponent);
}
public void deleteColumnIndex(IndexContext indexContext)
public void deleteColumnIndex(IndexTermType indexTermType, IndexIdentifier indexIdentifier)
{
version.onDiskFormat()
.perColumnIndexComponents(indexContext)
.perColumnIndexComponents(indexTermType)
.stream()
.map(c -> fileFor(c, indexContext))
.map(c -> fileFor(c, indexIdentifier))
.filter(File::exists)
.forEach(this::deleteComponent);
}
@ -467,18 +465,18 @@ public class IndexDescriptor
message);
}
private File createFile(IndexComponent component, IndexContext indexContext)
private File createFile(IndexComponent component, IndexIdentifier indexIdentifier)
{
Component customComponent = version.makePerIndexComponent(component, indexContext);
Component customComponent = version.makePerIndexComponent(component, indexIdentifier);
return sstableDescriptor.fileFor(customComponent);
}
private long numberOfPerIndexComponents(IndexContext indexContext)
private long numberOfPerIndexComponents(IndexTermType indexTermType, IndexIdentifier indexIdentifier)
{
return version.onDiskFormat()
.perColumnIndexComponents(indexContext)
.perColumnIndexComponents(indexTermType)
.stream()
.map(c -> fileFor(c, indexContext))
.map(c -> fileFor(c, indexIdentifier))
.filter(File::exists)
.count();
}

30
src/java/org/apache/cassandra/index/sai/disk/format/OnDiskFormat.java
View File

@ -23,7 +23,6 @@ import java.io.UncheckedIOException;
import java.util.Set;
import org.apache.cassandra.db.lifecycle.LifecycleNewTracker;
import org.apache.cassandra.index.sai.IndexContext;
import org.apache.cassandra.index.sai.SSTableContext;
import org.apache.cassandra.index.sai.StorageAttachedIndex;
import org.apache.cassandra.index.sai.disk.PerColumnIndexWriter;
@ -31,6 +30,8 @@ import org.apache.cassandra.index.sai.disk.PerSSTableIndexWriter;
import org.apache.cassandra.index.sai.disk.PrimaryKeyMap;
import org.apache.cassandra.index.sai.disk.RowMapping;
import org.apache.cassandra.index.sai.disk.SSTableIndex;
import org.apache.cassandra.index.sai.utils.IndexIdentifier;
import org.apache.cassandra.index.sai.utils.IndexTermType;
import org.apache.cassandra.io.sstable.format.SSTableReader;
/**
@ -42,12 +43,10 @@ import org.apache.cassandra.io.sstable.format.SSTableReader;
* <ul>
* <li>Methods taking no parameters. These methods return static information about the
* format. This can include static information about the per-sstable components</li>
* <li>Methods taking just an {@link IndexContext}. These methods return static information
* specific to the index. This can be information relating to the type of index being used</li>
* <li>Methods taking an {@link IndexDescriptor}. These methods interact with the on-disk components, or
* return objects that will interact with the on-disk components, or return information about the on-disk
* components. If they take an {@link IndexContext} as well they will be interacting with per-column index files;
* otherwise they will be interacting with per-sstable index files</li>
* components. If they take an {@link IndexTermType} and/or a {@link IndexIdentifier} as well they will be
* interacting with per-column index files; otherwise they will be interacting with per-sstable index files</li>
* <li>Methods taking an {@link IndexComponent}. These methods only interact with a single index component or
* set of index components</li>
*
@ -67,10 +66,10 @@ public interface OnDiskFormat
* Create a new {@link SSTableIndex} for an on-disk index.
*
* @param sstableContext The {@link SSTableContext} holding the per-SSTable information for the index
* @param indexContext The {@link IndexContext} holding the per-index information for the index
* @param index The {@link StorageAttachedIndex}
* @return the new {@link SSTableIndex} for the on-disk index
*/
SSTableIndex newSSTableIndex(SSTableContext sstableContext, IndexContext indexContext);
SSTableIndex newSSTableIndex(SSTableContext sstableContext, StorageAttachedIndex index);
/**
* Create a new {@link PerSSTableIndexWriter} to write the per-SSTable on-disk components of an index.
@ -107,9 +106,9 @@ public interface OnDiskFormat
* Returns true if the per-column index components have been built and are valid.
*
* @param indexDescriptor The {@link IndexDescriptor} for the SSTable SAI index
* @param indexContext The {@link IndexContext} for the index
* @param indexIdentifier The {@link IndexIdentifier} for the index
*/
boolean isPerColumnIndexBuildComplete(IndexDescriptor indexDescriptor, IndexContext indexContext);
boolean isPerColumnIndexBuildComplete(IndexDescriptor indexDescriptor, IndexIdentifier indexIdentifier);
/**
* Validate all the per-SSTable on-disk components and throw if a component is not valid
@ -125,12 +124,13 @@ public interface OnDiskFormat
* Validate all the per-column on-disk components and throw if a component is not valid
*
* @param indexDescriptor The {@link IndexDescriptor} for the SSTable SAI index
* @param indexContext The {@link IndexContext} holding the per-index information for the index
* @param indexTermType The {@link IndexTermType} of the index
* @param indexIdentifier The {@link IndexIdentifier} for the index
* @param checksum {@code true} if the checksum should be tested as part of the validation
*
* @throws UncheckedIOException if there is a problem validating any on-disk component
*/
void validatePerColumnIndexComponents(IndexDescriptor indexDescriptor, IndexContext indexContext, boolean checksum);
void validatePerColumnIndexComponents(IndexDescriptor indexDescriptor, IndexTermType indexTermType, IndexIdentifier indexIdentifier, boolean checksum);
/**
* Returns the set of {@link IndexComponent} for the per-SSTable part of an index.
@ -146,9 +146,9 @@ public interface OnDiskFormat
* This is a complete set of components that could exist on-disk. It does not imply that the
* components currently exist on-disk.
*
* @param indexContext The {@link IndexContext} for the index
* @param indexTermType the {@link IndexTermType} of the index
*/
Set<IndexComponent> perColumnIndexComponents(IndexContext indexContext);
Set<IndexComponent> perColumnIndexComponents(IndexTermType indexTermType);
/**
* Return the number of open per-SSTable files that can be open during a query.
@ -163,8 +163,6 @@ public interface OnDiskFormat
* Return the number of open per-column index files that can be open during a query.
* This is a static indication of the files that can be help open by an index
* for queries. It is not a dynamic calculation.
*
* @param indexContext The {@link IndexContext} for the index
*/
int openFilesPerColumnIndex(IndexContext indexContext);
int openFilesPerColumnIndex();
}

20
src/java/org/apache/cassandra/index/sai/disk/format/Version.java
View File

@ -22,11 +22,13 @@ import java.util.SortedSet;
import java.util.TreeSet;
import java.util.stream.Collectors;
import javax.annotation.Nullable;
import com.google.common.base.Objects;
import org.apache.cassandra.config.CassandraRelevantProperties;
import org.apache.cassandra.index.sai.IndexContext;
import org.apache.cassandra.index.sai.disk.v1.V1OnDiskFormat;
import org.apache.cassandra.index.sai.utils.IndexIdentifier;
import org.apache.cassandra.io.sstable.Component;
import org.apache.cassandra.io.sstable.Descriptor;
@ -115,9 +117,9 @@ public class Version implements Comparable<Version>
return indexComponent.type.createComponent(fileNameFormatter.format(indexComponent, null));
}
public Component makePerIndexComponent(IndexComponent indexComponent, IndexContext indexContext)
public Component makePerIndexComponent(IndexComponent indexComponent, IndexIdentifier indexIdentifier)
{
return indexComponent.type.createComponent(fileNameFormatter.format(indexComponent, indexContext));
return indexComponent.type.createComponent(fileNameFormatter.format(indexComponent, indexIdentifier));
}
public FileNameFormatter fileNameFormatter()
@ -127,24 +129,26 @@ public class Version implements Comparable<Version>
public interface FileNameFormatter
{
String format(IndexComponent indexComponent, IndexContext indexContext);
String format(IndexComponent indexComponent, IndexIdentifier indexIdentifier);
}
/**
* SAI default filename formatter. This is the current SAI on-disk filename format
*
* <p>
* Format: {@code <sstable descriptor>-SAI+<version>(+<index name>)+<component name>.db}
* Note: The index name is excluded for per-SSTable index files that are shared
* across all the per-column indexes for the SSTable.
*/
private static String defaultFileNameFormat(IndexComponent indexComponent, IndexContext indexContext, String version)
private static String defaultFileNameFormat(IndexComponent indexComponent,
@Nullable IndexIdentifier indexIdentifier,
String version)
{
StringBuilder stringBuilder = new StringBuilder();
stringBuilder.append(SAI_DESCRIPTOR);
stringBuilder.append(SAI_SEPARATOR).append(version);
if (indexContext != null)
stringBuilder.append(SAI_SEPARATOR).append(indexContext.getIndexName());
if (indexIdentifier != null)
stringBuilder.append(SAI_SEPARATOR).append(indexIdentifier.indexName);
stringBuilder.append(SAI_SEPARATOR).append(indexComponent.name);
stringBuilder.append(Descriptor.EXTENSION);

14
src/java/org/apache/cassandra/index/sai/disk/v1/ColumnCompletionMarkerUtil.java
View File

@ -20,10 +20,10 @@ package org.apache.cassandra.index.sai.disk.v1;
import java.io.IOException;
import org.apache.cassandra.index.sai.IndexContext;
import org.apache.cassandra.index.sai.disk.format.IndexComponent;
import org.apache.cassandra.index.sai.disk.format.IndexDescriptor;
import org.apache.cassandra.index.sai.disk.io.IndexOutputWriter;
import org.apache.cassandra.index.sai.utils.IndexIdentifier;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
@ -43,12 +43,12 @@ public class ColumnCompletionMarkerUtil
* Creates a column index completion marker for the specified column index, storing in it whether the index is empty.
*
* @param descriptor the index descriptor
* @param context the column index context
* @param indexIdentifier the column index identifier
* @param isEmpty whether the index is empty
*/
public static void create(IndexDescriptor descriptor, IndexContext context, boolean isEmpty) throws IOException
public static void create(IndexDescriptor descriptor, IndexIdentifier indexIdentifier, boolean isEmpty) throws IOException
{
try (IndexOutputWriter output = descriptor.openPerIndexOutput(IndexComponent.COLUMN_COMPLETION_MARKER, context))
try (IndexOutputWriter output = descriptor.openPerIndexOutput(IndexComponent.COLUMN_COMPLETION_MARKER, indexIdentifier))
{
SAICodecUtils.writeHeader(output);
output.writeByte(isEmpty ? EMPTY : NOT_EMPTY);
@ -60,12 +60,12 @@ public class ColumnCompletionMarkerUtil
* Reads the column index completion marker and returns whether if the index is empty.
*
* @param descriptor the index descriptor
* @param context the column index context
* @param indexIdentifier the column index identifier
* @return {@code true} if the index is empty, {@code false} otherwise.
*/
public static boolean isEmptyIndex(IndexDescriptor descriptor, IndexContext context) throws IOException
public static boolean isEmptyIndex(IndexDescriptor descriptor, IndexIdentifier indexIdentifier) throws IOException
{
try (IndexInput input = descriptor.openPerIndexInput(IndexComponent.COLUMN_COMPLETION_MARKER, context))
try (IndexInput input = descriptor.openPerIndexInput(IndexComponent.COLUMN_COMPLETION_MARKER, indexIdentifier))
{
SAICodecUtils.checkHeader(input); // consume header
return input.readByte() == EMPTY;

8
src/java/org/apache/cassandra/index/sai/disk/v1/IndexWriterConfig.java
View File

@ -23,9 +23,9 @@ import java.util.stream.Collectors;
import io.github.jbellis.jvector.vector.VectorSimilarityFunction;
import org.apache.cassandra.config.CassandraRelevantProperties;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.exceptions.InvalidRequestException;
import org.apache.cassandra.index.sai.disk.v1.vector.OptimizeFor;
import org.apache.cassandra.index.sai.utils.IndexTermType;
import static org.apache.cassandra.config.CassandraRelevantProperties.SAI_VECTOR_SEARCH_MAX_TOP_K;
@ -100,7 +100,7 @@ public class IndexWriterConfig
return optimizeFor;
}
public static IndexWriterConfig fromOptions(String indexName, AbstractType<?> type, Map<String, String> options)
public static IndexWriterConfig fromOptions(String indexName, IndexTermType indexTermType, Map<String, String> options)
{
int maximumNodeConnections = DEFAULT_MAXIMUM_NODE_CONNECTIONS;
int queueSize = DEFAULT_CONSTRUCTION_BEAM_WIDTH;
@ -112,8 +112,8 @@ public class IndexWriterConfig
options.get(SIMILARITY_FUNCTION) != null ||
options.get(OPTIMIZE_FOR) != null)
{
if (!type.isVector())
throw new InvalidRequestException(String.format("CQL type %s cannot have vector options", type.asCQL3Type()));
if (!indexTermType.isVector())
throw new InvalidRequestException(String.format("CQL type %s cannot have vector options", indexTermType.asCQL3Type()));
if (options.containsKey(MAXIMUM_NODE_CONNECTIONS))
{

62
src/java/org/apache/cassandra/index/sai/disk/v1/MemtableIndexWriter.java
View File

@ -27,9 +27,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.carrotsearch.hppc.LongArrayList;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.db.rows.Row;
import org.apache.cassandra.index.sai.IndexContext;
import org.apache.cassandra.index.sai.disk.PerColumnIndexWriter;
import org.apache.cassandra.index.sai.disk.RowMapping;
import org.apache.cassandra.index.sai.disk.format.IndexComponent;
@ -40,8 +38,10 @@ import org.apache.cassandra.index.sai.disk.v1.segment.SegmentMetadata;
import org.apache.cassandra.index.sai.disk.v1.trie.LiteralIndexWriter;
import org.apache.cassandra.index.sai.memory.MemtableIndex;
import org.apache.cassandra.index.sai.memory.MemtableTermsIterator;
import org.apache.cassandra.index.sai.metrics.IndexMetrics;
import org.apache.cassandra.index.sai.utils.IndexIdentifier;
import org.apache.cassandra.index.sai.utils.IndexTermType;
import org.apache.cassandra.index.sai.utils.PrimaryKey;
import org.apache.cassandra.index.sai.utils.TypeUtil;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.Pair;
import org.apache.cassandra.utils.bytecomparable.ByteComparable;
@ -55,19 +55,25 @@ public class MemtableIndexWriter implements PerColumnIndexWriter
private static final Logger logger = LoggerFactory.getLogger(MemtableIndexWriter.class);
private final IndexDescriptor indexDescriptor;
private final IndexContext indexContext;
private final IndexTermType indexTermType;
private final IndexIdentifier indexIdentifier;
private final IndexMetrics indexMetrics;
private final MemtableIndex memtable;
private final RowMapping rowMapping;
public MemtableIndexWriter(MemtableIndex memtable,
IndexDescriptor indexDescriptor,
IndexContext indexContext,
IndexTermType indexTermType,
IndexIdentifier indexIdentifier,
IndexMetrics indexMetrics,
RowMapping rowMapping)
{
assert rowMapping != null && rowMapping != RowMapping.DUMMY : "Row mapping must exist during FLUSH.";
this.indexDescriptor = indexDescriptor;
this.indexContext = indexContext;
this.indexTermType = indexTermType;
this.indexIdentifier = indexIdentifier;
this.indexMetrics = indexMetrics;
this.memtable = memtable;
this.rowMapping = rowMapping;
}
@ -83,8 +89,8 @@ public class MemtableIndexWriter implements PerColumnIndexWriter
@Override
public void abort(Throwable cause)
{
logger.warn(indexContext.logMessage("Aborting index memtable flush for {}..."), indexDescriptor.sstableDescriptor, cause);
indexDescriptor.deleteColumnIndex(indexContext);
logger.warn(indexIdentifier.logMessage("Aborting index memtable flush for {}..."), indexDescriptor.sstableDescriptor, cause);
indexDescriptor.deleteColumnIndex(indexTermType, indexIdentifier);
}
@Override
@ -98,15 +104,15 @@ public class MemtableIndexWriter implements PerColumnIndexWriter
{
if (!rowMapping.hasRows() || memtable == null || memtable.isEmpty())
{
logger.debug(indexContext.logMessage("No indexed rows to flush from SSTable {}."), indexDescriptor.sstableDescriptor);
logger.debug(indexIdentifier.logMessage("No indexed rows to flush from SSTable {}."), indexDescriptor.sstableDescriptor);
// Write a completion marker even though we haven't written anything to the index,
// so we won't try to build the index again for the SSTable
ColumnCompletionMarkerUtil.create(indexDescriptor, indexContext, true);
ColumnCompletionMarkerUtil.create(indexDescriptor, indexIdentifier, true);
return;
}
if (indexContext.isVector())
if (indexTermType.isVector())
{
flushVectorIndex(start, stopwatch);
}
@ -116,7 +122,7 @@ public class MemtableIndexWriter implements PerColumnIndexWriter
try (MemtableTermsIterator terms = new MemtableTermsIterator(memtable.getMinTerm(), memtable.getMaxTerm(), iterator))
{
long cellCount = flush(indexContext.getValidator(), terms, rowMapping.maxSSTableRowId);
long cellCount = flush(terms, rowMapping.maxSSTableRowId);
completeIndexFlush(cellCount, start, stopwatch);
}
@ -124,21 +130,21 @@ public class MemtableIndexWriter implements PerColumnIndexWriter
}
catch (Throwable t)
{
logger.error(indexContext.logMessage("Error while flushing index {}"), t.getMessage(), t);
indexContext.getIndexMetrics().memtableIndexFlushErrors.inc();
logger.error(indexIdentifier.logMessage("Error while flushing index {}"), t.getMessage(), t);
indexMetrics.memtableIndexFlushErrors.inc();
throw t;
}
}
private long flush(AbstractType<?> termComparator, MemtableTermsIterator terms, long maxSSTableRowId) throws IOException
private long flush(MemtableTermsIterator terms, long maxSSTableRowId) throws IOException
{
long numRows;
SegmentMetadata.ComponentMetadataMap indexMetas;
if (TypeUtil.isLiteral(termComparator))
if (indexTermType.isLiteral())
{
try (LiteralIndexWriter writer = new LiteralIndexWriter(indexDescriptor, indexContext))
try (LiteralIndexWriter writer = new LiteralIndexWriter(indexDescriptor, indexIdentifier))
{
indexMetas = writer.writeCompleteSegment(terms);
numRows = writer.getPostingsCount();
@ -147,10 +153,10 @@ public class MemtableIndexWriter implements PerColumnIndexWriter
else
{
NumericIndexWriter writer = new NumericIndexWriter(indexDescriptor,
indexContext,
TypeUtil.fixedSizeOf(termComparator),
indexIdentifier,
indexTermType.fixedSizeOf(),
maxSSTableRowId);
indexMetas = writer.writeCompleteSegment(BlockBalancedTreeIterator.fromTermsIterator(terms, termComparator));
indexMetas = writer.writeCompleteSegment(BlockBalancedTreeIterator.fromTermsIterator(terms, indexTermType));
numRows = writer.getValueCount();
}
@ -158,7 +164,7 @@ public class MemtableIndexWriter implements PerColumnIndexWriter
// so that the index is correctly identified as being empty (only having a completion marker)
if (numRows == 0)
{
indexDescriptor.deleteColumnIndex(indexContext);
indexDescriptor.deleteColumnIndex(indexTermType, indexIdentifier);
return 0;
}
@ -173,7 +179,7 @@ public class MemtableIndexWriter implements PerColumnIndexWriter
terms.getMaxTerm(),
indexMetas);
try (MetadataWriter writer = new MetadataWriter(indexDescriptor.openPerIndexOutput(IndexComponent.META, indexContext)))
try (MetadataWriter writer = new MetadataWriter(indexDescriptor.openPerIndexOutput(IndexComponent.META, indexIdentifier)))
{
SegmentMetadata.write(writer, Collections.singletonList(metadata));
}
@ -183,7 +189,7 @@ public class MemtableIndexWriter implements PerColumnIndexWriter
private void flushVectorIndex(long startTime, Stopwatch stopwatch) throws IOException
{
SegmentMetadata.ComponentMetadataMap metadataMap = memtable.writeDirect(indexDescriptor, indexContext, rowMapping::get);
SegmentMetadata.ComponentMetadataMap metadataMap = memtable.writeDirect(indexDescriptor, indexIdentifier, rowMapping::get);
completeIndexFlush(rowMapping.size(), startTime, stopwatch);
@ -197,7 +203,7 @@ public class MemtableIndexWriter implements PerColumnIndexWriter
ByteBufferUtil.bytes(0),
metadataMap);
try (MetadataWriter writer = new MetadataWriter(indexDescriptor.openPerIndexOutput(IndexComponent.META, indexContext)))
try (MetadataWriter writer = new MetadataWriter(indexDescriptor.openPerIndexOutput(IndexComponent.META, indexIdentifier)))
{
SegmentMetadata.write(writer, Collections.singletonList(metadata));
}
@ -206,18 +212,18 @@ public class MemtableIndexWriter implements PerColumnIndexWriter
private void completeIndexFlush(long cellCount, long startTime, Stopwatch stopwatch) throws IOException
{
// create a completion marker indicating that the index is complete and not-empty
ColumnCompletionMarkerUtil.create(indexDescriptor, indexContext, false);
ColumnCompletionMarkerUtil.create(indexDescriptor, indexIdentifier, false);
indexContext.getIndexMetrics().memtableIndexFlushCount.inc();
indexMetrics.memtableIndexFlushCount.inc();
long elapsedTime = stopwatch.elapsed(TimeUnit.MILLISECONDS);
logger.debug(indexContext.logMessage("Completed flushing {} memtable index cells to SSTable {}. Duration: {} ms. Total elapsed: {} ms"),
logger.debug(indexIdentifier.logMessage("Completed flushing {} memtable index cells to SSTable {}. Duration: {} ms. Total elapsed: {} ms"),
cellCount,
indexDescriptor.sstableDescriptor,
elapsedTime - startTime,
elapsedTime);
indexContext.getIndexMetrics().memtableFlushCellsPerSecond.update((long) (cellCount * 1000.0 / Math.max(1, elapsedTime - startTime)));
indexMetrics.memtableFlushCellsPerSecond.update((long) (cellCount * 1000.0 / Math.max(1, elapsedTime - startTime)));
}
}

6
src/java/org/apache/cassandra/index/sai/disk/v1/MetadataSource.java
View File

@ -22,9 +22,9 @@ import java.util.HashMap;
import java.util.Map;
import javax.annotation.concurrent.NotThreadSafe;
import org.apache.cassandra.index.sai.IndexContext;
import org.apache.cassandra.index.sai.disk.format.IndexComponent;
import org.apache.cassandra.index.sai.disk.format.IndexDescriptor;
import org.apache.cassandra.index.sai.utils.IndexIdentifier;
import org.apache.cassandra.index.sai.disk.io.IndexFileUtils;
import org.apache.lucene.store.ByteArrayDataInput;
import org.apache.lucene.store.ChecksumIndexInput;
@ -47,9 +47,9 @@ public class MetadataSource
return MetadataSource.load(indexDescriptor.openPerSSTableInput(IndexComponent.GROUP_META));
}
public static MetadataSource loadColumnMetadata(IndexDescriptor indexDescriptor, IndexContext indexContext) throws IOException
public static MetadataSource loadColumnMetadata(IndexDescriptor indexDescriptor, IndexIdentifier indexIdentifier) throws IOException
{
return MetadataSource.load(indexDescriptor.openPerIndexInput(IndexComponent.META, indexContext));
return MetadataSource.load(indexDescriptor.openPerIndexInput(IndexComponent.META, indexIdentifier));
}
private static MetadataSource load(IndexInput indexInput) throws IOException

15
src/java/org/apache/cassandra/index/sai/disk/v1/PerColumnIndexFiles.java
View File

@ -22,9 +22,10 @@ import java.io.Closeable;
import java.util.EnumMap;
import java.util.Map;
import org.apache.cassandra.index.sai.IndexContext;
import org.apache.cassandra.index.sai.disk.format.IndexComponent;
import org.apache.cassandra.index.sai.disk.format.IndexDescriptor;
import org.apache.cassandra.index.sai.utils.IndexIdentifier;
import org.apache.cassandra.index.sai.utils.IndexTermType;
import org.apache.cassandra.io.util.FileHandle;
import org.apache.cassandra.io.util.FileUtils;
@ -38,17 +39,17 @@ public class PerColumnIndexFiles implements Closeable
{
private final Map<IndexComponent, FileHandle> files = new EnumMap<>(IndexComponent.class);
private final IndexDescriptor indexDescriptor;
private final IndexContext indexContext;
private final IndexIdentifier indexIdentifier;
public PerColumnIndexFiles(IndexDescriptor indexDescriptor, IndexContext indexContext)
public PerColumnIndexFiles(IndexDescriptor indexDescriptor, IndexTermType indexTermType, IndexIdentifier indexIdentifier)
{
this.indexDescriptor = indexDescriptor;
this.indexContext = indexContext;
for (IndexComponent component : indexDescriptor.version.onDiskFormat().perColumnIndexComponents(indexContext))
this.indexIdentifier = indexIdentifier;
for (IndexComponent component : indexDescriptor.version.onDiskFormat().perColumnIndexComponents(indexTermType))
{
if (component == IndexComponent.META || component == IndexComponent.COLUMN_COMPLETION_MARKER)
continue;
files.put(component, indexDescriptor.createPerIndexFileHandle(component, indexContext, this::close));
files.put(component, indexDescriptor.createPerIndexFileHandle(component, indexIdentifier, this::close));
}
}
@ -76,7 +77,7 @@ public class PerColumnIndexFiles implements Closeable
{
FileHandle file = files.get(indexComponent);
if (file == null)
throw new IllegalArgumentException(String.format(indexContext.logMessage("Component %s not found for SSTable %s"),
throw new IllegalArgumentException(String.format(indexIdentifier.logMessage("Component %s not found for SSTable %s"),
indexComponent, indexDescriptor.sstableDescriptor));
return file.sharedCopy();

98
src/java/org/apache/cassandra/index/sai/disk/v1/SSTableIndexWriter.java
View File

@ -30,9 +30,8 @@ import com.google.common.base.Stopwatch;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.db.rows.Row;
import org.apache.cassandra.index.sai.IndexContext;
import org.apache.cassandra.index.sai.StorageAttachedIndex;
import org.apache.cassandra.index.sai.analyzer.AbstractAnalyzer;
import org.apache.cassandra.index.sai.disk.PerColumnIndexWriter;
import org.apache.cassandra.index.sai.disk.format.IndexComponent;
@ -41,7 +40,6 @@ import org.apache.cassandra.index.sai.disk.v1.segment.SegmentBuilder;
import org.apache.cassandra.index.sai.disk.v1.segment.SegmentMetadata;
import org.apache.cassandra.index.sai.utils.NamedMemoryLimiter;
import org.apache.cassandra.index.sai.utils.PrimaryKey;
import org.apache.cassandra.index.sai.utils.TypeUtil;
import org.apache.cassandra.utils.Clock;
import org.apache.cassandra.utils.FBUtilities;
@ -54,7 +52,7 @@ public class SSTableIndexWriter implements PerColumnIndexWriter
private static final Logger logger = LoggerFactory.getLogger(SSTableIndexWriter.class);
private final IndexDescriptor indexDescriptor;
private final IndexContext indexContext;
private final StorageAttachedIndex index;
private final long nowInSec = FBUtilities.nowInSeconds();
private final AbstractAnalyzer analyzer;
private final NamedMemoryLimiter limiter;
@ -64,11 +62,14 @@ public class SSTableIndexWriter implements PerColumnIndexWriter
private boolean aborted = false;
private SegmentBuilder currentBuilder;
public SSTableIndexWriter(IndexDescriptor indexDescriptor, IndexContext indexContext, NamedMemoryLimiter limiter, BooleanSupplier isIndexValid)
public SSTableIndexWriter(IndexDescriptor indexDescriptor,
StorageAttachedIndex index,
NamedMemoryLimiter limiter,
BooleanSupplier isIndexValid)
{
this.indexDescriptor = indexDescriptor;
this.indexContext = indexContext;
this.analyzer = indexContext.getAnalyzerFactory().create();
this.index = index;
this.analyzer = index.hasAnalyzer() ? index.analyzer() : null;
this.limiter = limiter;
this.isIndexValid = isIndexValid;
}
@ -79,23 +80,23 @@ public class SSTableIndexWriter implements PerColumnIndexWriter
if (maybeAbort())
return;
if (indexContext.isNonFrozenCollection())
if (index.termType().isNonFrozenCollection())
{
Iterator<ByteBuffer> valueIterator = indexContext.getValuesOf(row, nowInSec);
Iterator<ByteBuffer> valueIterator = index.termType().valuesOf(row, nowInSec);
if (valueIterator != null)
{
while (valueIterator.hasNext())
{
ByteBuffer value = valueIterator.next();
addTerm(TypeUtil.asIndexBytes(value.duplicate(), indexContext.getValidator()), key, sstableRowId, indexContext.getValidator());
addTerm(index.termType().asIndexBytes(value.duplicate()), key, sstableRowId);
}
}
}
else
{
ByteBuffer value = indexContext.getValueOf(key.partitionKey(), row, nowInSec);
ByteBuffer value = index.termType().valueOf(key.partitionKey(), row, nowInSec);
if (value != null)
addTerm(TypeUtil.asIndexBytes(value.duplicate(), indexContext.getValidator()), key, sstableRowId, indexContext.getValidator());
addTerm(index.termType().asIndexBytes(value.duplicate()), key, sstableRowId);
}
}
@ -109,7 +110,7 @@ public class SSTableIndexWriter implements PerColumnIndexWriter
long elapsed;
boolean emptySegment = currentBuilder == null || currentBuilder.isEmpty();
logger.debug(indexContext.logMessage("Completing index flush with {}buffered data..."), emptySegment ? "no " : "");
logger.debug(index.identifier().logMessage("Completing index flush with {}buffered data..."), emptySegment ? "no " : "");
try
{
@ -118,7 +119,7 @@ public class SSTableIndexWriter implements PerColumnIndexWriter
{
flushSegment();
elapsed = stopwatch.elapsed(TimeUnit.MILLISECONDS);
logger.debug(indexContext.logMessage("Completed flush of final segment for SSTable {}. Duration: {} ms. Total elapsed: {} ms"),
logger.debug(index.identifier().logMessage("Completed flush of final segment for SSTable {}. Duration: {} ms. Total elapsed: {} ms"),
indexDescriptor.sstableDescriptor,
elapsed - start,
elapsed);
@ -128,24 +129,21 @@ public class SSTableIndexWriter implements PerColumnIndexWriter
if (currentBuilder != null)
{
long bytesAllocated = currentBuilder.totalBytesAllocated();
long globalBytesUsed = currentBuilder.release(indexContext);
logger.debug(indexContext.logMessage("Flushing final segment for SSTable {} released {}. Global segment memory usage now at {}."),
long globalBytesUsed = currentBuilder.release(index.identifier());
logger.debug(index.identifier().logMessage("Flushing final segment for SSTable {} released {}. Global segment memory usage now at {}."),
indexDescriptor.sstableDescriptor, FBUtilities.prettyPrintMemory(bytesAllocated), FBUtilities.prettyPrintMemory(globalBytesUsed));
}
writeSegmentsMetadata();
// write column index completion marker, indicating whether the index is empty
ColumnCompletionMarkerUtil.create(indexDescriptor, indexContext, segments.isEmpty());
ColumnCompletionMarkerUtil.create(indexDescriptor, index.identifier(), segments.isEmpty());
}
finally
{
if (indexContext.getIndexMetrics() != null)
{
indexContext.getIndexMetrics().segmentsPerCompaction.update(segments.size());
segments.clear();
indexContext.getIndexMetrics().compactionCount.inc();
}
index.indexMetrics().segmentsPerCompaction.update(segments.size());
segments.clear();
index.indexMetrics().compactionCount.inc();
}
}
@ -154,7 +152,7 @@ public class SSTableIndexWriter implements PerColumnIndexWriter
{
aborted = true;
logger.warn(indexContext.logMessage("Aborting SSTable index flush for {}..."), indexDescriptor.sstableDescriptor, cause);
logger.warn(index.identifier().logMessage("Aborting SSTable index flush for {}..."), indexDescriptor.sstableDescriptor, cause);
// It's possible for the current builder to be unassigned after we flush a final segment.
if (currentBuilder != null)
@ -162,12 +160,12 @@ public class SSTableIndexWriter implements PerColumnIndexWriter
// If an exception is thrown out of any writer operation prior to successful segment
// flush, we will end up here, and we need to free up builder memory tracked by the limiter:
long allocated = currentBuilder.totalBytesAllocated();
long globalBytesUsed = currentBuilder.release(indexContext);
logger.debug(indexContext.logMessage("Aborting index writer for SSTable {} released {}. Global segment memory usage now at {}."),
long globalBytesUsed = currentBuilder.release(index.identifier());
logger.debug(index.identifier().logMessage("Aborting index writer for SSTable {} released {}. Global segment memory usage now at {}."),
indexDescriptor.sstableDescriptor, FBUtilities.prettyPrintMemory(allocated), FBUtilities.prettyPrintMemory(globalBytesUsed));
}
indexDescriptor.deleteColumnIndex(indexContext);
indexDescriptor.deleteColumnIndex(index.termType(), index.identifier());
}
/**
@ -183,13 +181,13 @@ public class SSTableIndexWriter implements PerColumnIndexWriter
if (isIndexValid.getAsBoolean())
return false;
abort(new RuntimeException(String.format("index %s is dropped", indexContext.getIndexName())));
abort(new RuntimeException(String.format("index %s is dropped", index.identifier())));
return true;
}
private void addTerm(ByteBuffer term, PrimaryKey key, long sstableRowId, AbstractType<?> type) throws IOException
private void addTerm(ByteBuffer term, PrimaryKey key, long sstableRowId) throws IOException
{
if (!indexContext.validateMaxTermSize(key.partitionKey(), term, false))
if (!index.validateMaxTermSize(key.partitionKey(), term, false))
return;
if (currentBuilder == null)
@ -204,7 +202,7 @@ public class SSTableIndexWriter implements PerColumnIndexWriter
if (term.remaining() == 0) return;
if (!TypeUtil.isLiteral(type))
if (analyzer == null || !index.termType().isLiteral())
{
limiter.increment(currentBuilder.add(term, key, sstableRowId));
}
@ -233,8 +231,8 @@ public class SSTableIndexWriter implements PerColumnIndexWriter
if (reachMemoryLimit)
{
logger.debug(indexContext.logMessage("Global limit of {} and minimum flush size of {} exceeded. " +
"Current builder usage is {} for {} cells. Global Usage is {}. Flushing..."),
logger.debug(index.identifier().logMessage("Global limit of {} and minimum flush size of {} exceeded. " +
"Current builder usage is {} for {} cells. Global Usage is {}. Flushing..."),
FBUtilities.prettyPrintMemory(limiter.limitBytes()),
FBUtilities.prettyPrintMemory(currentBuilder.getMinimumFlushBytes()),
FBUtilities.prettyPrintMemory(currentBuilder.totalBytesAllocated()),
@ -253,7 +251,7 @@ public class SSTableIndexWriter implements PerColumnIndexWriter
{
long bytesAllocated = currentBuilder.totalBytesAllocated();
SegmentMetadata segmentMetadata = currentBuilder.flush(indexDescriptor, indexContext);
SegmentMetadata segmentMetadata = currentBuilder.flush(indexDescriptor, index.identifier());
long flushMillis = Math.max(1, TimeUnit.NANOSECONDS.toMillis(Clock.Global.nanoTime() - start));
@ -262,14 +260,12 @@ public class SSTableIndexWriter implements PerColumnIndexWriter
segments.add(segmentMetadata);
double rowCount = segmentMetadata.numRows;
if (indexContext.getIndexMetrics() != null)
indexContext.getIndexMetrics().compactionSegmentCellsPerSecond.update((long)(rowCount / flushMillis * 1000.0));
index.indexMetrics().compactionSegmentCellsPerSecond.update((long)(rowCount / flushMillis * 1000.0));
double segmentBytes = segmentMetadata.componentMetadatas.indexSize();
if (indexContext.getIndexMetrics() != null)
indexContext.getIndexMetrics().compactionSegmentBytesPerSecond.update((long)(segmentBytes / flushMillis * 1000.0));
index.indexMetrics().compactionSegmentBytesPerSecond.update((long)(segmentBytes / flushMillis * 1000.0));
logger.debug(indexContext.logMessage("Flushed segment with {} cells for a total of {} in {} ms."),
logger.debug(index.identifier().logMessage("Flushed segment with {} cells for a total of {} in {} ms."),
(long) rowCount, FBUtilities.prettyPrintMemory((long) segmentBytes), flushMillis);
}
@ -277,17 +273,17 @@ public class SSTableIndexWriter implements PerColumnIndexWriter
// flush. Note that any failure that occurs before this (even in term addition) will
// actuate this column writer's abort logic from the parent SSTable-level writer, and
// that abort logic will release the current builder's memory against the limiter.
long globalBytesUsed = currentBuilder.release(indexContext);
long globalBytesUsed = currentBuilder.release(index.identifier());
currentBuilder = null;
logger.debug(indexContext.logMessage("Flushing index segment for SSTable {} released {}. Global segment memory usage now at {}."),
logger.debug(index.identifier().logMessage("Flushing index segment for SSTable {} released {}. Global segment memory usage now at {}."),
indexDescriptor.sstableDescriptor, FBUtilities.prettyPrintMemory(bytesAllocated), FBUtilities.prettyPrintMemory(globalBytesUsed));
}
catch (Throwable t)
{
logger.error(indexContext.logMessage("Failed to build index for SSTable {}."), indexDescriptor.sstableDescriptor, t);
indexDescriptor.deleteColumnIndex(indexContext);
indexContext.getIndexMetrics().segmentFlushErrors.inc();
logger.error(index.identifier().logMessage("Failed to build index for SSTable {}."), indexDescriptor.sstableDescriptor, t);
indexDescriptor.deleteColumnIndex(index.termType(), index.identifier());
index.indexMetrics().segmentFlushErrors.inc();
throw t;
}
}
@ -297,7 +293,7 @@ public class SSTableIndexWriter implements PerColumnIndexWriter
if (segments.isEmpty())
return;
try (MetadataWriter writer = new MetadataWriter(indexDescriptor.openPerIndexOutput(IndexComponent.META, indexContext)))
try (MetadataWriter writer = new MetadataWriter(indexDescriptor.openPerIndexOutput(IndexComponent.META, index.identifier())))
{
SegmentMetadata.write(writer, segments);
}
@ -312,15 +308,15 @@ public class SSTableIndexWriter implements PerColumnIndexWriter
{
SegmentBuilder builder;
if (indexContext.isVector())
builder = new SegmentBuilder.VectorSegmentBuilder(indexContext.getValidator(), limiter, indexContext.getIndexWriterConfig());
else if (indexContext.isLiteral())
builder = new SegmentBuilder.RAMStringSegmentBuilder(indexContext.getValidator(), limiter);
if (index.termType().isVector())
builder = new SegmentBuilder.VectorSegmentBuilder(index.termType(), limiter, index.indexWriterConfig());
else if (index.termType().isLiteral())
builder = new SegmentBuilder.RAMStringSegmentBuilder(index.termType(), limiter);
else
builder = new SegmentBuilder.BlockBalancedTreeSegmentBuilder(indexContext.getValidator(), limiter);
builder = new SegmentBuilder.BlockBalancedTreeSegmentBuilder(index.termType(), limiter);
long globalBytesUsed = limiter.increment(builder.totalBytesAllocated());
logger.debug(indexContext.logMessage("Created new segment builder while flushing SSTable {}. Global segment memory usage now at {}."),
logger.debug(index.identifier().logMessage("Created new segment builder while flushing SSTable {}. Global segment memory usage now at {}."),
indexDescriptor.sstableDescriptor,
FBUtilities.prettyPrintMemory(globalBytesUsed));

44
src/java/org/apache/cassandra/index/sai/disk/v1/V1OnDiskFormat.java
View File

@ -24,8 +24,6 @@ import java.util.EnumSet;
import java.util.Set;
import com.google.common.annotations.VisibleForTesting;
import org.apache.cassandra.utils.Throwables;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -33,7 +31,6 @@ import com.codahale.metrics.Gauge;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.compaction.OperationType;
import org.apache.cassandra.db.lifecycle.LifecycleNewTracker;
import org.apache.cassandra.index.sai.IndexContext;
import org.apache.cassandra.index.sai.SSTableContext;
import org.apache.cassandra.index.sai.StorageAttachedIndex;
import org.apache.cassandra.index.sai.disk.PerColumnIndexWriter;
@ -46,10 +43,13 @@ import org.apache.cassandra.index.sai.disk.format.IndexDescriptor;
import org.apache.cassandra.index.sai.disk.format.OnDiskFormat;
import org.apache.cassandra.index.sai.disk.v1.segment.SegmentBuilder;
import org.apache.cassandra.index.sai.metrics.AbstractMetrics;
import org.apache.cassandra.index.sai.utils.IndexIdentifier;
import org.apache.cassandra.index.sai.utils.IndexTermType;
import org.apache.cassandra.index.sai.utils.NamedMemoryLimiter;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.metrics.CassandraMetricsRegistry;
import org.apache.cassandra.metrics.DefaultNameFactory;
import org.apache.cassandra.utils.Throwables;
import org.apache.lucene.store.IndexInput;
import static org.apache.cassandra.utils.FBUtilities.prettyPrintMemory;
@ -136,9 +136,9 @@ public class V1OnDiskFormat implements OnDiskFormat
}
@Override
public SSTableIndex newSSTableIndex(SSTableContext sstableContext, IndexContext indexContext)
public SSTableIndex newSSTableIndex(SSTableContext sstableContext, StorageAttachedIndex index)
{
return new V1SSTableIndex(sstableContext, indexContext);
return new V1SSTableIndex(sstableContext, index);
}
@Override
@ -157,15 +157,17 @@ public class V1OnDiskFormat implements OnDiskFormat
if (tracker.opType() != OperationType.FLUSH || !index.isInitBuildStarted())
{
NamedMemoryLimiter limiter = SEGMENT_BUILD_MEMORY_LIMITER;
logger.info(index.getIndexContext().logMessage("Starting a compaction index build. Global segment memory usage: {}"),
logger.info(index.identifier().logMessage("Starting a compaction index build. Global segment memory usage: {}"),
prettyPrintMemory(limiter.currentBytesUsed()));
return new SSTableIndexWriter(indexDescriptor, index.getIndexContext(), limiter, index.isIndexValid());
return new SSTableIndexWriter(indexDescriptor, index, limiter, index.isIndexValid());
}
return new MemtableIndexWriter(index.getIndexContext().getMemtableIndexManager().getPendingMemtableIndex(tracker),
return new MemtableIndexWriter(index.memtableIndexManager().getPendingMemtableIndex(tracker),
indexDescriptor,
index.getIndexContext(),
index.termType(),
index.identifier(),
index.indexMetrics(),
rowMapping);
}
@ -176,10 +178,10 @@ public class V1OnDiskFormat implements OnDiskFormat
}
@Override
public boolean isPerColumnIndexBuildComplete(IndexDescriptor indexDescriptor, IndexContext indexContext)
public boolean isPerColumnIndexBuildComplete(IndexDescriptor indexDescriptor, IndexIdentifier indexIdentifier)
{
return indexDescriptor.hasComponent(IndexComponent.GROUP_COMPLETION_MARKER) &&
indexDescriptor.hasComponent(IndexComponent.COLUMN_COMPLETION_MARKER, indexContext);
indexDescriptor.hasComponent(IndexComponent.COLUMN_COMPLETION_MARKER, indexIdentifier);
}
@Override
@ -195,19 +197,19 @@ public class V1OnDiskFormat implements OnDiskFormat
}
@Override
public void validatePerColumnIndexComponents(IndexDescriptor indexDescriptor, IndexContext indexContext, boolean checksum)
public void validatePerColumnIndexComponents(IndexDescriptor indexDescriptor, IndexTermType indexTermType, IndexIdentifier indexIdentifier, boolean checksum)
{
// determine if the index is empty, which would be encoded in the column completion marker
boolean isEmptyIndex = false;
if (indexDescriptor.hasComponent(IndexComponent.COLUMN_COMPLETION_MARKER, indexContext))
if (indexDescriptor.hasComponent(IndexComponent.COLUMN_COMPLETION_MARKER, indexIdentifier))
{
// first validate the file...
validateIndexComponent(indexDescriptor, indexContext, IndexComponent.COLUMN_COMPLETION_MARKER, checksum);
validateIndexComponent(indexDescriptor, indexIdentifier, IndexComponent.COLUMN_COMPLETION_MARKER, checksum);
// ...then read to check if the index is empty
try
{
isEmptyIndex = ColumnCompletionMarkerUtil.isEmptyIndex(indexDescriptor, indexContext);
isEmptyIndex = ColumnCompletionMarkerUtil.isEmptyIndex(indexDescriptor, indexIdentifier);
}
catch (IOException e)
{
@ -215,17 +217,17 @@ public class V1OnDiskFormat implements OnDiskFormat
}
}
for (IndexComponent indexComponent : perColumnIndexComponents(indexContext))
for (IndexComponent indexComponent : perColumnIndexComponents(indexTermType))
{
if (!isEmptyIndex && isNotBuildCompletionMarker(indexComponent))
{
validateIndexComponent(indexDescriptor, indexContext, indexComponent, checksum);
validateIndexComponent(indexDescriptor, indexIdentifier, indexComponent, checksum);
}
}
}
private static void validateIndexComponent(IndexDescriptor indexDescriptor,
IndexContext indexContext,
IndexIdentifier indexContext,
IndexComponent indexComponent,
boolean checksum)
{
@ -264,9 +266,9 @@ public class V1OnDiskFormat implements OnDiskFormat
}
@Override
public Set<IndexComponent> perColumnIndexComponents(IndexContext indexContext)
public Set<IndexComponent> perColumnIndexComponents(IndexTermType indexTermType)
{
return indexContext.isVector() ? VECTOR_COMPONENTS : indexContext.isLiteral() ? LITERAL_COMPONENTS : NUMERIC_COMPONENTS;
return indexTermType.isVector() ? VECTOR_COMPONENTS : indexTermType.isLiteral() ? LITERAL_COMPONENTS : NUMERIC_COMPONENTS;
}
@Override
@ -281,7 +283,7 @@ public class V1OnDiskFormat implements OnDiskFormat
}
@Override
public int openFilesPerColumnIndex(IndexContext indexContext)
public int openFilesPerColumnIndex()
{
// For the V1 format there are always 2 open files per index - index (balanced tree or terms) + auxiliary postings
// for the balanced tree and postings for the literal terms

25
src/java/org/apache/cassandra/index/sai/disk/v1/V1SSTableIndex.java
View File

@ -30,9 +30,9 @@ import org.apache.cassandra.db.PartitionPosition;
import org.apache.cassandra.db.virtual.SimpleDataSet;
import org.apache.cassandra.dht.AbstractBounds;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.index.sai.IndexContext;
import org.apache.cassandra.index.sai.QueryContext;
import org.apache.cassandra.index.sai.SSTableContext;
import org.apache.cassandra.index.sai.StorageAttachedIndex;
import org.apache.cassandra.index.sai.disk.SSTableIndex;
import org.apache.cassandra.index.sai.disk.v1.segment.Segment;
import org.apache.cassandra.index.sai.disk.v1.segment.SegmentMetadata;
@ -40,7 +40,6 @@ import org.apache.cassandra.index.sai.iterators.KeyRangeIterator;
import org.apache.cassandra.index.sai.iterators.KeyRangeUnionIterator;
import org.apache.cassandra.index.sai.plan.Expression;
import org.apache.cassandra.index.sai.utils.PrimaryKey;
import org.apache.cassandra.index.sai.utils.TypeUtil;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.util.FileUtils;
import org.apache.cassandra.utils.Throwables;
@ -72,23 +71,23 @@ public class V1SSTableIndex extends SSTableIndex
private PerColumnIndexFiles indexFiles;
public V1SSTableIndex(SSTableContext sstableContext, IndexContext indexContext)
public V1SSTableIndex(SSTableContext sstableContext, StorageAttachedIndex index)
{
super(sstableContext, indexContext);
super(sstableContext, index);
try
{
this.indexFiles = new PerColumnIndexFiles(sstableContext.indexDescriptor, indexContext);
this.indexFiles = new PerColumnIndexFiles(sstableContext.indexDescriptor, indexTermType, indexIdentifier);
ImmutableList.Builder<Segment> segmentsBuilder = ImmutableList.builder();
final MetadataSource source = MetadataSource.loadColumnMetadata(sstableContext.indexDescriptor, indexContext);
final MetadataSource source = MetadataSource.loadColumnMetadata(sstableContext.indexDescriptor, indexIdentifier);
metadatas = SegmentMetadata.load(source, sstableContext.indexDescriptor.primaryKeyFactory);
for (SegmentMetadata metadata : metadatas)
{
segmentsBuilder.add(new Segment(indexContext, sstableContext, indexFiles, metadata));
segmentsBuilder.add(new Segment(index, sstableContext, indexFiles, metadata));
}
segments = segmentsBuilder.build();
@ -99,8 +98,8 @@ public class V1SSTableIndex extends SSTableIndex
this.bounds = AbstractBounds.bounds(minKey, true, maxKey, true);
this.minTerm = metadatas.stream().map(m -> m.minTerm).min(TypeUtil.comparator(indexContext.getValidator())).orElse(null);
this.maxTerm = metadatas.stream().map(m -> m.maxTerm).max(TypeUtil.comparator(indexContext.getValidator())).orElse(null);
this.minTerm = metadatas.stream().map(m -> m.minTerm).min(indexTermType.comparator()).orElse(null);
this.maxTerm = metadatas.stream().map(m -> m.maxTerm).max(indexTermType.comparator()).orElse(null);
this.numRows = metadatas.stream().mapToLong(m -> m.numRows).sum();
@ -193,16 +192,16 @@ public class V1SSTableIndex extends SSTableIndex
for (SegmentMetadata metadata : metadatas)
{
dataset.row(sstable.metadata().keyspace, indexContext.getIndexName(), sstable.getFilename(), metadata.rowIdOffset)
dataset.row(sstable.metadata().keyspace, indexIdentifier.indexName, sstable.getFilename(), metadata.rowIdOffset)
.column(TABLE_NAME, sstable.descriptor.cfname)
.column(COLUMN_NAME, indexContext.getColumnName())
.column(COLUMN_NAME, indexTermType.columnName())
.column(CELL_COUNT, metadata.numRows)
.column(MIN_SSTABLE_ROW_ID, metadata.minSSTableRowId)
.column(MAX_SSTABLE_ROW_ID, metadata.maxSSTableRowId)
.column(START_TOKEN, tokenFactory.toString(metadata.minKey.token()))
.column(END_TOKEN, tokenFactory.toString(metadata.maxKey.token()))
.column(MIN_TERM, indexContext.getValidator().getSerializer().deserialize(metadata.minTerm).toString())
.column(MAX_TERM, indexContext.getValidator().getSerializer().deserialize(metadata.maxTerm).toString())
.column(MIN_TERM, indexTermType.indexType().getSerializer().deserialize(metadata.minTerm).toString())
.column(MAX_TERM, indexTermType.indexType().getSerializer().deserialize(metadata.maxTerm).toString())
.column(COMPONENT_METADATA, metadata.componentMetadatas.asMap());
}
}

7
src/java/org/apache/cassandra/index/sai/disk/v1/bbtree/BlockBalancedTreeIterator.java
View File

@ -21,10 +21,9 @@ package org.apache.cassandra.index.sai.disk.v1.bbtree;
import java.util.Iterator;
import java.util.Map;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.index.sai.postings.PostingList;
import org.apache.cassandra.index.sai.utils.IndexTermType;
import org.apache.cassandra.index.sai.utils.TermsIterator;
import org.apache.cassandra.index.sai.utils.TypeUtil;
import org.apache.cassandra.utils.Pair;
import org.apache.cassandra.utils.bytecomparable.ByteComparable;
import org.apache.cassandra.utils.bytecomparable.ByteSourceInverse;
@ -36,11 +35,11 @@ import org.apache.lucene.util.packed.PackedLongValues;
*/
public interface BlockBalancedTreeIterator extends Iterator<Pair<byte[], PostingList>>
{
static BlockBalancedTreeIterator fromTermsIterator(final TermsIterator termsIterator, AbstractType<?> termsComparator)
static BlockBalancedTreeIterator fromTermsIterator(final TermsIterator termsIterator, IndexTermType indexTermType)
{
return new BlockBalancedTreeIterator()
{
final byte[] scratch = new byte[TypeUtil.fixedSizeOf(termsComparator)];
final byte[] scratch = new byte[indexTermType.fixedSizeOf()];
@Override
public boolean hasNext()

9
src/java/org/apache/cassandra/index/sai/disk/v1/bbtree/BlockBalancedTreePostingsWriter.java
View File

@ -29,7 +29,6 @@ import java.util.PriorityQueue;
import java.util.TreeMap;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import javax.annotation.concurrent.NotThreadSafe;
import com.google.common.base.Stopwatch;
@ -41,7 +40,7 @@ import org.slf4j.LoggerFactory;
import org.agrona.collections.IntArrayList;
import org.apache.cassandra.config.CassandraRelevantProperties;
import org.apache.cassandra.index.sai.IndexContext;
import org.apache.cassandra.index.sai.utils.IndexIdentifier;
import org.apache.cassandra.index.sai.disk.io.IndexOutputWriter;
import org.apache.cassandra.index.sai.disk.v1.postings.MergePostingList;
import org.apache.cassandra.index.sai.disk.v1.postings.PackedLongsPostingList;
@ -127,7 +126,7 @@ public class BlockBalancedTreePostingsWriter implements BlockBalancedTreeWalker.
* After writing out the postings, it writes a map of node ID -> postings file pointer for all
* nodes with an attached postings list. It then returns the file pointer to this map.
*/
public long finish(IndexOutputWriter out, List<PackedLongValues> leafPostings, IndexContext indexContext) throws IOException
public long finish(IndexOutputWriter out, List<PackedLongValues> leafPostings, IndexIdentifier indexIdentifier) throws IOException
{
checkState(leafPostings.size() == leafOffsetToNodeID.size(),
"Expected equal number of postings lists (%s) and leaf offsets (%s).",
@ -150,7 +149,7 @@ public class BlockBalancedTreePostingsWriter implements BlockBalancedTreeWalker.
Collection<Integer> leafNodeIDs = leafOffsetToNodeID.values();
logger.debug(indexContext.logMessage("Writing posting lists for {} internal and {} leaf balanced tree nodes. Leaf postings memory usage: {}."),
logger.debug(indexIdentifier.logMessage("Writing posting lists for {} internal and {} leaf balanced tree nodes. Leaf postings memory usage: {}."),
internalNodeIDs.size(), leafNodeIDs.size(), FBUtilities.prettyPrintMemory(postingsRamBytesUsed));
long startFP = out.getFilePointer();
@ -185,7 +184,7 @@ public class BlockBalancedTreePostingsWriter implements BlockBalancedTreeWalker.
postingLists.clear();
}
flushTime.stop();
logger.debug(indexContext.logMessage("Flushed {} of posting lists for balanced tree nodes in {} ms."),
logger.debug(indexIdentifier.logMessage("Flushed {} of posting lists for balanced tree nodes in {} ms."),
FBUtilities.prettyPrintMemory(out.getFilePointer() - startFP),
flushTime.elapsed(TimeUnit.MILLISECONDS));

23
src/java/org/apache/cassandra/index/sai/disk/v1/bbtree/BlockBalancedTreeQueries.java
View File

@ -19,9 +19,8 @@ package org.apache.cassandra.index.sai.disk.v1.bbtree;
import java.nio.ByteBuffer;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.index.sai.plan.Expression;
import org.apache.cassandra.index.sai.utils.TypeUtil;
import org.apache.cassandra.index.sai.utils.IndexTermType;
import org.apache.cassandra.utils.ByteArrayUtil;
import org.apache.lucene.index.PointValues.Relation;
@ -44,33 +43,33 @@ public class BlockBalancedTreeQueries
public static BlockBalancedTreeReader.IntersectVisitor balancedTreeQueryFrom(Expression expression, int bytesPerValue)
{
if (expression.lower == null && expression.upper == null)
if (expression.lower() == null && expression.upper() == null)
{
return MATCH_ALL;
}
Bound lower = null ;
if (expression.lower != null)
if (expression.lower() != null)
{
final byte[] lowerBound = toComparableBytes(bytesPerValue, expression.lower.value.encoded, expression.validator);
lower = new Bound(lowerBound, !expression.lower.inclusive);
final byte[] lowerBound = toComparableBytes(bytesPerValue, expression.lower().value.encoded, expression.getIndexTermType());
lower = new Bound(lowerBound, !expression.lower().inclusive);
}
Bound upper = null;
if (expression.upper != null)
if (expression.upper() != null)
{
final byte[] upperBound = toComparableBytes(bytesPerValue, expression.upper.value.encoded, expression.validator);
upper = new Bound(upperBound, !expression.upper.inclusive);
final byte[] upperBound = toComparableBytes(bytesPerValue, expression.upper().value.encoded, expression.getIndexTermType());
upper = new Bound(upperBound, !expression.upper().inclusive);
}
return new RangeQueryVisitor(lower, upper);
}
private static byte[] toComparableBytes(int bytesPerDim, ByteBuffer value, AbstractType<?> type)
private static byte[] toComparableBytes(int bytesPerDim, ByteBuffer value, IndexTermType indexTermType)
{
byte[] buffer = new byte[TypeUtil.fixedSizeOf(type)];
byte[] buffer = new byte[indexTermType.fixedSizeOf()];
assert buffer.length == bytesPerDim;
TypeUtil.toComparableBytes(value, type, buffer);
indexTermType.toComparableBytes(value, buffer);
return buffer;
}

14
src/java/org/apache/cassandra/index/sai/disk/v1/bbtree/BlockBalancedTreeReader.java
View File

@ -29,8 +29,8 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.exceptions.QueryCancelledException;
import org.apache.cassandra.index.sai.IndexContext;
import org.apache.cassandra.index.sai.QueryContext;
import org.apache.cassandra.index.sai.utils.IndexIdentifier;
import org.apache.cassandra.index.sai.disk.io.IndexFileUtils;
import org.apache.cassandra.index.sai.disk.io.SeekingRandomAccessInput;
import org.apache.cassandra.index.sai.disk.v1.postings.FilteringPostingList;
@ -61,21 +61,21 @@ public class BlockBalancedTreeReader extends BlockBalancedTreeWalker implements
private static final Comparator<PeekablePostingList> COMPARATOR = Comparator.comparingLong(PeekablePostingList::peek);
private final IndexContext indexContext;
private final IndexIdentifier indexIdentifier;
private final FileHandle postingsFile;
private final BlockBalancedTreePostingsIndex postingsIndex;
private final int leafOrderMapBitsRequired;
/**
* Performs a blocking read.
*/
public BlockBalancedTreeReader(IndexContext indexContext,
public BlockBalancedTreeReader(IndexIdentifier indexIdentifier,
FileHandle treeIndexFile,
long treeIndexRoot,
FileHandle postingsFile,
long treePostingsRoot) throws IOException
{
super(treeIndexFile, treeIndexRoot);
this.indexContext = indexContext;
this.indexIdentifier = indexIdentifier;
this.postingsFile = postingsFile;
this.postingsIndex = new BlockBalancedTreePostingsIndex(postingsFile, treePostingsRoot);
leafOrderMapBitsRequired = DirectWriter.unsignedBitsRequired(maxValuesInLeafNode - 1);
@ -161,7 +161,7 @@ public class BlockBalancedTreeReader extends BlockBalancedTreeWalker implements
catch (Throwable t)
{
if (!(t instanceof QueryCancelledException))
logger.error(indexContext.logMessage("Balanced tree intersection failed on {}"), treeIndexFile.path(), t);
logger.error(indexIdentifier.logMessage("Balanced tree intersection failed on {}"), treeIndexFile.path(), t);
closeOnException();
throw Throwables.cleaned(t);
@ -196,7 +196,7 @@ public class BlockBalancedTreeReader extends BlockBalancedTreeWalker implements
else
{
if (logger.isTraceEnabled())
logger.trace(indexContext.logMessage("[{}] Intersection completed in {} microseconds. {} leaf and internal posting lists hit."),
logger.trace(indexIdentifier.logMessage("[{}] Intersection completed in {} microseconds. {} leaf and internal posting lists hit."),
treeIndexFile.path(), elapsedMicros, postingLists.size());
return MergePostingList.merge(postingLists, () -> FileUtils.close(postingsInput, postingsSummaryInput));
}
@ -215,7 +215,7 @@ public class BlockBalancedTreeReader extends BlockBalancedTreeWalker implements
}
if (state.atLeafNode())
throw new CorruptIndexException(indexContext.logMessage(String.format("Leaf node %s does not have balanced tree postings.", state.nodeID)), "");
throw new CorruptIndexException(indexIdentifier.logMessage(String.format("Leaf node %s does not have balanced tree postings.", state.nodeID)), "");
// Recurse on left subtree:
state.pushLeft();

30
src/java/org/apache/cassandra/index/sai/disk/v1/bbtree/NumericIndexWriter.java
View File

@ -26,9 +26,9 @@ import java.util.Map;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.MoreObjects;
import org.apache.cassandra.index.sai.IndexContext;
import org.apache.cassandra.index.sai.disk.format.IndexComponent;
import org.apache.cassandra.index.sai.disk.format.IndexDescriptor;
import org.apache.cassandra.index.sai.utils.IndexIdentifier;
import org.apache.cassandra.index.sai.disk.io.IndexOutputWriter;
import org.apache.cassandra.index.sai.disk.v1.segment.SegmentMetadata;
import org.apache.lucene.store.IndexOutput;
@ -51,31 +51,31 @@ public class NumericIndexWriter
private final BlockBalancedTreeWriter writer;
private final IndexDescriptor indexDescriptor;
private final IndexContext indexContext;
private final IndexIdentifier indexIdentifier;
private final int bytesPerValue;
/**
* @param maxSegmentRowId maximum possible segment row ID, used to create `maxRows` for the balanced tree
*/
public NumericIndexWriter(IndexDescriptor indexDescriptor,
IndexContext indexContext,
IndexIdentifier indexIdentifier,
int bytesPerValue,
long maxSegmentRowId)
{
this(indexDescriptor, indexContext, MAX_POINTS_IN_LEAF_NODE, bytesPerValue, maxSegmentRowId);
this(indexDescriptor, indexIdentifier, MAX_POINTS_IN_LEAF_NODE, bytesPerValue, maxSegmentRowId);
}
@VisibleForTesting
public NumericIndexWriter(IndexDescriptor indexDescriptor,
IndexContext indexContext,
IndexIdentifier indexIdentifier,
int maxPointsInLeafNode,
int bytesPerValue,
long maxSegmentRowId)
{
checkArgument(maxSegmentRowId >= 0, "[%s] maxSegmentRowId must be non-negative value, but got %s", indexContext.getIndexName(), maxSegmentRowId);
checkArgument(maxSegmentRowId >= 0, "[%s] maxSegmentRowId must be non-negative value, but got %s", indexIdentifier, maxSegmentRowId);
this.indexDescriptor = indexDescriptor;
this.indexContext = indexContext;
this.indexIdentifier = indexIdentifier;
this.bytesPerValue = bytesPerValue;
this.writer = new BlockBalancedTreeWriter(bytesPerValue, maxPointsInLeafNode);
}
@ -83,10 +83,7 @@ public class NumericIndexWriter
@Override
public String toString()
{
return MoreObjects.toStringHelper(this)
.add("indexContext", indexContext)
.add("bytesPerValue", bytesPerValue)
.toString();
return MoreObjects.toStringHelper(this).add("indexName", indexIdentifier).add("bytesPerValue", bytesPerValue).toString();
}
private static class LeafCallback implements BlockBalancedTreeWriter.Callback
@ -126,7 +123,7 @@ public class NumericIndexWriter
LeafCallback leafCallback = new LeafCallback();
try (IndexOutput treeOutput = indexDescriptor.openPerIndexOutput(IndexComponent.BALANCED_TREE, indexContext, true))
try (IndexOutput treeOutput = indexDescriptor.openPerIndexOutput(IndexComponent.BALANCED_TREE, indexIdentifier, true))
{
// The SSTable balanced tree component file is opened in append mode, so our offset is the current file pointer.
long treeOffset = treeOutput.getFilePointer();
@ -148,8 +145,11 @@ public class NumericIndexWriter
components.put(IndexComponent.BALANCED_TREE, treePosition, treeOffset, treeLength, attributes);
}
try (BlockBalancedTreeWalker reader = new BlockBalancedTreeWalker(indexDescriptor.createPerIndexFileHandle(IndexComponent.BALANCED_TREE, indexContext, null), treePosition);
IndexOutputWriter postingsOutput = indexDescriptor.openPerIndexOutput(IndexComponent.POSTING_LISTS, indexContext, true))
try (BlockBalancedTreeWalker reader = new BlockBalancedTreeWalker(indexDescriptor.createPerIndexFileHandle(IndexComponent.BALANCED_TREE,
indexIdentifier,
null),
treePosition);
IndexOutputWriter postingsOutput = indexDescriptor.openPerIndexOutput(IndexComponent.POSTING_LISTS, indexIdentifier, true))
{
long postingsOffset = postingsOutput.getFilePointer();
@ -157,7 +157,7 @@ public class NumericIndexWriter
reader.traverse(postingsWriter);
// The balanced tree postings writer already writes its own header & footer.
long postingsPosition = postingsWriter.finish(postingsOutput, leafCallback.leafPostings, indexContext);
long postingsPosition = postingsWriter.finish(postingsOutput, leafCallback.leafPostings, indexIdentifier);
Map<String, String> attributes = new LinkedHashMap<>();
attributes.put("num_leaf_postings", Integer.toString(postingsWriter.numLeafPostings));

16
src/java/org/apache/cassandra/index/sai/disk/v1/postings/PostingListRangeIterator.java
View File

@ -27,9 +27,9 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.exceptions.QueryCancelledException;
import org.apache.cassandra.index.sai.IndexContext;
import org.apache.cassandra.index.sai.QueryContext;
import org.apache.cassandra.index.sai.disk.PrimaryKeyMap;
import org.apache.cassandra.index.sai.utils.IndexIdentifier;
import org.apache.cassandra.index.sai.disk.v1.segment.IndexSegmentSearcherContext;
import org.apache.cassandra.index.sai.iterators.KeyRangeIterator;
import org.apache.cassandra.index.sai.postings.PostingList;
@ -62,7 +62,7 @@ public class PostingListRangeIterator extends KeyRangeIterator
private final QueryContext queryContext;
private final PostingList postingList;
private final IndexContext indexContext;
private final IndexIdentifier indexIdentifier;
private final PrimaryKeyMap primaryKeyMap;
private final long rowIdOffset;
@ -73,20 +73,20 @@ public class PostingListRangeIterator extends KeyRangeIterator
* Create a direct PostingListRangeIterator where the underlying PostingList is materialised
* immediately so the posting list size can be used.
*/
public PostingListRangeIterator(IndexContext indexContext,
public PostingListRangeIterator(IndexIdentifier indexIdentifier,
PrimaryKeyMap primaryKeyMap,
IndexSegmentSearcherContext searcherContext)
{
super(searcherContext.minimumKey, searcherContext.maximumKey, searcherContext.count());
this.indexContext = indexContext;
this.indexIdentifier = indexIdentifier;
this.primaryKeyMap = primaryKeyMap;
this.postingList = searcherContext.postingList;
this.rowIdOffset = searcherContext.segmentRowIdOffset;
this.queryContext = searcherContext.context;
}
public PostingListRangeIterator(IndexContext indexContext,
public PostingListRangeIterator(IndexIdentifier indexIdentifier,
PrimaryKeyMap primaryKeyMap,
PostingList postingList,
QueryContext queryContext)
@ -94,7 +94,7 @@ public class PostingListRangeIterator extends KeyRangeIterator
super(primaryKeyMap.primaryKeyFromRowId(postingList.minimum()),
primaryKeyMap.primaryKeyFromRowId(postingList.maximum()),
postingList.size());
this.indexContext = indexContext;
this.indexIdentifier = indexIdentifier;
this.primaryKeyMap = primaryKeyMap;
this.postingList = postingList;
this.rowIdOffset = 0;
@ -131,7 +131,7 @@ public class PostingListRangeIterator extends KeyRangeIterator
catch (Throwable t)
{
if (!(t instanceof QueryCancelledException))
logger.error(indexContext.logMessage("Unable to provide next token!"), t);
logger.error(indexIdentifier.logMessage("Unable to provide next token!"), t);
throw Throwables.cleaned(t);
}
@ -143,7 +143,7 @@ public class PostingListRangeIterator extends KeyRangeIterator
if (logger.isTraceEnabled())
{
final long exhaustedInMills = timeToExhaust.stop().elapsed(TimeUnit.MILLISECONDS);
logger.trace(indexContext.logMessage("PostingListRangeIterator exhausted after {} ms"), exhaustedInMills);
logger.trace(indexIdentifier.logMessage("PostingListRangeIterator exhausted after {} ms"), exhaustedInMills);
}
FileUtils.closeQuietly(Arrays.asList(postingList, primaryKeyMap));

10
src/java/org/apache/cassandra/index/sai/disk/v1/postings/PostingsWriter.java
View File

@ -25,10 +25,10 @@ import javax.annotation.concurrent.NotThreadSafe;
import com.google.common.annotations.VisibleForTesting;
import org.agrona.collections.LongArrayList;
import org.apache.cassandra.index.sai.IndexContext;
import org.apache.cassandra.index.sai.disk.ResettableByteBuffersIndexOutput;
import org.apache.cassandra.index.sai.disk.format.IndexComponent;
import org.apache.cassandra.index.sai.disk.format.IndexDescriptor;
import org.apache.cassandra.index.sai.utils.IndexIdentifier;
import org.apache.cassandra.index.sai.disk.io.IndexOutputWriter;
import org.apache.cassandra.index.sai.disk.v1.SAICodecUtils;
import org.apache.cassandra.index.sai.postings.PostingList;
@ -105,9 +105,9 @@ public class PostingsWriter implements Closeable
private long maxDelta;
private long totalPostings;
public PostingsWriter(IndexDescriptor indexDescriptor, IndexContext indexContext) throws IOException
public PostingsWriter(IndexDescriptor indexDescriptor, IndexIdentifier indexIdentifier) throws IOException
{
this(indexDescriptor, indexContext, BLOCK_SIZE);
this(indexDescriptor, indexIdentifier, BLOCK_SIZE);
}
public PostingsWriter(IndexOutputWriter dataOutput) throws IOException
@ -116,9 +116,9 @@ public class PostingsWriter implements Closeable
}
@VisibleForTesting
PostingsWriter(IndexDescriptor indexDescriptor, IndexContext indexContext, int blockSize) throws IOException
PostingsWriter(IndexDescriptor indexDescriptor, IndexIdentifier indexIdentifier, int blockSize) throws IOException
{
this(indexDescriptor.openPerIndexOutput(IndexComponent.POSTING_LISTS, indexContext, true), blockSize);
this(indexDescriptor.openPerIndexOutput(IndexComponent.POSTING_LISTS, indexIdentifier, true), blockSize);
}
private PostingsWriter(IndexOutputWriter dataOutput, int blockSize) throws IOException

22
src/java/org/apache/cassandra/index/sai/disk/v1/segment/IndexSegmentSearcher.java
View File

@ -22,8 +22,8 @@ import java.io.IOException;
import org.apache.cassandra.db.PartitionPosition;
import org.apache.cassandra.dht.AbstractBounds;
import org.apache.cassandra.index.sai.IndexContext;
import org.apache.cassandra.index.sai.QueryContext;
import org.apache.cassandra.index.sai.StorageAttachedIndex;
import org.apache.cassandra.index.sai.disk.PrimaryKeyMap;
import org.apache.cassandra.index.sai.disk.v1.PerColumnIndexFiles;
import org.apache.cassandra.index.sai.disk.v1.postings.PostingListRangeIterator;
@ -43,30 +43,30 @@ public abstract class IndexSegmentSearcher implements SegmentOrdering, Closeable
final PrimaryKeyMap.Factory primaryKeyMapFactory;
final PerColumnIndexFiles indexFiles;
final SegmentMetadata metadata;
final IndexContext indexContext;
final StorageAttachedIndex index;
IndexSegmentSearcher(PrimaryKeyMap.Factory primaryKeyMapFactory,
PerColumnIndexFiles perIndexFiles,
SegmentMetadata segmentMetadata,
IndexContext indexContext)
StorageAttachedIndex index)
{
this.primaryKeyMapFactory = primaryKeyMapFactory;
this.indexFiles = perIndexFiles;
this.metadata = segmentMetadata;
this.indexContext = indexContext;
this.index = index;
}
public static IndexSegmentSearcher open(PrimaryKeyMap.Factory primaryKeyMapFactory,
PerColumnIndexFiles indexFiles,
SegmentMetadata segmentMetadata,
IndexContext indexContext) throws IOException
StorageAttachedIndex index) throws IOException
{
if (indexContext.isVector())
return new VectorIndexSegmentSearcher(primaryKeyMapFactory, indexFiles, segmentMetadata, indexContext);
else if (indexContext.isLiteral())
return new LiteralIndexSegmentSearcher(primaryKeyMapFactory, indexFiles, segmentMetadata, indexContext);
if (index.termType().isVector())
return new VectorIndexSegmentSearcher(primaryKeyMapFactory, indexFiles, segmentMetadata, index);
else if (index.termType().isLiteral())
return new LiteralIndexSegmentSearcher(primaryKeyMapFactory, indexFiles, segmentMetadata, index);
else
return new NumericIndexSegmentSearcher(primaryKeyMapFactory, indexFiles, segmentMetadata, indexContext);
return new NumericIndexSegmentSearcher(primaryKeyMapFactory, indexFiles, segmentMetadata, index);
}
/**
@ -95,6 +95,6 @@ public abstract class IndexSegmentSearcher implements SegmentOrdering, Closeable
queryContext,
PeekablePostingList.makePeekable(postingList));
return new PostingListRangeIterator(indexContext, primaryKeyMapFactory.newPerSSTablePrimaryKeyMap(), searcherContext);
return new PostingListRangeIterator(index.identifier(), primaryKeyMapFactory.newPerSSTablePrimaryKeyMap(), searcherContext);
}
}

25
src/java/org/apache/cassandra/index/sai/disk/v1/segment/LiteralIndexSegmentSearcher.java
View File

@ -27,8 +27,8 @@ import org.slf4j.LoggerFactory;
import org.apache.cassandra.db.PartitionPosition;
import org.apache.cassandra.dht.AbstractBounds;
import org.apache.cassandra.index.sai.IndexContext;
import org.apache.cassandra.index.sai.QueryContext;
import org.apache.cassandra.index.sai.StorageAttachedIndex;
import org.apache.cassandra.index.sai.disk.PrimaryKeyMap;
import org.apache.cassandra.index.sai.disk.format.IndexComponent;
import org.apache.cassandra.index.sai.disk.v1.PerColumnIndexFiles;
@ -52,23 +52,20 @@ public class LiteralIndexSegmentSearcher extends IndexSegmentSearcher
LiteralIndexSegmentSearcher(PrimaryKeyMap.Factory primaryKeyMapFactory,
PerColumnIndexFiles perIndexFiles,
SegmentMetadata segmentMetadata,
IndexContext indexContext) throws IOException
StorageAttachedIndex index) throws IOException
{
super(primaryKeyMapFactory, perIndexFiles, segmentMetadata, indexContext);
super(primaryKeyMapFactory, perIndexFiles, segmentMetadata, index);
long root = metadata.getIndexRoot(IndexComponent.TERMS_DATA);
assert root >= 0;
perColumnEventListener = (QueryEventListener.TrieIndexEventListener)indexContext.getColumnQueryMetrics();
perColumnEventListener = (QueryEventListener.TrieIndexEventListener)index.columnQueryMetrics();
Map<String,String> map = metadata.componentMetadatas.get(IndexComponent.TERMS_DATA).attributes;
String footerPointerString = map.get(SAICodecUtils.FOOTER_POINTER);
long footerPointer = footerPointerString == null ? -1 : Long.parseLong(footerPointerString);
reader = new LiteralIndexSegmentTermsReader(indexContext,
indexFiles.termsData(),
indexFiles.postingLists(),
root, footerPointer);
reader = new LiteralIndexSegmentTermsReader(index.identifier(), indexFiles.termsData(), indexFiles.postingLists(), root, footerPointer);
}
@Override
@ -82,12 +79,12 @@ public class LiteralIndexSegmentSearcher extends IndexSegmentSearcher
public KeyRangeIterator search(Expression expression, AbstractBounds<PartitionPosition> keyRange, QueryContext queryContext) throws IOException
{
if (logger.isTraceEnabled())
logger.trace(indexContext.logMessage("Searching on expression '{}'..."), expression);
logger.trace(index.identifier().logMessage("Searching on expression '{}'..."), expression);
if (!expression.getOp().isEquality())
throw new IllegalArgumentException(indexContext.logMessage("Unsupported expression: " + expression));
if (!expression.getIndexOperator().isEquality())
throw new IllegalArgumentException(index.identifier().logMessage("Unsupported expression: " + expression));
final ByteComparable term = ByteComparable.fixedLength(expression.lower.value.encoded);
final ByteComparable term = ByteComparable.fixedLength(expression.lower().value.encoded);
QueryEventListener.TrieIndexEventListener listener = MulticastQueryEventListeners.of(queryContext, perColumnEventListener);
return toPrimaryKeyIterator(reader.exactMatch(term, listener, queryContext), queryContext);
}
@ -95,9 +92,7 @@ public class LiteralIndexSegmentSearcher extends IndexSegmentSearcher
@Override
public String toString()
{
return MoreObjects.toStringHelper(this)
.add("indexContext", indexContext)
.toString();
return MoreObjects.toStringHelper(this).add("index", index).toString();
}
@Override

10
src/java/org/apache/cassandra/index/sai/disk/v1/segment/LiteralIndexSegmentTermsReader.java
View File

@ -26,8 +26,8 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.exceptions.QueryCancelledException;
import org.apache.cassandra.index.sai.IndexContext;
import org.apache.cassandra.index.sai.QueryContext;
import org.apache.cassandra.index.sai.utils.IndexIdentifier;
import org.apache.cassandra.index.sai.disk.io.IndexFileUtils;
import org.apache.cassandra.index.sai.disk.v1.postings.PostingsReader;
import org.apache.cassandra.index.sai.disk.v1.trie.TrieTermsDictionaryReader;
@ -57,18 +57,18 @@ public class LiteralIndexSegmentTermsReader implements Closeable
{
private static final Logger logger = LoggerFactory.getLogger(LiteralIndexSegmentTermsReader.class);
private final IndexContext indexContext;
private final IndexIdentifier indexIdentifier;
private final FileHandle termDictionaryFile;
private final FileHandle postingsFile;
private final long termDictionaryRoot;
public LiteralIndexSegmentTermsReader(IndexContext indexContext,
public LiteralIndexSegmentTermsReader(IndexIdentifier indexIdentifier,
FileHandle termsData,
FileHandle postingLists,
long root,
long termsFooterPointer) throws IOException
{
this.indexContext = indexContext;
this.indexIdentifier = indexIdentifier;
termDictionaryFile = termsData;
postingsFile = postingLists;
termDictionaryRoot = root;
@ -137,7 +137,7 @@ public class LiteralIndexSegmentTermsReader implements Closeable
catch (Throwable e)
{
if (!(e instanceof QueryCancelledException))
logger.error(indexContext.logMessage("Failed to execute term query"), e);
logger.error(indexIdentifier.logMessage("Failed to execute term query"), e);
closeOnException();
throw Throwables.cleaned(e);

22
src/java/org/apache/cassandra/index/sai/disk/v1/segment/NumericIndexSegmentSearcher.java
View File

@ -26,8 +26,8 @@ import org.slf4j.LoggerFactory;
import org.apache.cassandra.db.PartitionPosition;
import org.apache.cassandra.dht.AbstractBounds;
import org.apache.cassandra.index.sai.IndexContext;
import org.apache.cassandra.index.sai.QueryContext;
import org.apache.cassandra.index.sai.StorageAttachedIndex;
import org.apache.cassandra.index.sai.disk.PrimaryKeyMap;
import org.apache.cassandra.index.sai.disk.format.IndexComponent;
import org.apache.cassandra.index.sai.disk.v1.PerColumnIndexFiles;
@ -53,23 +53,23 @@ public class NumericIndexSegmentSearcher extends IndexSegmentSearcher
NumericIndexSegmentSearcher(PrimaryKeyMap.Factory primaryKeyMapFactory,
PerColumnIndexFiles perIndexFiles,
SegmentMetadata segmentMetadata,
IndexContext indexContext) throws IOException
StorageAttachedIndex index) throws IOException
{
super(primaryKeyMapFactory, perIndexFiles, segmentMetadata, indexContext);
super(primaryKeyMapFactory, perIndexFiles, segmentMetadata, index);
final long treePosition = metadata.getIndexRoot(IndexComponent.BALANCED_TREE);
if (treePosition < 0)
throw new CorruptIndexException(indexContext.logMessage("The tree position is less than zero."), IndexComponent.BALANCED_TREE.name);
throw new CorruptIndexException(index.identifier().logMessage("The tree position is less than zero."), IndexComponent.BALANCED_TREE.name);
final long postingsPosition = metadata.getIndexRoot(IndexComponent.POSTING_LISTS);
if (postingsPosition < 0)
throw new CorruptIndexException(indexContext.logMessage("The postings position is less than zero."), IndexComponent.BALANCED_TREE.name);
throw new CorruptIndexException(index.identifier().logMessage("The postings position is less than zero."), IndexComponent.BALANCED_TREE.name);
treeReader = new BlockBalancedTreeReader(indexContext,
treeReader = new BlockBalancedTreeReader(index.identifier(),
indexFiles.balancedTree(),
treePosition,
indexFiles.postingLists(),
postingsPosition);
perColumnEventListener = (QueryEventListener.BalancedTreeEventListener)indexContext.getColumnQueryMetrics();
perColumnEventListener = (QueryEventListener.BalancedTreeEventListener)index.columnQueryMetrics();
}
@Override
@ -82,9 +82,9 @@ public class NumericIndexSegmentSearcher extends IndexSegmentSearcher
public KeyRangeIterator search(Expression exp, AbstractBounds<PartitionPosition> keyRange, QueryContext context) throws IOException
{
if (logger.isTraceEnabled())
logger.trace(indexContext.logMessage("Searching on expression '{}'..."), exp);
logger.trace(index.identifier().logMessage("Searching on expression '{}'..."), exp);
if (exp.getOp().isEqualityOrRange())
if (exp.getIndexOperator().isEqualityOrRange())
{
final BlockBalancedTreeReader.IntersectVisitor query = balancedTreeQueryFrom(exp, treeReader.getBytesPerValue());
QueryEventListener.BalancedTreeEventListener listener = MulticastQueryEventListeners.of(context, perColumnEventListener);
@ -92,7 +92,7 @@ public class NumericIndexSegmentSearcher extends IndexSegmentSearcher
}
else
{
throw new IllegalArgumentException(indexContext.logMessage("Unsupported expression during index query: " + exp));
throw new IllegalArgumentException(index.identifier().logMessage("Unsupported expression during index query: " + exp));
}
}
@ -100,7 +100,7 @@ public class NumericIndexSegmentSearcher extends IndexSegmentSearcher
public String toString()
{
return MoreObjects.toStringHelper(this)
.add("indexContext", indexContext)
.add("index", index)
.add("count", treeReader.getPointCount())
.add("bytesPerValue", treeReader.getBytesPerValue())
.toString();

8
src/java/org/apache/cassandra/index/sai/disk/v1/segment/Segment.java
View File

@ -27,13 +27,13 @@ import org.apache.cassandra.db.PartitionPosition;
import org.apache.cassandra.dht.AbstractBounds;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.index.sai.IndexContext;
import org.apache.cassandra.index.sai.QueryContext;
import org.apache.cassandra.index.sai.SSTableContext;
import org.apache.cassandra.index.sai.StorageAttachedIndex;
import org.apache.cassandra.index.sai.disk.PrimaryKeyMap;
import org.apache.cassandra.index.sai.disk.v1.PerColumnIndexFiles;
import org.apache.cassandra.index.sai.plan.Expression;
import org.apache.cassandra.index.sai.iterators.KeyRangeIterator;
import org.apache.cassandra.index.sai.plan.Expression;
import org.apache.cassandra.index.sai.utils.PrimaryKey;
import org.apache.cassandra.io.util.FileUtils;
@ -54,7 +54,7 @@ public class Segment implements SegmentOrdering, Closeable
private final IndexSegmentSearcher index;
public Segment(IndexContext indexContext, SSTableContext sstableContext, PerColumnIndexFiles indexFiles, SegmentMetadata metadata) throws IOException
public Segment(StorageAttachedIndex index, SSTableContext sstableContext, PerColumnIndexFiles indexFiles, SegmentMetadata metadata) throws IOException
{
this.minKeyBound = metadata.minKey.token().minKeyBound();
this.maxKeyBound = metadata.maxKey.token().maxKeyBound();
@ -62,7 +62,7 @@ public class Segment implements SegmentOrdering, Closeable
this.primaryKeyMapFactory = sstableContext.primaryKeyMapFactory;
this.metadata = metadata;
this.index = IndexSegmentSearcher.open(primaryKeyMapFactory, indexFiles, metadata, indexContext);
this.index = IndexSegmentSearcher.open(primaryKeyMapFactory, indexFiles, metadata, index);
}
@VisibleForTesting

63
src/java/org/apache/cassandra/index/sai/disk/v1/segment/SegmentBuilder.java
View File

@ -26,8 +26,6 @@ import com.google.common.annotations.VisibleForTesting;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.index.sai.IndexContext;
import org.apache.cassandra.index.sai.disk.format.IndexDescriptor;
import org.apache.cassandra.index.sai.disk.v1.IndexWriterConfig;
import org.apache.cassandra.index.sai.disk.v1.bbtree.BlockBalancedTreeRamBuffer;
@ -35,9 +33,10 @@ import org.apache.cassandra.index.sai.disk.v1.bbtree.NumericIndexWriter;
import org.apache.cassandra.index.sai.disk.v1.trie.LiteralIndexWriter;
import org.apache.cassandra.index.sai.disk.v1.vector.OnHeapGraph;
import org.apache.cassandra.index.sai.memory.RAMStringIndexer;
import org.apache.cassandra.index.sai.utils.IndexIdentifier;
import org.apache.cassandra.index.sai.utils.IndexTermType;
import org.apache.cassandra.index.sai.utils.NamedMemoryLimiter;
import org.apache.cassandra.index.sai.utils.PrimaryKey;
import org.apache.cassandra.index.sai.utils.TypeUtil;
import org.apache.cassandra.utils.FastByteOperations;
import org.apache.lucene.util.BytesRefBuilder;
@ -74,7 +73,7 @@ public abstract class SegmentBuilder
private ByteBuffer minTerm;
private ByteBuffer maxTerm;
final AbstractType<?> termComparator;
final IndexTermType indexTermType;
long totalBytesAllocated;
int rowCount = 0;
int maxSegmentRowId = -1;
@ -84,12 +83,12 @@ public abstract class SegmentBuilder
private final byte[] scratch;
private final BlockBalancedTreeRamBuffer trieBuffer;
public BlockBalancedTreeSegmentBuilder(AbstractType<?> termComparator, NamedMemoryLimiter limiter)
public BlockBalancedTreeSegmentBuilder(IndexTermType indexTermType, NamedMemoryLimiter limiter)
{
super(termComparator, limiter);
super(indexTermType, limiter);
scratch = new byte[TypeUtil.fixedSizeOf(termComparator)];
trieBuffer = new BlockBalancedTreeRamBuffer(TypeUtil.fixedSizeOf(termComparator));
scratch = new byte[indexTermType.fixedSizeOf()];
trieBuffer = new BlockBalancedTreeRamBuffer(indexTermType.fixedSizeOf());
totalBytesAllocated = this.trieBuffer.memoryUsed();
}
@ -102,16 +101,16 @@ public abstract class SegmentBuilder
@Override
protected long addInternal(ByteBuffer term, int segmentRowId)
{
TypeUtil.toComparableBytes(term, termComparator, scratch);
indexTermType.toComparableBytes(term, scratch);
return trieBuffer.add(segmentRowId, scratch);
}
@Override
protected SegmentMetadata.ComponentMetadataMap flushInternal(IndexDescriptor indexDescriptor, IndexContext indexContext) throws IOException
protected SegmentMetadata.ComponentMetadataMap flushInternal(IndexDescriptor indexDescriptor, IndexIdentifier indexIdentifier) throws IOException
{
NumericIndexWriter writer = new NumericIndexWriter(indexDescriptor,
indexContext,
TypeUtil.fixedSizeOf(termComparator),
indexIdentifier,
indexTermType.fixedSizeOf(),
maxSegmentRowId);
return writer.writeCompleteSegment(trieBuffer.iterator());
}
@ -123,9 +122,9 @@ public abstract class SegmentBuilder
final BytesRefBuilder stringBuffer = new BytesRefBuilder();
public RAMStringSegmentBuilder(AbstractType<?> termComparator, NamedMemoryLimiter limiter)
public RAMStringSegmentBuilder(IndexTermType indexTermType, NamedMemoryLimiter limiter)
{
super(termComparator, limiter);
super(indexTermType, limiter);
ramIndexer = new RAMStringIndexer();
totalBytesAllocated = ramIndexer.estimatedBytesUsed();
@ -145,9 +144,9 @@ public abstract class SegmentBuilder
}
@Override
protected SegmentMetadata.ComponentMetadataMap flushInternal(IndexDescriptor indexDescriptor, IndexContext indexContext) throws IOException
protected SegmentMetadata.ComponentMetadataMap flushInternal(IndexDescriptor indexDescriptor, IndexIdentifier indexIdentifier) throws IOException
{
try (LiteralIndexWriter writer = new LiteralIndexWriter(indexDescriptor, indexContext))
try (LiteralIndexWriter writer = new LiteralIndexWriter(indexDescriptor, indexIdentifier))
{
return writer.writeCompleteSegment(ramIndexer.getTermsWithPostings());
}
@ -167,10 +166,10 @@ public abstract class SegmentBuilder
{
private final OnHeapGraph<Integer> graphIndex;
public VectorSegmentBuilder(AbstractType<?> termComparator, NamedMemoryLimiter limiter, IndexWriterConfig indexWriterConfig)
public VectorSegmentBuilder(IndexTermType indexTermType, NamedMemoryLimiter limiter, IndexWriterConfig indexWriterConfig)
{
super(termComparator, limiter);
graphIndex = new OnHeapGraph<>(termComparator, indexWriterConfig, false);
super(indexTermType, limiter);
graphIndex = new OnHeapGraph<>(indexTermType.indexType(), indexWriterConfig, false);
}
@Override
@ -186,9 +185,9 @@ public abstract class SegmentBuilder
}
@Override
protected SegmentMetadata.ComponentMetadataMap flushInternal(IndexDescriptor indexDescriptor, IndexContext indexContext) throws IOException
protected SegmentMetadata.ComponentMetadataMap flushInternal(IndexDescriptor indexDescriptor, IndexIdentifier indexIdentifier) throws IOException
{
return graphIndex.writeData(indexDescriptor, indexContext, p -> p);
return graphIndex.writeData(indexDescriptor, indexIdentifier, p -> p);
}
}
@ -197,27 +196,27 @@ public abstract class SegmentBuilder
return ACTIVE_BUILDER_COUNT.get();
}
private SegmentBuilder(AbstractType<?> termComparator, NamedMemoryLimiter limiter)
private SegmentBuilder(IndexTermType indexTermType, NamedMemoryLimiter limiter)
{
this.termComparator = termComparator;
this.indexTermType = indexTermType;
this.limiter = limiter;
lastValidSegmentRowID = testLastValidSegmentRowId >= 0 ? testLastValidSegmentRowId : LAST_VALID_SEGMENT_ROW_ID;
minimumFlushBytes = limiter.limitBytes() / ACTIVE_BUILDER_COUNT.incrementAndGet();
}
public SegmentMetadata flush(IndexDescriptor indexDescriptor, IndexContext indexContext) throws IOException
public SegmentMetadata flush(IndexDescriptor indexDescriptor, IndexIdentifier indexIdentifier) throws IOException
{
assert !flushed : "Cannot flush an already flushed segment";
flushed = true;
if (getRowCount() == 0)
{
logger.warn(indexContext.logMessage("No rows to index during flush of SSTable {}."), indexDescriptor.sstableDescriptor);
logger.warn(indexIdentifier.logMessage("No rows to index during flush of SSTable {}."), indexDescriptor.sstableDescriptor);
return null;
}
SegmentMetadata.ComponentMetadataMap indexMetas = flushInternal(indexDescriptor, indexContext);
SegmentMetadata.ComponentMetadataMap indexMetas = flushInternal(indexDescriptor, indexIdentifier);
return new SegmentMetadata(segmentRowIdOffset, rowCount, minSSTableRowId, maxSSTableRowId, minKey, maxKey, minTerm, maxTerm, indexMetas);
}
@ -234,8 +233,8 @@ public abstract class SegmentBuilder
minKey = key;
maxKey = key;
minTerm = TypeUtil.min(term, minTerm, termComparator);
maxTerm = TypeUtil.max(term, maxTerm, termComparator);
minTerm = indexTermType.min(term, minTerm);
maxTerm = indexTermType.max(term, maxTerm);
if (rowCount == 0)
{
@ -282,11 +281,11 @@ public abstract class SegmentBuilder
* 2. It releases the builder's memory against its limiter.
* 3. It defensively marks the builder inactive to make sure nothing bad happens if we try to close it twice.
*
* @param indexContext an {@link IndexContext} used for creating logging messages
* @param indexIdentifier an {@link IndexIdentifier} used for creating logging messages
*
* @return the number of bytes used by the memory limiter after releasing this builder
*/
public long release(IndexContext indexContext)
public long release(IndexIdentifier indexIdentifier)
{
if (active)
{
@ -296,7 +295,7 @@ public abstract class SegmentBuilder
return used;
}
logger.warn(indexContext.logMessage("Attempted to release storage-attached index segment builder memory after builder marked inactive."));
logger.warn(indexIdentifier.logMessage("Attempted to release storage-attached index segment builder memory after builder marked inactive."));
return limiter.currentBytesUsed();
}
@ -304,7 +303,7 @@ public abstract class SegmentBuilder
protected abstract long addInternal(ByteBuffer term, int segmentRowId);
protected abstract SegmentMetadata.ComponentMetadataMap flushInternal(IndexDescriptor indexDescriptor, IndexContext indexContext) throws IOException;
protected abstract SegmentMetadata.ComponentMetadataMap flushInternal(IndexDescriptor indexDescriptor, IndexIdentifier indexIdentifier) throws IOException;
public int getRowCount()
{

25
src/java/org/apache/cassandra/index/sai/disk/v1/segment/VectorIndexSegmentSearcher.java
View File

@ -33,8 +33,8 @@ import io.github.jbellis.jvector.util.SparseFixedBitSet;
import org.agrona.collections.IntArrayList;
import org.apache.cassandra.db.PartitionPosition;
import org.apache.cassandra.dht.AbstractBounds;
import org.apache.cassandra.index.sai.IndexContext;
import org.apache.cassandra.index.sai.QueryContext;
import org.apache.cassandra.index.sai.StorageAttachedIndex;
import org.apache.cassandra.index.sai.VectorQueryContext;
import org.apache.cassandra.index.sai.disk.PrimaryKeyMap;
import org.apache.cassandra.index.sai.disk.v1.PerColumnIndexFiles;
@ -49,7 +49,6 @@ import org.apache.cassandra.index.sai.postings.PostingList;
import org.apache.cassandra.index.sai.utils.AtomicRatio;
import org.apache.cassandra.index.sai.utils.PrimaryKey;
import org.apache.cassandra.index.sai.utils.RangeUtil;
import org.apache.cassandra.index.sai.utils.TypeUtil;
import org.apache.cassandra.tracing.Tracing;
import static java.lang.Math.max;
@ -71,13 +70,13 @@ public class VectorIndexSegmentSearcher extends IndexSegmentSearcher
VectorIndexSegmentSearcher(PrimaryKeyMap.Factory primaryKeyMapFactory,
PerColumnIndexFiles perIndexFiles,
SegmentMetadata segmentMetadata,
IndexContext indexContext) throws IOException
StorageAttachedIndex index) throws IOException
{
super(primaryKeyMapFactory, perIndexFiles, segmentMetadata, indexContext);
graph = new DiskAnn(segmentMetadata.componentMetadatas, perIndexFiles, indexContext);
super(primaryKeyMapFactory, perIndexFiles, segmentMetadata, index);
graph = new DiskAnn(segmentMetadata.componentMetadatas, perIndexFiles, index.indexWriterConfig());
cachedBitSets = ThreadLocal.withInitial(() -> new SparseFixedBitSet(graph.size()));
globalBruteForceRows = Integer.MAX_VALUE;
optimizeFor = indexContext.getIndexWriterConfig().getOptimizeFor();
optimizeFor = index.indexWriterConfig().getOptimizeFor();
}
@Override
@ -92,17 +91,17 @@ public class VectorIndexSegmentSearcher extends IndexSegmentSearcher
int limit = context.vectorContext().limit();
if (logger.isTraceEnabled())
logger.trace(indexContext.logMessage("Searching on expression '{}'..."), exp);
logger.trace(index.identifier().logMessage("Searching on expression '{}'..."), exp);
if (exp.getOp() != Expression.IndexOperator.ANN)
throw new IllegalArgumentException(indexContext.logMessage("Unsupported expression during ANN index query: " + exp));
if (exp.getIndexOperator() != Expression.IndexOperator.ANN)
throw new IllegalArgumentException(index.identifier().logMessage("Unsupported expression during ANN index query: " + exp));
int topK = optimizeFor.topKFor(limit);
BitsOrPostingList bitsOrPostingList = bitsOrPostingListForKeyRange(context.vectorContext(), keyRange, topK);
if (bitsOrPostingList.skipANN())
return toPrimaryKeyIterator(bitsOrPostingList.postingList(), context);
float[] queryVector = TypeUtil.decomposeVector(indexContext, exp.lower.value.raw.duplicate());
float[] queryVector = index.termType().decomposeVector(exp.lower().value.raw.duplicate());
var vectorPostings = graph.search(queryVector, topK, limit, bitsOrPostingList.getBits());
if (bitsOrPostingList.expectedNodesVisited >= 0)
updateExpectedNodes(vectorPostings.getVisitedCount(), bitsOrPostingList.expectedNodesVisited);
@ -258,7 +257,7 @@ public class VectorIndexSegmentSearcher extends IndexSegmentSearcher
return toPrimaryKeyIterator(new IntArrayPostingList(rowIds.toIntArray()), context);
// else ask the index to perform a search limited to the bits we created
float[] queryVector = TypeUtil.decomposeVector(indexContext, expression.lower.value.raw.duplicate());
float[] queryVector = index.termType().decomposeVector(expression.lower().value.raw.duplicate());
var results = graph.search(queryVector, topK, limit, bits);
updateExpectedNodes(results.getVisitedCount(), expectedNodesVisited(topK, maxSegmentRowId, graph.size()));
return toPrimaryKeyIterator(results, context);
@ -304,9 +303,7 @@ public class VectorIndexSegmentSearcher extends IndexSegmentSearcher
@Override
public String toString()
{
return MoreObjects.toStringHelper(this)
.add("indexContext", indexContext)
.toString();
return MoreObjects.toStringHelper(this).add("index", index).toString();
}
@Override

16
src/java/org/apache/cassandra/index/sai/disk/v1/trie/LiteralIndexWriter.java
View File

@ -25,14 +25,14 @@ import javax.annotation.concurrent.NotThreadSafe;
import org.apache.commons.lang3.mutable.MutableLong;
import org.apache.cassandra.index.sai.IndexContext;
import org.apache.cassandra.index.sai.postings.PostingList;
import org.apache.cassandra.index.sai.utils.TermsIterator;
import org.apache.cassandra.index.sai.disk.format.IndexComponent;
import org.apache.cassandra.index.sai.disk.format.IndexDescriptor;
import org.apache.cassandra.index.sai.disk.v1.segment.SegmentMetadata;
import org.apache.cassandra.index.sai.disk.v1.postings.PostingsWriter;
import org.apache.cassandra.index.sai.utils.IndexIdentifier;
import org.apache.cassandra.index.sai.disk.v1.SAICodecUtils;
import org.apache.cassandra.index.sai.disk.v1.postings.PostingsWriter;
import org.apache.cassandra.index.sai.disk.v1.segment.SegmentMetadata;
import org.apache.cassandra.index.sai.postings.PostingList;
import org.apache.cassandra.index.sai.utils.TermsIterator;
import org.apache.cassandra.utils.bytecomparable.ByteComparable;
/**
@ -45,10 +45,10 @@ public class LiteralIndexWriter implements Closeable
private final PostingsWriter postingsWriter;
private long postingsAdded;
public LiteralIndexWriter(IndexDescriptor indexDescriptor, IndexContext indexContext) throws IOException
public LiteralIndexWriter(IndexDescriptor indexDescriptor, IndexIdentifier indexIdentifier) throws IOException
{
this.termsDictionaryWriter = new TrieTermsDictionaryWriter(indexDescriptor, indexContext);
this.postingsWriter = new PostingsWriter(indexDescriptor, indexContext);
this.termsDictionaryWriter = new TrieTermsDictionaryWriter(indexDescriptor, indexIdentifier);
this.postingsWriter = new PostingsWriter(indexDescriptor, indexIdentifier);
}
/**

8
src/java/org/apache/cassandra/index/sai/disk/v1/trie/TrieTermsDictionaryWriter.java
View File

@ -23,9 +23,9 @@ import javax.annotation.concurrent.NotThreadSafe;
import org.apache.commons.lang3.mutable.MutableLong;
import org.apache.cassandra.index.sai.IndexContext;
import org.apache.cassandra.index.sai.disk.format.IndexComponent;
import org.apache.cassandra.index.sai.disk.format.IndexDescriptor;
import org.apache.cassandra.index.sai.utils.IndexIdentifier;
import org.apache.cassandra.index.sai.disk.io.IndexOutputWriter;
import org.apache.cassandra.index.sai.disk.v1.SAICodecUtils;
import org.apache.cassandra.io.tries.IncrementalDeepTrieWriterPageAware;
@ -34,7 +34,7 @@ import org.apache.cassandra.utils.bytecomparable.ByteComparable;
/**
* Writes terms dictionary to disk in a trie format (see {@link IncrementalTrieWriter}).
*
* <p>
* Allows for variable-length keys. Trie values are 64-bit offsets to the posting file, pointing to the beginning of
* summary block for that postings list.
*/
@ -45,9 +45,9 @@ public class TrieTermsDictionaryWriter implements Closeable
private final IndexOutputWriter termDictionaryOutput;
private final long startOffset;
TrieTermsDictionaryWriter(IndexDescriptor indexDescriptor, IndexContext indexContext) throws IOException
TrieTermsDictionaryWriter(IndexDescriptor indexDescriptor, IndexIdentifier indexIdentifier) throws IOException
{
termDictionaryOutput = indexDescriptor.openPerIndexOutput(IndexComponent.TERMS_DATA, indexContext, true);
termDictionaryOutput = indexDescriptor.openPerIndexOutput(IndexComponent.TERMS_DATA, indexIdentifier, true);
startOffset = termDictionaryOutput.getFilePointer();
SAICodecUtils.writeHeader(termDictionaryOutput);

101
src/java/org/apache/cassandra/index/sai/disk/v1/vector/CheckpointingIterator.java
View File

@ -1,101 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.index.sai.disk.v1.vector;
import com.google.common.collect.Iterables;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.index.sai.QueryContext;
import org.apache.cassandra.index.sai.disk.SSTableIndex;
import org.apache.cassandra.index.sai.iterators.KeyRangeIterator;
import org.apache.cassandra.index.sai.utils.PrimaryKey;
import org.apache.cassandra.io.util.FileUtils;
/**
* A {@link KeyRangeIterator}, used when returning results from a vector search, that checks for query timeouts
* during the iteration process.
*/
public class CheckpointingIterator extends KeyRangeIterator
{
private static final Logger logger = LoggerFactory.getLogger(CheckpointingIterator.class);
private final QueryContext context;
private final KeyRangeIterator union;
private final Iterable<SSTableIndex> referencedIndexes;
public CheckpointingIterator(KeyRangeIterator wrapped,
Iterable<SSTableIndex> referencedIndexes,
Iterable<SSTableIndex> referencedAnnIndexesInHybridSearch,
QueryContext queryContext)
{
super(wrapped.getMinimum(), wrapped.getMaximum(), wrapped.getCount());
this.union = wrapped;
if (referencedAnnIndexesInHybridSearch != null)
this.referencedIndexes = Iterables.concat(referencedIndexes, referencedAnnIndexesInHybridSearch);
else
this.referencedIndexes = referencedIndexes;
this.context = queryContext;
}
@Override
protected PrimaryKey computeNext()
{
try
{
return union.hasNext() ? union.next() : endOfData();
}
finally
{
context.checkpoint();
}
}
@Override
protected void performSkipTo(PrimaryKey nextKey)
{
try
{
union.skipTo(nextKey);
}
finally
{
context.checkpoint();
}
}
@Override
public void close()
{
FileUtils.closeQuietly(union);
referencedIndexes.forEach(CheckpointingIterator::releaseQuietly);
}
private static void releaseQuietly(SSTableIndex index)
{
try
{
index.release();
}
catch (Throwable e)
{
logger.error(index.getIndexContext().logMessage(String.format("Failed to release index on SSTable %s", index.getSSTable())), e);
}
}
}

6
src/java/org/apache/cassandra/index/sai/disk/v1/vector/DiskAnn.java
View File

@ -34,8 +34,8 @@ import io.github.jbellis.jvector.graph.SearchResult.NodeScore;
import io.github.jbellis.jvector.pq.CompressedVectors;
import io.github.jbellis.jvector.util.Bits;
import io.github.jbellis.jvector.vector.VectorSimilarityFunction;
import org.apache.cassandra.index.sai.IndexContext;
import org.apache.cassandra.index.sai.disk.format.IndexComponent;
import org.apache.cassandra.index.sai.disk.v1.IndexWriterConfig;
import org.apache.cassandra.index.sai.disk.v1.PerColumnIndexFiles;
import org.apache.cassandra.index.sai.disk.v1.postings.VectorPostingList;
import org.apache.cassandra.index.sai.disk.v1.segment.SegmentMetadata;
@ -52,9 +52,9 @@ public class DiskAnn implements AutoCloseable
// only one of these will be not null
private final CompressedVectors compressedVectors;
public DiskAnn(SegmentMetadata.ComponentMetadataMap componentMetadatas, PerColumnIndexFiles indexFiles, IndexContext context) throws IOException
public DiskAnn(SegmentMetadata.ComponentMetadataMap componentMetadatas, PerColumnIndexFiles indexFiles, IndexWriterConfig config) throws IOException
{
similarityFunction = context.getIndexWriterConfig().getSimilarityFunction();
similarityFunction = config.getSimilarityFunction();
SegmentMetadata.ComponentMetadata termsMetadata = componentMetadatas.get(IndexComponent.TERMS_DATA);
graphHandle = indexFiles.termsData();

10
src/java/org/apache/cassandra/index/sai/disk/v1/vector/OnHeapGraph.java
View File

@ -49,9 +49,9 @@ import io.github.jbellis.jvector.vector.VectorSimilarityFunction;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.db.marshal.VectorType;
import org.apache.cassandra.exceptions.InvalidRequestException;
import org.apache.cassandra.index.sai.IndexContext;
import org.apache.cassandra.index.sai.disk.format.IndexComponent;
import org.apache.cassandra.index.sai.disk.format.IndexDescriptor;
import org.apache.cassandra.index.sai.utils.IndexIdentifier;
import org.apache.cassandra.index.sai.disk.io.IndexFileUtils;
import org.apache.cassandra.index.sai.disk.v1.IndexWriterConfig;
import org.apache.cassandra.index.sai.disk.v1.SAICodecUtils;
@ -280,7 +280,7 @@ public class OnHeapGraph<T>
return keyQueue;
}
public SegmentMetadata.ComponentMetadataMap writeData(IndexDescriptor indexDescriptor, IndexContext indexContext, Function<T, Integer> postingTransformer) throws IOException
public SegmentMetadata.ComponentMetadataMap writeData(IndexDescriptor indexDescriptor, IndexIdentifier indexIdentifier, Function<T, Integer> postingTransformer) throws IOException
{
int nInProgress = builder.insertsInProgress();
assert nInProgress == 0 : String.format("Attempting to write graph while %d inserts are in progress", nInProgress);
@ -292,9 +292,9 @@ public class OnHeapGraph<T>
postingsMap.keySet().size(), vectorValues.size());
logger.debug("Writing graph with {} rows and {} distinct vectors", postingsMap.values().stream().mapToInt(VectorPostings::size).sum(), vectorValues.size());
try (var pqOutput = IndexFileUtils.instance.openOutput(indexDescriptor.fileFor(IndexComponent.COMPRESSED_VECTORS, indexContext), true);
var postingsOutput = IndexFileUtils.instance.openOutput(indexDescriptor.fileFor(IndexComponent.POSTING_LISTS, indexContext), true);
var indexOutput = IndexFileUtils.instance.openOutput(indexDescriptor.fileFor(IndexComponent.TERMS_DATA, indexContext), true))
try (var pqOutput = IndexFileUtils.instance.openOutput(indexDescriptor.fileFor(IndexComponent.COMPRESSED_VECTORS, indexIdentifier), true);
var postingsOutput = IndexFileUtils.instance.openOutput(indexDescriptor.fileFor(IndexComponent.POSTING_LISTS, indexIdentifier), true);
var indexOutput = IndexFileUtils.instance.openOutput(indexDescriptor.fileFor(IndexComponent.TERMS_DATA, indexIdentifier), true))
{
SAICodecUtils.writeHeader(pqOutput);
SAICodecUtils.writeHeader(postingsOutput);

11
src/java/org/apache/cassandra/index/sai/memory/MemoryIndex.java
View File

@ -22,9 +22,10 @@ import org.apache.cassandra.db.Clustering;
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.db.PartitionPosition;
import org.apache.cassandra.dht.AbstractBounds;
import org.apache.cassandra.index.sai.IndexContext;
import org.apache.cassandra.index.sai.QueryContext;
import org.apache.cassandra.index.sai.StorageAttachedIndex;
import org.apache.cassandra.index.sai.disk.format.IndexDescriptor;
import org.apache.cassandra.index.sai.utils.IndexIdentifier;
import org.apache.cassandra.index.sai.disk.v1.segment.SegmentMetadata;
import org.apache.cassandra.index.sai.iterators.KeyRangeIterator;
import org.apache.cassandra.index.sai.plan.Expression;
@ -40,11 +41,11 @@ import java.util.function.Function;
public abstract class MemoryIndex implements MemtableOrdering
{
protected final IndexContext indexContext;
protected final StorageAttachedIndex index;
protected MemoryIndex(IndexContext indexContext)
protected MemoryIndex(StorageAttachedIndex index)
{
this.indexContext = indexContext;
this.index = index;
}
public abstract long add(DecoratedKey key, Clustering<?> clustering, ByteBuffer value);
@ -65,6 +66,6 @@ public abstract class MemoryIndex implements MemtableOrdering
public abstract Iterator<Pair<ByteComparable, PrimaryKeys>> iterator();
public abstract SegmentMetadata.ComponentMetadataMap writeDirect(IndexDescriptor indexDescriptor,
IndexContext indexContext,
IndexIdentifier indexIdentifier,
Function<PrimaryKey, Integer> postingTransformer) throws IOException;
}

31
src/java/org/apache/cassandra/index/sai/memory/MemtableIndex.java
View File

@ -29,26 +29,27 @@ import org.apache.cassandra.db.Clustering;
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.db.PartitionPosition;
import org.apache.cassandra.dht.AbstractBounds;
import org.apache.cassandra.index.sai.IndexContext;
import org.apache.cassandra.index.sai.QueryContext;
import org.apache.cassandra.index.sai.StorageAttachedIndex;
import org.apache.cassandra.index.sai.disk.format.IndexDescriptor;
import org.apache.cassandra.index.sai.utils.IndexIdentifier;
import org.apache.cassandra.index.sai.disk.v1.segment.SegmentMetadata;
import org.apache.cassandra.index.sai.iterators.KeyRangeIterator;
import org.apache.cassandra.index.sai.plan.Expression;
import org.apache.cassandra.index.sai.utils.PrimaryKey;
import org.apache.cassandra.index.sai.utils.PrimaryKeys;
import org.apache.cassandra.index.sai.iterators.KeyRangeIterator;
import org.apache.cassandra.utils.Pair;
import org.apache.cassandra.utils.bytecomparable.ByteComparable;
public class MemtableIndex implements MemtableOrdering
{
private final MemoryIndex index;
private final MemoryIndex memoryIndex;
private final LongAdder writeCount = new LongAdder();
private final LongAdder estimatedMemoryUsed = new LongAdder();
public MemtableIndex(IndexContext indexContext)
public MemtableIndex(StorageAttachedIndex index)
{
this.index = indexContext.isVector() ? new VectorMemoryIndex(indexContext) : new TrieMemoryIndex(indexContext);
this.memoryIndex = index.termType().isVector() ? new VectorMemoryIndex(index) : new TrieMemoryIndex(index);
}
public long writeCount()
@ -63,17 +64,17 @@ public class MemtableIndex implements MemtableOrdering
public boolean isEmpty()
{
return index.isEmpty();
return memoryIndex.isEmpty();
}
public ByteBuffer getMinTerm()
{
return index.getMinTerm();
return memoryIndex.getMinTerm();
}
public ByteBuffer getMaxTerm()
{
return index.getMaxTerm();
return memoryIndex.getMaxTerm();
}
public long index(DecoratedKey key, Clustering<?> clustering, ByteBuffer value)
@ -81,7 +82,7 @@ public class MemtableIndex implements MemtableOrdering
if (value == null || value.remaining() == 0)
return 0;
long ram = index.add(key, clustering, value);
long ram = memoryIndex.add(key, clustering, value);
writeCount.increment();
estimatedMemoryUsed.add(ram);
return ram;
@ -89,29 +90,29 @@ public class MemtableIndex implements MemtableOrdering
public long update(DecoratedKey key, Clustering<?> clustering, ByteBuffer oldValue, ByteBuffer newValue)
{
return index.update(key, clustering, oldValue, newValue);
return memoryIndex.update(key, clustering, oldValue, newValue);
}
public KeyRangeIterator search(QueryContext queryContext, Expression expression, AbstractBounds<PartitionPosition> keyRange)
{
return index.search(queryContext, expression, keyRange);
return memoryIndex.search(queryContext, expression, keyRange);
}
public Iterator<Pair<ByteComparable, PrimaryKeys>> iterator()
{
return index.iterator();
return memoryIndex.iterator();
}
public SegmentMetadata.ComponentMetadataMap writeDirect(IndexDescriptor indexDescriptor,
IndexContext indexContext,
IndexIdentifier indexIdentifier,
Function<PrimaryKey, Integer> postingTransformer) throws IOException
{
return index.writeDirect(indexDescriptor, indexContext, postingTransformer);
return memoryIndex.writeDirect(indexDescriptor, indexIdentifier, postingTransformer);
}
@Override
public KeyRangeIterator limitToTopResults(List<PrimaryKey> primaryKeys, Expression expression, int limit)
{
return index.limitToTopResults(primaryKeys, expression, limit);
return memoryIndex.limitToTopResults(primaryKeys, expression, limit);
}
}

24
src/java/org/apache/cassandra/index/sai/memory/MemtableIndexManager.java
View File

@ -36,8 +36,8 @@ import org.apache.cassandra.db.lifecycle.LifecycleNewTracker;
import org.apache.cassandra.db.memtable.Memtable;
import org.apache.cassandra.db.rows.Row;
import org.apache.cassandra.dht.AbstractBounds;
import org.apache.cassandra.index.sai.IndexContext;
import org.apache.cassandra.index.sai.QueryContext;
import org.apache.cassandra.index.sai.StorageAttachedIndex;
import org.apache.cassandra.index.sai.plan.Expression;
import org.apache.cassandra.index.sai.iterators.KeyRangeIterator;
import org.apache.cassandra.index.sai.iterators.KeyRangeUnionIterator;
@ -47,12 +47,12 @@ import org.apache.cassandra.utils.FBUtilities;
public class MemtableIndexManager
{
private final IndexContext indexContext;
private final StorageAttachedIndex index;
private final ConcurrentMap<Memtable, MemtableIndex> liveMemtableIndexMap;
public MemtableIndexManager(IndexContext indexContext)
public MemtableIndexManager(StorageAttachedIndex index)
{
this.indexContext = indexContext;
this.index = index;
this.liveMemtableIndexMap = new ConcurrentHashMap<>();
}
@ -64,15 +64,15 @@ public class MemtableIndexManager
// call to computeIfAbsent() if it's not. (see https://bugs.openjdk.java.net/browse/JDK-8161372)
MemtableIndex target = (current != null)
? current
: liveMemtableIndexMap.computeIfAbsent(mt, memtable -> new MemtableIndex(indexContext));
: liveMemtableIndexMap.computeIfAbsent(mt, memtable -> new MemtableIndex(index));
long start = Clock.Global.nanoTime();
long bytes = 0;
if (indexContext.isNonFrozenCollection())
if (index.termType().isNonFrozenCollection())
{
Iterator<ByteBuffer> bufferIterator = indexContext.getValuesOf(row, FBUtilities.nowInSeconds());
Iterator<ByteBuffer> bufferIterator = index.termType().valuesOf(row, FBUtilities.nowInSeconds());
if (bufferIterator != null)
{
while (bufferIterator.hasNext())
@ -84,16 +84,16 @@ public class MemtableIndexManager
}
else
{
ByteBuffer value = indexContext.getValueOf(key, row, FBUtilities.nowInSeconds());
ByteBuffer value = index.termType().valueOf(key, row, FBUtilities.nowInSeconds());
bytes += target.index(key, row.clustering(), value);
}
indexContext.getIndexMetrics().memtableIndexWriteLatency.update(Clock.Global.nanoTime() - start, TimeUnit.NANOSECONDS);
index.indexMetrics().memtableIndexWriteLatency.update(Clock.Global.nanoTime() - start, TimeUnit.NANOSECONDS);
return bytes;
}
public long update(DecoratedKey key, Row oldRow, Row newRow, Memtable memtable)
{
if (!indexContext.isVector())
if (!index.termType().isVector())
{
return index(key, newRow, memtable);
}
@ -102,8 +102,8 @@ public class MemtableIndexManager
if (target == null)
return 0;
ByteBuffer oldValue = indexContext.getValueOf(key, oldRow, FBUtilities.nowInSeconds());
ByteBuffer newValue = indexContext.getValueOf(key, newRow, FBUtilities.nowInSeconds());
ByteBuffer oldValue = index.termType().valueOf(key, oldRow, FBUtilities.nowInSeconds());
ByteBuffer newValue = index.termType().valueOf(key, newRow, FBUtilities.nowInSeconds());
return target.update(key, oldRow.clustering(), oldValue, newValue);
}

133
src/java/org/apache/cassandra/index/sai/memory/TrieMemoryIndex.java
View File

@ -26,7 +26,6 @@ import java.util.SortedSet;
import java.util.concurrent.atomic.LongAdder;
import java.util.function.Function;
import org.apache.cassandra.index.sai.QueryContext;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -34,20 +33,20 @@ import io.netty.util.concurrent.FastThreadLocal;
import org.apache.cassandra.db.Clustering;
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.db.PartitionPosition;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.db.memtable.TrieMemtable;
import org.apache.cassandra.db.tries.InMemoryTrie;
import org.apache.cassandra.db.tries.Trie;
import org.apache.cassandra.dht.AbstractBounds;
import org.apache.cassandra.index.sai.IndexContext;
import org.apache.cassandra.index.sai.QueryContext;
import org.apache.cassandra.index.sai.StorageAttachedIndex;
import org.apache.cassandra.index.sai.analyzer.AbstractAnalyzer;
import org.apache.cassandra.index.sai.disk.format.IndexDescriptor;
import org.apache.cassandra.index.sai.disk.v1.segment.SegmentMetadata;
import org.apache.cassandra.index.sai.iterators.KeyRangeIterator;
import org.apache.cassandra.index.sai.plan.Expression;
import org.apache.cassandra.index.sai.utils.IndexIdentifier;
import org.apache.cassandra.index.sai.utils.PrimaryKey;
import org.apache.cassandra.index.sai.utils.PrimaryKeys;
import org.apache.cassandra.index.sai.iterators.KeyRangeIterator;
import org.apache.cassandra.index.sai.utils.TypeUtil;
import org.apache.cassandra.utils.Pair;
import org.apache.cassandra.utils.bytecomparable.ByteComparable;
import org.apache.cassandra.utils.bytecomparable.ByteSource;
@ -65,22 +64,18 @@ public class TrieMemoryIndex extends MemoryIndex
private final InMemoryTrie<PrimaryKeys> data;
private final PrimaryKeysReducer primaryKeysReducer;
private final AbstractAnalyzer.AnalyzerFactory analyzerFactory;
private final AbstractType<?> validator;
private final boolean isLiteral;
private ByteBuffer minTerm;
private ByteBuffer maxTerm;
public TrieMemoryIndex(IndexContext indexContext)
public TrieMemoryIndex(StorageAttachedIndex index)
{
super(indexContext);
super(index);
this.data = new InMemoryTrie<>(TrieMemtable.BUFFER_TYPE);
this.primaryKeysReducer = new PrimaryKeysReducer();
// The use of the analyzer is within a synchronized block so can be considered thread-safe
this.analyzerFactory = indexContext.getAnalyzerFactory();
this.validator = indexContext.getValidator();
this.isLiteral = TypeUtil.isLiteral(validator);
this.isLiteral = index.termType().isLiteral();
}
/**
@ -94,53 +89,37 @@ public class TrieMemoryIndex extends MemoryIndex
@Override
public synchronized long add(DecoratedKey key, Clustering<?> clustering, ByteBuffer value)
{
AbstractAnalyzer analyzer = analyzerFactory.create();
try
value = index.termType().asIndexBytes(value);
final PrimaryKey primaryKey = index.hasClustering() ? index.keyFactory().create(key, clustering)
: index.keyFactory().create(key);
final long initialSizeOnHeap = data.sizeOnHeap();
final long initialSizeOffHeap = data.sizeOffHeap();
final long reducerHeapSize = primaryKeysReducer.heapAllocations();
if (index.hasAnalyzer())
{
value = TypeUtil.asIndexBytes(value, validator);
analyzer.reset(value);
final PrimaryKey primaryKey = indexContext.hasClustering() ? indexContext.keyFactory().create(key, clustering)
: indexContext.keyFactory().create(key);
final long initialSizeOnHeap = data.sizeOnHeap();
final long initialSizeOffHeap = data.sizeOffHeap();
final long reducerHeapSize = primaryKeysReducer.heapAllocations();
while (analyzer.hasNext())
AbstractAnalyzer analyzer = index.analyzer();
try
{
final ByteBuffer term = analyzer.next();
if (!indexContext.validateMaxTermSize(key, term, false))
continue;
setMinMaxTerm(term.duplicate());
final ByteComparable comparableBytes = asComparableBytes(term);
try
analyzer.reset(value);
while (analyzer.hasNext())
{
if (term.limit() <= MAX_RECURSIVE_KEY_LENGTH)
{
data.putRecursive(comparableBytes, primaryKey, primaryKeysReducer);
}
else
{
data.apply(Trie.singleton(comparableBytes, primaryKey), primaryKeysReducer);
}
}
catch (InMemoryTrie.SpaceExhaustedException e)
{
throw new RuntimeException(e);
addTerm(primaryKey, analyzer.next());
}
}
long onHeap = data.sizeOnHeap();
long offHeap = data.sizeOffHeap();
long heapAllocations = primaryKeysReducer.heapAllocations();
return (onHeap - initialSizeOnHeap) + (offHeap - initialSizeOffHeap) + (heapAllocations - reducerHeapSize);
finally
{
analyzer.end();
}
}
finally
else
{
analyzer.end();
addTerm(primaryKey, value);
}
long onHeap = data.sizeOnHeap();
long offHeap = data.sizeOffHeap();
long heapAllocations = primaryKeysReducer.heapAllocations();
return (onHeap - initialSizeOnHeap) + (offHeap - initialSizeOffHeap) + (heapAllocations - reducerHeapSize);
}
@Override
@ -162,7 +141,7 @@ public class TrieMemoryIndex extends MemoryIndex
if (logger.isTraceEnabled())
logger.trace("Searching memtable index on expression '{}'...", expression);
switch (expression.getOp())
switch (expression.getIndexOperator())
{
case EQ:
case CONTAINS_KEY:
@ -204,7 +183,7 @@ public class TrieMemoryIndex extends MemoryIndex
@Override
public SegmentMetadata.ComponentMetadataMap writeDirect(IndexDescriptor indexDescriptor,
IndexContext indexContext,
IndexIdentifier indexIdentifier,
Function<PrimaryKey, Integer> postingTransformer)
{
throw new UnsupportedOperationException();
@ -228,18 +207,44 @@ public class TrieMemoryIndex extends MemoryIndex
return maxTerm;
}
private void addTerm(PrimaryKey primaryKey, ByteBuffer term)
{
if (index.validateMaxTermSize(primaryKey.partitionKey(), term, false))
{
setMinMaxTerm(term.duplicate());
final ByteComparable comparableBytes = asComparableBytes(term);
try
{
if (term.limit() <= MAX_RECURSIVE_KEY_LENGTH)
{
data.putRecursive(comparableBytes, primaryKey, primaryKeysReducer);
}
else
{
data.apply(Trie.singleton(comparableBytes, primaryKey), primaryKeysReducer);
}
}
catch (InMemoryTrie.SpaceExhaustedException e)
{
throw new RuntimeException(e);
}
}
}
private void setMinMaxTerm(ByteBuffer term)
{
assert term != null;
minTerm = TypeUtil.min(term, minTerm, indexContext.getValidator());
maxTerm = TypeUtil.max(term, maxTerm, indexContext.getValidator());
minTerm = index.termType().min(term, minTerm);
maxTerm = index.termType().max(term, maxTerm);
}
private ByteComparable asComparableBytes(ByteBuffer input)
{
return isLiteral ? version -> terminated(ByteSource.of(input, version))
: version -> TypeUtil.asComparableBytes(input, validator, version);
: version -> index.termType().asComparableBytes(input, version);
}
private ByteComparable decode(ByteComparable term)
@ -271,8 +276,8 @@ public class TrieMemoryIndex extends MemoryIndex
private KeyRangeIterator exactMatch(Expression expression, AbstractBounds<PartitionPosition> keyRange)
{
ByteComparable comparableMatch = expression.lower == null ? ByteComparable.EMPTY
: asComparableBytes(expression.lower.value.encoded);
ByteComparable comparableMatch = expression.lower() == null ? ByteComparable.EMPTY
: asComparableBytes(expression.lower().value.encoded);
PrimaryKeys primaryKeys = data.get(comparableMatch);
return primaryKeys == null ? KeyRangeIterator.empty()
: new FilteringInMemoryKeyRangeIterator(primaryKeys.keys(), keyRange);
@ -348,10 +353,10 @@ public class TrieMemoryIndex extends MemoryIndex
{
ByteComparable lowerBound, upperBound;
boolean lowerInclusive, upperInclusive;
if (expression.lower != null)
if (expression.lower() != null)
{
lowerBound = asComparableBytes(expression.lower.value.encoded);
lowerInclusive = expression.lower.inclusive;
lowerBound = asComparableBytes(expression.lower().value.encoded);
lowerInclusive = expression.lower().inclusive;
}
else
{
@ -359,10 +364,10 @@ public class TrieMemoryIndex extends MemoryIndex
lowerInclusive = false;
}
if (expression.upper != null)
if (expression.upper() != null)
{
upperBound = asComparableBytes(expression.upper.value.encoded);
upperInclusive = expression.upper.inclusive;
upperBound = asComparableBytes(expression.upper().value.encoded);
upperInclusive = expression.upper().inclusive;
}
else
{

40
src/java/org/apache/cassandra/index/sai/memory/VectorMemoryIndex.java
View File

@ -37,10 +37,11 @@ import org.apache.cassandra.db.Clustering;
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.db.PartitionPosition;
import org.apache.cassandra.dht.AbstractBounds;
import org.apache.cassandra.index.sai.IndexContext;
import org.apache.cassandra.index.sai.QueryContext;
import org.apache.cassandra.index.sai.StorageAttachedIndex;
import org.apache.cassandra.index.sai.VectorQueryContext;
import org.apache.cassandra.index.sai.disk.format.IndexDescriptor;
import org.apache.cassandra.index.sai.utils.IndexIdentifier;
import org.apache.cassandra.index.sai.disk.v1.segment.SegmentMetadata;
import org.apache.cassandra.index.sai.disk.v1.vector.OnHeapGraph;
import org.apache.cassandra.index.sai.iterators.KeyRangeIterator;
@ -49,7 +50,6 @@ import org.apache.cassandra.index.sai.plan.Expression;
import org.apache.cassandra.index.sai.utils.PrimaryKey;
import org.apache.cassandra.index.sai.utils.PrimaryKeys;
import org.apache.cassandra.index.sai.utils.RangeUtil;
import org.apache.cassandra.index.sai.utils.TypeUtil;
import org.apache.cassandra.tracing.Tracing;
import org.apache.cassandra.utils.Pair;
import org.apache.cassandra.utils.bytecomparable.ByteComparable;
@ -69,20 +69,20 @@ public class VectorMemoryIndex extends MemoryIndex
private final NavigableSet<PrimaryKey> primaryKeys = new ConcurrentSkipListSet<>();
public VectorMemoryIndex(IndexContext indexContext)
public VectorMemoryIndex(StorageAttachedIndex index)
{
super(indexContext);
this.graph = new OnHeapGraph<>(indexContext.getValidator(), indexContext.getIndexWriterConfig());
super(index);
this.graph = new OnHeapGraph<>(index.termType().indexType(), index.indexWriterConfig());
}
@Override
public synchronized long add(DecoratedKey key, Clustering<?> clustering, ByteBuffer value)
{
if (value == null || value.remaining() == 0 || !indexContext.validateMaxTermSize(key, value, false))
if (value == null || value.remaining() == 0 || !index.validateMaxTermSize(key, value, false))
return 0;
var primaryKey = indexContext.hasClustering() ? indexContext.keyFactory().create(key, clustering)
: indexContext.keyFactory().create(key);
var primaryKey = index.hasClustering() ? index.keyFactory().create(key, clustering)
: index.keyFactory().create(key);
return index(primaryKey, value);
}
@ -117,8 +117,8 @@ public class VectorMemoryIndex extends MemoryIndex
long bytesUsed = 0;
if (different)
{
var primaryKey = indexContext.hasClustering() ? indexContext.keyFactory().create(key, clustering)
: indexContext.keyFactory().create(key);
var primaryKey = index.hasClustering() ? index.keyFactory().create(key, clustering)
: index.keyFactory().create(key);
// update bounds because only rows with vectors are included in the key bounds,
// so if the vector was null before, we won't have included it
updateKeyBounds(primaryKey);
@ -151,12 +151,12 @@ public class VectorMemoryIndex extends MemoryIndex
@Override
public KeyRangeIterator search(QueryContext queryContext, Expression expr, AbstractBounds<PartitionPosition> keyRange)
{
assert expr.getOp() == Expression.IndexOperator.ANN : "Only ANN is supported for vector search, received " + expr.getOp();
assert expr.getIndexOperator() == Expression.IndexOperator.ANN : "Only ANN is supported for vector search, received " + expr.getIndexOperator();
VectorQueryContext vectorQueryContext = queryContext.vectorContext();
var buffer = expr.lower.value.raw;
float[] qv = TypeUtil.decomposeVector(indexContext, buffer);
var buffer = expr.lower().value.raw;
float[] qv = index.termType().decomposeVector(buffer);
Bits bits;
if (!RangeUtil.coversFullRing(keyRange))
@ -168,8 +168,8 @@ public class VectorMemoryIndex extends MemoryIndex
// if right token is MAX (Long.MIN_VALUE), there is no upper bound
boolean isMaxToken = keyRange.right.getToken().isMinimum(); // max token
PrimaryKey left = indexContext.keyFactory().create(keyRange.left.getToken()); // lower bound
PrimaryKey right = isMaxToken ? null : indexContext.keyFactory().create(keyRange.right.getToken()); // upper bound
PrimaryKey left = index.keyFactory().create(keyRange.left.getToken()); // lower bound
PrimaryKey right = isMaxToken ? null : index.keyFactory().create(keyRange.right.getToken()); // upper bound
Set<PrimaryKey> resultKeys = isMaxToken ? primaryKeys.tailSet(left, leftInclusive) : primaryKeys.subSet(left, leftInclusive, right, rightInclusive);
if (!vectorQueryContext.getShadowedPrimaryKeys().isEmpty())
@ -220,8 +220,8 @@ public class VectorMemoryIndex extends MemoryIndex
return new KeyRangeListIterator(minimumKey, maximumKey, results);
}
ByteBuffer buffer = expression.lower.value.raw;
float[] qv = TypeUtil.decomposeVector(indexContext, buffer);
ByteBuffer buffer = expression.lower().value.raw;
float[] qv = index.termType().decomposeVector(buffer);
var bits = new KeyFilteringBits(results);
var keyQueue = graph.search(qv, limit, bits);
if (keyQueue.isEmpty())
@ -232,7 +232,7 @@ public class VectorMemoryIndex extends MemoryIndex
private int maxBruteForceRows(int limit, int nPermittedOrdinals, int graphSize)
{
int expectedNodesVisited = expectedNodesVisited(limit, nPermittedOrdinals, graphSize);
int expectedComparisons = indexContext.getIndexWriterConfig().getMaximumNodeConnections() * expectedNodesVisited;
int expectedComparisons = index.indexWriterConfig().getMaximumNodeConnections() * expectedNodesVisited;
// in-memory comparisons are cheaper than pulling a row off disk and then comparing
// VSTODO this is dramatically oversimplified
// larger dimension should increase this, because comparisons are more expensive
@ -266,10 +266,10 @@ public class VectorMemoryIndex extends MemoryIndex
}
public SegmentMetadata.ComponentMetadataMap writeDirect(IndexDescriptor indexDescriptor,
IndexContext indexContext,
IndexIdentifier indexIdentifier,
Function<PrimaryKey, Integer> postingTransformer) throws IOException
{
return graph.writeData(indexDescriptor, indexContext, postingTransformer);
return graph.writeData(indexDescriptor, indexIdentifier, postingTransformer);
}
@Override

6
src/java/org/apache/cassandra/index/sai/metrics/AbstractMetrics.java
View File

@ -20,6 +20,7 @@ package org.apache.cassandra.index.sai.metrics;
import java.util.ArrayList;
import java.util.List;
import org.apache.cassandra.index.sai.utils.IndexIdentifier;
import org.apache.cassandra.metrics.CassandraMetricsRegistry;
import org.apache.cassandra.metrics.DefaultNameFactory;
@ -35,6 +36,11 @@ public abstract class AbstractMetrics
private final String scope;
protected final List<CassandraMetricsRegistry.MetricName> tracked = new ArrayList<>();
AbstractMetrics(IndexIdentifier indexIdentifier, String scope)
{
this(indexIdentifier.keyspaceName, indexIdentifier.tableName, indexIdentifier.indexName, scope);
}
AbstractMetrics(String keyspace, String table, String scope)
{
this(keyspace, table, null, scope);

14
src/java/org/apache/cassandra/index/sai/metrics/ColumnQueryMetrics.java
View File

@ -21,15 +21,15 @@ import java.util.concurrent.TimeUnit;
import com.codahale.metrics.Meter;
import com.codahale.metrics.Timer;
import org.apache.cassandra.index.sai.IndexContext;
import org.apache.cassandra.index.sai.utils.IndexIdentifier;
import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics;
public abstract class ColumnQueryMetrics extends AbstractMetrics
{
protected ColumnQueryMetrics(IndexContext indexContext)
protected ColumnQueryMetrics(IndexIdentifier indexIdentifier)
{
super(indexContext.getKeyspace(), indexContext.getTable(), indexContext.getIndexName(), "ColumnQueryMetrics");
super(indexIdentifier, "ColumnQueryMetrics");
}
public static class TrieIndexMetrics extends ColumnQueryMetrics implements QueryEventListener.TrieIndexEventListener
@ -43,9 +43,9 @@ public abstract class ColumnQueryMetrics extends AbstractMetrics
private final QueryEventListener.PostingListEventListener postingsListener;
public TrieIndexMetrics(IndexContext indexContext)
public TrieIndexMetrics(IndexIdentifier indexIdentifier)
{
super(indexContext);
super(indexIdentifier);
termsTraversalTotalTime = Metrics.timer(createMetricName("TermsLookupLatency"));
@ -83,9 +83,9 @@ public abstract class ColumnQueryMetrics extends AbstractMetrics
private final QueryEventListener.PostingListEventListener postingsListener;
public BalancedTreeIndexMetrics(IndexContext indexContext)
public BalancedTreeIndexMetrics(IndexIdentifier indexIdentifier)
{
super(indexContext);
super(indexIdentifier);
intersectionLatency = Metrics.timer(createMetricName("BalancedTreeIntersectionLatency"));
intersectionEarlyExits = Metrics.meter(createMetricName("BalancedTreeIntersectionEarlyExits"));

13
src/java/org/apache/cassandra/index/sai/metrics/IndexMetrics.java
View File

@ -21,7 +21,7 @@ import com.codahale.metrics.Counter;
import com.codahale.metrics.Gauge;
import com.codahale.metrics.Histogram;
import com.codahale.metrics.Timer;
import org.apache.cassandra.index.sai.IndexContext;
import org.apache.cassandra.index.sai.StorageAttachedIndex;
import org.apache.cassandra.index.sai.memory.MemtableIndexManager;
import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics;
@ -40,11 +40,10 @@ public class IndexMetrics extends AbstractMetrics
public final Histogram compactionSegmentCellsPerSecond;
public final Histogram compactionSegmentBytesPerSecond;
public IndexMetrics(IndexContext indexContext)
public IndexMetrics(StorageAttachedIndex index, MemtableIndexManager memtableIndexManager)
{
super(indexContext.getKeyspace(), indexContext.getTable(), indexContext.getIndexName(), "IndexMetrics");
super(index.identifier(), "IndexMetrics");
MemtableIndexManager memtableIndexManager = indexContext.getMemtableIndexManager();
memtableIndexWriteLatency = Metrics.timer(createMetricName("MemtableIndexWriteLatency"));
compactionSegmentCellsPerSecond = Metrics.histogram(createMetricName("CompactionSegmentCellsPerSecond"), false);
compactionSegmentBytesPerSecond = Metrics.histogram(createMetricName("CompactionSegmentBytesPerSecond"), false);
@ -54,10 +53,10 @@ public class IndexMetrics extends AbstractMetrics
compactionCount = Metrics.counter(createMetricName("CompactionCount"));
memtableIndexFlushErrors = Metrics.counter(createMetricName("MemtableIndexFlushErrors"));
segmentFlushErrors = Metrics.counter(createMetricName("CompactionSegmentFlushErrors"));
Metrics.register(createMetricName("SSTableCellCount"), (Gauge<Long>) indexContext::getCellCount);
Metrics.register(createMetricName("SSTableCellCount"), (Gauge<Long>) index::cellCount);
Metrics.register(createMetricName("LiveMemtableIndexWriteCount"), (Gauge<Long>) memtableIndexManager::liveMemtableWriteCount);
Metrics.register(createMetricName("MemtableIndexBytes"), (Gauge<Long>) memtableIndexManager::estimatedMemIndexMemoryUsed);
Metrics.register(createMetricName("DiskUsedBytes"), (Gauge<Long>) indexContext::diskUsage);
Metrics.register(createMetricName("IndexFileCacheBytes"), (Gauge<Long>) indexContext::indexFileCacheSize);
Metrics.register(createMetricName("DiskUsedBytes"), (Gauge<Long>) index::diskUsage);
Metrics.register(createMetricName("IndexFileCacheBytes"), (Gauge<Long>) index::indexFileCacheSize);
}
}

270
src/java/org/apache/cassandra/index/sai/plan/Expression.java
View File

@ -26,14 +26,54 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.cql3.Operator;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.index.sai.IndexContext;
import org.apache.cassandra.index.sai.StorageAttachedIndex;
import org.apache.cassandra.index.sai.analyzer.AbstractAnalyzer;
import org.apache.cassandra.index.sai.utils.TypeUtil;
import org.apache.cassandra.index.sai.utils.IndexTermType;
public class Expression
/**
* An {@link Expression} is an internal representation of an index query operation. They are built from
* CQL {@link Operator} and {@link ByteBuffer} value pairs for a single column.
* <p>
* Each {@link Expression} consists of an {@link IndexOperator} and optional lower and upper {@link Bound}s.
* <p>
* The {@link IndexedExpression} has a backing {@link StorageAttachedIndex} for the index query but order to support
* CQL expressions on columns that do not have indexes or use operators that are not supported by the index there is
* an {@link UnindexedExpression} that does not provide a {@link StorageAttachedIndex} and can only be used for
* post-filtering
*/
public abstract class Expression
{
private static final Logger logger = LoggerFactory.getLogger(Expression.class);
Logger logger = LoggerFactory.getLogger(Expression.class);
private final IndexTermType indexTermType;
protected IndexOperator operator;
public Bound lower, upper;
// The upperInclusive and lowerInclusive flags are maintained separately to the inclusive flags
// in the upper and lower bounds because the upper and lower bounds have their inclusivity relaxed
// if the datatype being filtered is rounded in the index. These flags are used in the post-filtering
// process to remove values equal to the bounds.
public boolean upperInclusive, lowerInclusive;
Expression(IndexTermType indexTermType)
{
this.indexTermType = indexTermType;
}
public static Expression create(StorageAttachedIndex index)
{
return new IndexedExpression(index);
}
public static Expression create(IndexTermType indexTermType)
{
return new UnindexedExpression(indexTermType);
}
public static boolean supportsOperator(Operator operator)
{
return IndexOperator.valueOf(operator) != null;
}
public enum IndexOperator
{
@ -77,25 +117,32 @@ public class Expression
}
}
public final AbstractAnalyzer.AnalyzerFactory analyzerFactory;
public abstract boolean isNotIndexed();
public final IndexContext context;
public final AbstractType<?> validator;
public abstract StorageAttachedIndex getIndex();
protected IndexOperator operator;
abstract boolean hasAnalyzer();
public Bound lower, upper;
// The upperInclusive and lowerInclusive flags are maintained separately to the inclusive flags
// in the upper and lower bounds because the upper and lower bounds have their inclusivity relaxed
// if the datatype being filtered is rounded in the index. These flags are used in the post-filtering
// process to remove values equal to the bounds.
public boolean upperInclusive, lowerInclusive;
abstract AbstractAnalyzer getAnalyzer();
public Expression(IndexContext indexContext)
public IndexOperator getIndexOperator()
{
this.context = indexContext;
this.analyzerFactory = indexContext.getAnalyzerFactory();
this.validator = indexContext.getValidator();
return operator;
}
public IndexTermType getIndexTermType()
{
return indexTermType;
}
public Bound lower()
{
return lower;
}
public Bound upper()
{
return upper;
}
/**
@ -113,19 +160,19 @@ public class Expression
// range search is always inclusive, otherwise we run the risk of
// missing values that are within the exclusive range but are rejected
// because their rounded value is the same as the value being queried.
lowerInclusive = upperInclusive = TypeUtil.supportsRounding(validator);
lowerInclusive = upperInclusive = indexTermType.supportsRounding();
switch (op)
{
case EQ:
case CONTAINS:
case CONTAINS_KEY:
lower = new Bound(value, validator, true);
lower = new Bound(value, indexTermType, true);
upper = lower;
operator = IndexOperator.valueOf(op);
break;
case LTE:
if (context.getDefinition().isReversedType())
if (indexTermType.isReversed())
{
this.lowerInclusive = true;
lowerInclusive = true;
@ -137,14 +184,14 @@ public class Expression
}
case LT:
operator = IndexOperator.RANGE;
if (context.getDefinition().isReversedType())
lower = new Bound(value, validator, lowerInclusive);
if (indexTermType.isReversed())
lower = new Bound(value, indexTermType, lowerInclusive);
else
upper = new Bound(value, validator, upperInclusive);
upper = new Bound(value, indexTermType, upperInclusive);
break;
case GTE:
if (context.getDefinition().isReversedType())
if (indexTermType.isReversed())
{
this.upperInclusive = true;
upperInclusive = true;
@ -156,20 +203,20 @@ public class Expression
}
case GT:
operator = IndexOperator.RANGE;
if (context.getDefinition().isReversedType())
upper = new Bound(value, validator, upperInclusive);
if (indexTermType.isReversed())
upper = new Bound(value, indexTermType, upperInclusive);
else
lower = new Bound(value, validator, lowerInclusive);
lower = new Bound(value, indexTermType, lowerInclusive);
break;
case ANN:
operator = IndexOperator.ANN;
lower = new Bound(value, validator, true);
lower = new Bound(value, indexTermType, true);
upper = lower;
break;
default:
throw new IllegalArgumentException("Index does not support the " + op + " operator");
}
assert operator != null;
return this;
}
@ -180,26 +227,26 @@ public class Expression
{
// If the expression represents an ANN ordering then we return true because the actual result
// is approximate and will rarely / never match the expression value
if (validator.isVector())
if (indexTermType.isVector())
return true;
if (!TypeUtil.isValid(columnValue, validator))
if (!indexTermType.isValid(columnValue))
{
logger.error(context.logMessage("Value is not valid for indexed column {} with {}"), context.getColumnName(), validator);
logger.error("Value is not valid for indexed column {} with {}", indexTermType.columnName(), indexTermType.indexType());
return false;
}
Value value = new Value(columnValue, validator);
Value value = new Value(columnValue, indexTermType);
if (lower != null)
{
// suffix check
if (TypeUtil.isLiteral(validator))
if (indexTermType.isLiteral())
return validateStringValue(value.raw, lower.value.raw);
else
{
// range or (not-)equals - (mainly) for numeric values
int cmp = TypeUtil.comparePostFilter(lower.value, value, validator);
int cmp = indexTermType.comparePostFilter(lower.value, value);
// in case of EQ lower == upper
if (operator == IndexOperator.EQ || operator == IndexOperator.CONTAINS_KEY || operator == IndexOperator.CONTAINS_VALUE)
@ -213,12 +260,12 @@ public class Expression
if (upper != null && lower != upper)
{
// string (prefix or suffix) check
if (TypeUtil.isLiteral(validator))
if (indexTermType.isLiteral())
return validateStringValue(value.raw, upper.value.raw);
else
{
// range - mainly for numeric values
int cmp = TypeUtil.comparePostFilter(upper.value, value, validator);
int cmp = indexTermType.comparePostFilter(upper.value, value);
return (cmp > 0 || (cmp == 0 && upperInclusive));
}
}
@ -228,41 +275,45 @@ public class Expression
private boolean validateStringValue(ByteBuffer columnValue, ByteBuffer requestedValue)
{
AbstractAnalyzer analyzer = analyzerFactory.create();
analyzer.reset(columnValue.duplicate());
try
if (hasAnalyzer())
{
while (analyzer.hasNext())
AbstractAnalyzer analyzer = getAnalyzer();
analyzer.reset(columnValue.duplicate());
try
{
final ByteBuffer term = analyzer.next();
boolean isMatch = false;
switch (operator)
while (analyzer.hasNext())
{
case EQ:
case CONTAINS_KEY:
case CONTAINS_VALUE:
isMatch = validator.compare(term, requestedValue) == 0;
break;
case RANGE:
isMatch = isLowerSatisfiedBy(term) && isUpperSatisfiedBy(term);
break;
if (termMatches(analyzer.next(), requestedValue))
return true;
}
if (isMatch)
return true;
return false;
}
finally
{
analyzer.end();
}
return false;
}
finally
else
{
analyzer.end();
return termMatches(columnValue, requestedValue);
}
}
public IndexOperator getOp()
private boolean termMatches(ByteBuffer term, ByteBuffer requestedValue)
{
return operator;
boolean isMatch = false;
switch (operator)
{
case EQ:
case CONTAINS_KEY:
case CONTAINS_VALUE:
isMatch = indexTermType.compare(term, requestedValue) == 0;
break;
case RANGE:
isMatch = isLowerSatisfiedBy(term) && isUpperSatisfiedBy(term);
break;
}
return isMatch;
}
private boolean hasLower()
@ -280,7 +331,7 @@ public class Expression
if (!hasLower())
return true;
int cmp = validator.compare(value, lower.value.raw);
int cmp = indexTermType.indexType().compare(value, lower.value.raw);
return cmp > 0 || cmp == 0 && lower.inclusive;
}
@ -289,7 +340,7 @@ public class Expression
if (!hasUpper())
return true;
int cmp = validator.compare(value, upper.value.raw);
int cmp = indexTermType.indexType().compare(value, upper.value.raw);
return cmp < 0 || cmp == 0 && upper.inclusive;
}
@ -297,20 +348,19 @@ public class Expression
public String toString()
{
return String.format("Expression{name: %s, op: %s, lower: (%s, %s), upper: (%s, %s)}",
context.getColumnName(),
indexTermType.columnName(),
operator,
lower == null ? "null" : validator.getString(lower.value.raw),
lower == null ? "null" : indexTermType.asString(lower.value.raw),
lower != null && lower.inclusive,
upper == null ? "null" : validator.getString(upper.value.raw),
upper == null ? "null" : indexTermType.asString(upper.value.raw),
upper != null && upper.inclusive);
}
@Override
public int hashCode()
{
return new HashCodeBuilder().append(context.getColumnName())
return new HashCodeBuilder().append(indexTermType)
.append(operator)
.append(validator)
.append(lower).append(upper).build();
}
@ -325,13 +375,79 @@ public class Expression
Expression o = (Expression) other;
return Objects.equals(context.getColumnName(), o.context.getColumnName())
&& validator.equals(o.validator)
return Objects.equals(indexTermType, o.indexTermType)
&& operator == o.operator
&& Objects.equals(lower, o.lower)
&& Objects.equals(upper, o.upper);
}
public static class IndexedExpression extends Expression
{
private final StorageAttachedIndex index;
public IndexedExpression(StorageAttachedIndex index)
{
super(index.termType());
this.index = index;
}
@Override
public boolean isNotIndexed()
{
return false;
}
@Override
public StorageAttachedIndex getIndex()
{
return index;
}
@Override
boolean hasAnalyzer()
{
return index.hasAnalyzer();
}
@Override
AbstractAnalyzer getAnalyzer()
{
return index.analyzer();
}
}
public static class UnindexedExpression extends Expression
{
private UnindexedExpression(IndexTermType indexTermType)
{
super(indexTermType);
}
@Override
public boolean isNotIndexed()
{
return true;
}
@Override
public StorageAttachedIndex getIndex()
{
throw new UnsupportedOperationException();
}
@Override
boolean hasAnalyzer()
{
return false;
}
@Override
AbstractAnalyzer getAnalyzer()
{
throw new UnsupportedOperationException();
}
}
/**
* A representation of a column value in its raw and encoded form.
*/
@ -340,10 +456,10 @@ public class Expression
public final ByteBuffer raw;
public final ByteBuffer encoded;
public Value(ByteBuffer value, AbstractType<?> type)
public Value(ByteBuffer value, IndexTermType indexTermType)
{
this.raw = value;
this.encoded = TypeUtil.asIndexBytes(value, type);
this.encoded = indexTermType.asIndexBytes(value);
}
@Override
@ -371,9 +487,9 @@ public class Expression
public final Value value;
public final boolean inclusive;
public Bound(ByteBuffer value, AbstractType<?> type, boolean inclusive)
public Bound(ByteBuffer value, IndexTermType indexTermType, boolean inclusive)
{
this.value = new Value(value, type);
this.value = new Value(value, indexTermType);
this.inclusive = inclusive;
}

12
src/java/org/apache/cassandra/index/sai/plan/FilterTree.java
View File

@ -28,7 +28,6 @@ import com.google.common.collect.ListMultimap;
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.db.rows.Row;
import org.apache.cassandra.db.rows.Unfiltered;
import org.apache.cassandra.index.sai.utils.TypeUtil;
import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.schema.ColumnMetadata.Kind;
import org.apache.cassandra.utils.FBUtilities;
@ -37,7 +36,7 @@ import static org.apache.cassandra.index.sai.plan.Operation.BooleanOperator;
/**
* Tree-like structure to filter base table data using indexed expressions and non-user-defined filters.
*
* <p>
* This is needed because:
* 1. SAI doesn't index tombstones, base data may have been shadowed.
* 2. Replica filter protecting may fetch data that doesn't match index expressions.
@ -48,8 +47,7 @@ public class FilterTree
protected final ListMultimap<ColumnMetadata, Expression> expressions;
protected final List<FilterTree> children = new ArrayList<>();
FilterTree(BooleanOperator operation,
ListMultimap<ColumnMetadata, Expression> expressions)
FilterTree(BooleanOperator operation, ListMultimap<ColumnMetadata, Expression> expressions)
{
this.op = operation;
this.expressions = expressions;
@ -95,14 +93,14 @@ public class FilterTree
{
Expression filter = filterIterator.previous();
if (TypeUtil.isNonFrozenCollection(column.type))
if (filter.getIndexTermType().isNonFrozenCollection())
{
Iterator<ByteBuffer> valueIterator = filter.context.getValuesOf(row, now);
Iterator<ByteBuffer> valueIterator = filter.getIndexTermType().valuesOf(row, now);
result = op.apply(result, collectionMatch(valueIterator, filter));
}
else
{
ByteBuffer value = filter.context.getValueOf(key, row, now);
ByteBuffer value = filter.getIndexTermType().valueOf(key, row, now);
result = op.apply(result, singletonMatch(value, filter));
}

160
src/java/org/apache/cassandra/index/sai/plan/Operation.java
View File

@ -31,11 +31,15 @@ import com.google.common.collect.Iterables;
import com.google.common.collect.ListMultimap;
import org.apache.cassandra.cql3.Operator;
import org.apache.cassandra.cql3.statements.schema.IndexTarget;
import org.apache.cassandra.db.filter.RowFilter;
import org.apache.cassandra.index.sai.IndexContext;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.db.marshal.CollectionType;
import org.apache.cassandra.exceptions.InvalidRequestException;
import org.apache.cassandra.index.sai.StorageAttachedIndex;
import org.apache.cassandra.index.sai.analyzer.AbstractAnalyzer;
import org.apache.cassandra.index.sai.iterators.KeyRangeIterator;
import org.apache.cassandra.index.sai.utils.TypeUtil;
import org.apache.cassandra.index.sai.utils.IndexTermType;
import org.apache.cassandra.schema.ColumnMetadata;
public class Operation
@ -58,8 +62,7 @@ public class Operation
}
@VisibleForTesting
protected static ListMultimap<ColumnMetadata, Expression> buildIndexExpressions(QueryController controller,
BooleanOperator booleanOperator,
protected static ListMultimap<ColumnMetadata, Expression> buildIndexExpressions(QueryController queryController,
List<RowFilter.Expression> expressions)
{
ListMultimap<ColumnMetadata, Expression> analyzed = ArrayListMultimap.create();
@ -72,40 +75,66 @@ public class Operation
return cmp == 0 ? -Integer.compare(getPriority(a.operator()), getPriority(b.operator())) : cmp;
});
for (final RowFilter.Expression e : expressions)
for (final RowFilter.Expression expression : expressions)
{
IndexContext indexContext = controller.getContext(e);
List<Expression> perColumn = analyzed.get(e.column());
if (Expression.supportsOperator(expression.operator()))
{
StorageAttachedIndex index = queryController.indexFor(expression);
AbstractAnalyzer analyzer = indexContext.getAnalyzerFactory().create();
List<Expression> perColumn = analyzed.get(expression.column());
if (index == null)
buildUnindexedExpression(queryController, expression, perColumn);
else
buildIndexedExpression(index, expression, perColumn);
}
}
return analyzed;
}
private static void buildUnindexedExpression(QueryController queryController,
RowFilter.Expression expression,
List<Expression> perColumn)
{
IndexTermType indexTermType = IndexTermType.create(expression.column(),
queryController.metadata().partitionKeyColumns(),
determineIndexTargetType(expression));
if (indexTermType.isMultiExpression(expression))
{
perColumn.add(Expression.create(indexTermType).add(expression.operator(), expression.getIndexValue().duplicate()));
}
else
{
Expression range;
if (perColumn.size() == 0)
{
range = Expression.create(indexTermType);
perColumn.add(range);
}
else
{
range = Iterables.getLast(perColumn);
}
range.add(expression.operator(), expression.getIndexValue().duplicate());
}
}
private static void buildIndexedExpression(StorageAttachedIndex index, RowFilter.Expression expression, List<Expression> perColumn)
{
if (index.hasAnalyzer())
{
AbstractAnalyzer analyzer = index.analyzer();
try
{
analyzer.reset(e.getIndexValue().duplicate());
analyzer.reset(expression.getIndexValue().duplicate());
// EQ can have multiple expressions e.g. text = "Hello World",
// becomes text = "Hello" OR text = "World" because "space" is always interpreted as a split point (by analyzer),
// CONTAINS/CONTAINS_KEY are always treated as multiple expressions since they currently only targetting
// collections.
boolean isMultiExpression = false;
switch (e.operator())
{
case EQ:
// EQ operator will always be a multiple expression because it is being used by
// map entries
isMultiExpression = indexContext.isNonFrozenCollection();
break;
case CONTAINS:
case CONTAINS_KEY:
isMultiExpression = true;
break;
}
if (isMultiExpression)
if (index.termType().isMultiExpression(expression))
{
while (analyzer.hasNext())
{
final ByteBuffer token = analyzer.next();
perColumn.add(new Expression(indexContext).add(e.operator(), token.duplicate()));
perColumn.add(Expression.create(index).add(expression.operator(), token.duplicate()));
}
}
else
@ -114,9 +143,9 @@ public class Operation
// not-equals is combined with the range iff operator is AND.
{
Expression range;
if (perColumn.size() == 0 || booleanOperator != BooleanOperator.AND)
if (perColumn.size() == 0)
{
range = new Expression(indexContext);
range = Expression.create(index);
perColumn.add(range);
}
else
@ -124,18 +153,18 @@ public class Operation
range = Iterables.getLast(perColumn);
}
if (!TypeUtil.isLiteral(indexContext.getValidator()))
{
range.add(e.operator(), e.getIndexValue().duplicate());
}
else
if (index.termType().isLiteral())
{
while (analyzer.hasNext())
{
ByteBuffer term = analyzer.next();
range.add(e.operator(), term.duplicate());
range.add(expression.operator(), term.duplicate());
}
}
else
{
range.add(expression.operator(), expression.getIndexValue().duplicate());
}
}
}
finally
@ -143,8 +172,59 @@ public class Operation
analyzer.end();
}
}
else
{
if (index.termType().isMultiExpression(expression))
{
perColumn.add(Expression.create(index).add(expression.operator(), expression.getIndexValue().duplicate()));
}
else
{
Expression range;
if (perColumn.size() == 0)
{
range = Expression.create(index);
perColumn.add(range);
}
else
{
range = Iterables.getLast(perColumn);
}
range.add(expression.operator(), expression.getIndexValue().duplicate());
}
}
}
return analyzed;
/**
* Determines the {@link IndexTarget.Type} for the expression. In this case we are only interested in map types and
* the operator being used in the expression.
*/
private static IndexTarget.Type determineIndexTargetType(RowFilter.Expression expression)
{
AbstractType<?> type = expression.column().type;
IndexTarget.Type indexTargetType = IndexTarget.Type.SIMPLE;
if (type.isCollection() && type.isMultiCell())
{
CollectionType<?> collection = ((CollectionType<?>) type);
if (collection.kind == CollectionType.Kind.MAP)
{
switch (expression.operator())
{
case EQ:
indexTargetType = IndexTarget.Type.KEYS_AND_VALUES;
break;
case CONTAINS:
indexTargetType = IndexTarget.Type.VALUES;
break;
case CONTAINS_KEY:
indexTargetType = IndexTarget.Type.KEYS;
break;
default:
throw new InvalidRequestException("Invalid operator");
}
}
}
return indexTargetType;
}
private static int getPriority(Operator op)
@ -300,7 +380,7 @@ public class Operation
@Override
public void analyze(List<RowFilter.Expression> expressionList, QueryController controller)
{
expressionMap = buildIndexExpressions(controller, BooleanOperator.AND, expressionList);
expressionMap = buildIndexExpressions(controller, expressionList);
}
@Override
@ -330,7 +410,7 @@ public class Operation
@Override
public void analyze(List<RowFilter.Expression> expressionList, QueryController controller)
{
expressionMap = buildIndexExpressions(controller, BooleanOperator.AND, expressionList);
expressionMap = buildIndexExpressions(controller, expressionList);
}
@Override

43
src/java/org/apache/cassandra/index/sai/plan/QueryController.java
View File

@ -29,7 +29,6 @@ import java.util.stream.Collectors;
import com.google.common.collect.Lists;
import org.apache.cassandra.cql3.Operator;
import org.apache.cassandra.cql3.statements.schema.IndexTarget;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.DataRange;
import org.apache.cassandra.db.DecoratedKey;
@ -45,7 +44,6 @@ import org.apache.cassandra.db.filter.RowFilter;
import org.apache.cassandra.db.rows.UnfilteredRowIterator;
import org.apache.cassandra.dht.AbstractBounds;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.index.sai.IndexContext;
import org.apache.cassandra.index.sai.QueryContext;
import org.apache.cassandra.index.sai.StorageAttachedIndex;
import org.apache.cassandra.index.sai.VectorQueryContext;
@ -133,24 +131,16 @@ public class QueryController
return ranges;
}
/**
* @return indexed {@code IndexContext} if index is found; otherwise return non-indexed {@code IndexContext}.
*/
public IndexContext getContext(RowFilter.Expression expression)
public StorageAttachedIndex indexFor(RowFilter.Expression expression)
{
Set<StorageAttachedIndex> indexes = cfs.indexManager.getBestIndexFor(expression, StorageAttachedIndex.class);
return indexes.isEmpty() ? null : indexes.iterator().next();
}
return indexes.isEmpty() ? new IndexContext(cfs.getKeyspaceName(),
cfs.getTableName(),
cfs.metadata().partitionKeyType,
cfs.getPartitioner(),
cfs.getComparator(),
expression.column(),
expression.operator().isContainsKey()
? IndexTarget.Type.KEYS
: IndexTarget.Type.VALUES,
null)
: indexes.iterator().next().getIndexContext();
public boolean hasAnalyzer(RowFilter.Expression expression)
{
StorageAttachedIndex index = indexFor(expression);
return index != null && index.hasAnalyzer();
}
public UnfilteredRowIterator queryStorage(PrimaryKey key, ReadExecutionController executionController)
@ -193,7 +183,7 @@ public class QueryController
public KeyRangeIterator.Builder getIndexQueryResults(Collection<Expression> expressions)
{
// VSTODO move ANN out of expressions and into its own abstraction? That will help get generic ORDER BY support
expressions = expressions.stream().filter(e -> e.getOp() != Expression.IndexOperator.ANN).collect(Collectors.toList());
expressions = expressions.stream().filter(e -> e.getIndexOperator() != Expression.IndexOperator.ANN).collect(Collectors.toList());
KeyRangeIterator.Builder builder = KeyRangeIntersectionIterator.builder(expressions.size());
@ -250,10 +240,11 @@ public class QueryController
public KeyRangeIterator getTopKRows(RowFilter.Expression expression)
{
assert expression.operator() == Operator.ANN;
var planExpression = new Expression(getContext(expression))
.add(Operator.ANN, expression.getIndexValue().duplicate());
StorageAttachedIndex index = indexFor(expression);
assert index != null;
var planExpression = Expression.create(index).add(Operator.ANN, expression.getIndexValue().duplicate());
// search memtable before referencing sstable indexes; otherwise we may miss newly flushed memtable index
KeyRangeIterator memtableResults = getContext(expression).getMemtableIndexManager().searchMemtableIndexes(queryContext, planExpression, mergeRange);
KeyRangeIterator memtableResults = index.memtableIndexManager().searchMemtableIndexes(queryContext, planExpression, mergeRange);
QueryViewBuilder.QueryView queryView = new QueryViewBuilder(Collections.singleton(planExpression), mergeRange).build();
@ -286,11 +277,13 @@ public class QueryController
// eagerly can save some work when going from PK to row id for on disk segments.
// Since the result is shared with multiple streams, we use an unmodifiable list.
var sourceKeys = rawSourceKeys.stream().filter(vectorQueryContext::shouldInclude).collect(Collectors.toList());
var planExpression = new Expression(this.getContext(expression));
StorageAttachedIndex index = indexFor(expression);
assert index != null : "Cannot do ANN ordering on an unindexed column";
var planExpression = Expression.create(index);
planExpression.add(Operator.ANN, expression.getIndexValue().duplicate());
// search memtable before referencing sstable indexes; otherwise we may miss newly flushed memtable index
KeyRangeIterator memtableResults = this.getContext(expression).getMemtableIndexManager().limitToTopResults(queryContext, sourceKeys, planExpression);
KeyRangeIterator memtableResults = index.memtableIndexManager().limitToTopResults(queryContext, sourceKeys, planExpression);
QueryViewBuilder.QueryView queryView = new QueryViewBuilder(Collections.singleton(planExpression), mergeRange).build();
try
@ -298,10 +291,10 @@ public class QueryController
List<KeyRangeIterator> sstableIntersections = queryView.view
.stream()
.flatMap(pair -> pair.right.stream())
.map(index -> {
.map(idx -> {
try
{
return index.limitToTopKResults(queryContext, sourceKeys, planExpression);
return idx.limitToTopKResults(queryContext, sourceKeys, planExpression);
}
catch (IOException e)
{

8
src/java/org/apache/cassandra/index/sai/plan/QueryViewBuilder.java
View File

@ -102,7 +102,7 @@ public class QueryViewBuilder
{
// Non-index column query should only act as FILTER BY for satisfiedBy(Row) method
// because otherwise it likely to go through the whole index.
if (expression.context.isNotIndexed())
if (expression.isNotIndexed())
continue;
// If we didn't get a most selective expression then none of the
@ -126,7 +126,7 @@ public class QueryViewBuilder
// Finally, we select all the sstable indexes for this expression that
// have overlapping keys with the sstable indexes of the most selective
// and have a term range that is satisfied by the expression.
View view = expression.context.getView();
View view = expression.getIndex().view();
Set<SSTableIndex> indexes = new TreeSet<>(SSTableIndex.COMPARATOR);
indexes.addAll(view.match(expression)
.stream()
@ -157,10 +157,10 @@ public class QueryViewBuilder
for (Expression expression : expressions)
{
if (expression.context.isNotIndexed())
if (expression.isNotIndexed())
continue;
View view = expression.context.getView();
View view = expression.getIndex().view();
NavigableSet<SSTableIndex> indexes = new TreeSet<>(SSTableIndex.COMPARATOR);
indexes.addAll(selectIndexesInRange(view.match(expression)));

31
src/java/org/apache/cassandra/index/sai/plan/StorageAttachedIndexQueryPlan.java
View File

@ -55,7 +55,7 @@ public class StorageAttachedIndexQueryPlan implements Index.QueryPlan
this.postIndexFilter = postIndexFilter;
this.filterOperation = filterOperation;
this.indexes = indexes;
this.isTopK = indexes.stream().anyMatch(i -> i instanceof StorageAttachedIndex && ((StorageAttachedIndex) i).getIndexContext().isVector());
this.isTopK = indexes.stream().anyMatch(i -> i instanceof StorageAttachedIndex && ((StorageAttachedIndex) i).termType().isVector());
}
@Nullable
@ -66,12 +66,25 @@ public class StorageAttachedIndexQueryPlan implements Index.QueryPlan
{
ImmutableSet.Builder<Index> selectedIndexesBuilder = ImmutableSet.builder();
RowFilter preIndexFilter = rowFilter;
RowFilter postIndexFilter = rowFilter;
for (RowFilter.Expression expression : rowFilter)
{
// we ignore user-defined expressions here because we don't have a way to translate their #isSatifiedBy
// method, they will be included in the filter returned by QueryPlan#postIndexQueryFilter()
if (expression.isUserDefined())
// we ignore any expressions here (currently IN and user-defined expressions) where we don't have a way to
// translate their #isSatifiedBy method, they will be included in the filter returned by QueryPlan#postIndexQueryFilter()
//
// Note: For both the pre- and post-filters we need to check that the expression exists before removing it
// because the without method assert if the expression doesn't exist. This can be the case if we are given
// a duplicate expression - a = 1 and a = 1. The without method removes all instances of the expression.
if (expression.operator().isIN() || expression.isUserDefined())
{
if (preIndexFilter.getExpressions().contains(expression))
preIndexFilter = preIndexFilter.without(expression);
continue;
}
if (postIndexFilter.getExpressions().contains(expression))
postIndexFilter = postIndexFilter.without(expression);
for (StorageAttachedIndex index : indexes)
{
@ -86,13 +99,7 @@ public class StorageAttachedIndexQueryPlan implements Index.QueryPlan
if (selectedIndexes.isEmpty())
return null;
/*
* postIndexFilter comprised by those expressions in the read command row filter that can't be handled by
* {@link FilterTree#satisfiedBy(Unfiltered, Row, boolean)}. This includes expressions targeted
* at {@link RowFilter.UserExpression}s.
*/
RowFilter postIndexFilter = rowFilter.restrict(RowFilter.Expression::isUserDefined);
return new StorageAttachedIndexQueryPlan(cfs, queryMetrics, postIndexFilter, rowFilter, selectedIndexes);
return new StorageAttachedIndexQueryPlan(cfs, queryMetrics, postIndexFilter, preIndexFilter, selectedIndexes);
}
@Override
@ -141,7 +148,7 @@ public class StorageAttachedIndexQueryPlan implements Index.QueryPlan
/**
* @return a filter with all the expressions that are user-defined or for a non-indexed partition key column
*
* <p>
* (currently index on partition columns is not supported, see {@link StorageAttachedIndex#validateOptions(Map, TableMetadata)})
*/
@Override

13
src/java/org/apache/cassandra/index/sai/plan/StorageAttachedIndexSearcher.java
View File

@ -48,7 +48,6 @@ import org.apache.cassandra.dht.Token;
import org.apache.cassandra.exceptions.RequestTimeoutException;
import org.apache.cassandra.index.Index;
import org.apache.cassandra.index.sai.QueryContext;
import org.apache.cassandra.index.sai.analyzer.AbstractAnalyzer;
import org.apache.cassandra.index.sai.metrics.TableQueryMetrics;
import org.apache.cassandra.index.sai.iterators.KeyRangeIterator;
import org.apache.cassandra.index.sai.utils.PrimaryKey;
@ -84,16 +83,8 @@ public class StorageAttachedIndexSearcher implements Index.Searcher
{
for (RowFilter.Expression expression : queryController.filterOperation())
{
AbstractAnalyzer analyzer = queryController.getContext(expression).getAnalyzerFactory().create();
try
{
if (analyzer.transformValue())
return applyIndexFilter(fullResponse, Operation.buildFilter(queryController), queryContext);
}
finally
{
analyzer.end();
}
if (queryController.hasAnalyzer(expression))
return applyIndexFilter(fullResponse, Operation.buildFilter(queryController), queryContext);
}
// if no analyzer does transformation

26
src/java/org/apache/cassandra/index/sai/plan/VectorTopKProcessor.java
View File

@ -43,12 +43,11 @@ import org.apache.cassandra.db.rows.Row;
import org.apache.cassandra.db.rows.Unfiltered;
import org.apache.cassandra.index.Index;
import org.apache.cassandra.index.SecondaryIndexManager;
import org.apache.cassandra.index.sai.IndexContext;
import org.apache.cassandra.index.sai.StorageAttachedIndex;
import org.apache.cassandra.index.sai.utils.InMemoryPartitionIterator;
import org.apache.cassandra.index.sai.utils.InMemoryUnfilteredPartitionIterator;
import org.apache.cassandra.index.sai.utils.IndexTermType;
import org.apache.cassandra.index.sai.utils.PartitionInfo;
import org.apache.cassandra.index.sai.utils.TypeUtil;
import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.Pair;
@ -65,7 +64,8 @@ import org.apache.cassandra.utils.Pair;
public class VectorTopKProcessor
{
private final ReadCommand command;
private final IndexContext indexContext;
private final StorageAttachedIndex index;
private final IndexTermType indexTermType;
private final float[] queryVector;
private final int limit;
@ -74,10 +74,11 @@ public class VectorTopKProcessor
{
this.command = command;
Pair<IndexContext, float[]> annIndexAndExpression = findTopKIndexContext();
Pair<StorageAttachedIndex, float[]> annIndexAndExpression = findTopKIndex();
Preconditions.checkNotNull(annIndexAndExpression);
this.indexContext = annIndexAndExpression.left;
this.index = annIndexAndExpression.left;
this.indexTermType = annIndexAndExpression.left().termType();
this.queryVector = annIndexAndExpression.right;
this.limit = command.limits().count();
}
@ -142,7 +143,7 @@ public class VectorTopKProcessor
*/
private float getScoreForRow(DecoratedKey key, Row row)
{
ColumnMetadata column = indexContext.getDefinition();
ColumnMetadata column = indexTermType.columnMetadata();
if (column.isPrimaryKeyColumn() && key == null)
return 0;
@ -153,17 +154,17 @@ public class VectorTopKProcessor
if ((column.isClusteringColumn() || column.isRegular()) && row.isStatic())
return 0;
ByteBuffer value = indexContext.getValueOf(key, row, FBUtilities.nowInSeconds());
ByteBuffer value = indexTermType.valueOf(key, row, FBUtilities.nowInSeconds());
if (value != null)
{
float[] vector = TypeUtil.decomposeVector(indexContext, value);
return indexContext.getIndexWriterConfig().getSimilarityFunction().compare(vector, queryVector);
float[] vector = indexTermType.decomposeVector(value);
return index.indexWriterConfig().getSimilarityFunction().compare(vector, queryVector);
}
return 0;
}
private Pair<IndexContext, float[]> findTopKIndexContext()
private Pair<StorageAttachedIndex, float[]> findTopKIndex()
{
ColumnFamilyStore cfs = Keyspace.openAndGetStore(command.metadata());
@ -172,9 +173,8 @@ public class VectorTopKProcessor
StorageAttachedIndex sai = findVectorIndexFor(cfs.indexManager, expression);
if (sai != null)
{
float[] qv = TypeUtil.decomposeVector(sai.getIndexContext(), expression.getIndexValue().duplicate());
return Pair.create(sai.getIndexContext(), qv);
float[] qv = sai.termType().decomposeVector(expression.getIndexValue().duplicate());
return Pair.create(sai, qv);
}
}

81
src/java/org/apache/cassandra/index/sai/utils/IndexIdentifier.java Normal file
View File

@ -0,0 +1,81 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.index.sai.utils;
import com.google.common.base.Objects;
/**
* This is a simple wrapper around the index identity. Its primary purpose is to isolate classes that only need
* access to the identity from the main index classes. This is useful in testing but also makes it easier to pass
* the log message wrapper {@link #logMessage(String)} to classes that don't need any other information about the index.
*/
public class IndexIdentifier
{
public final String keyspaceName;
public final String tableName;
public final String indexName;
public IndexIdentifier(String keyspaceName, String tableName, String indexName)
{
this.keyspaceName = keyspaceName;
this.tableName = tableName;
this.indexName = indexName;
}
/**
* A helper method for constructing consistent log messages for specific column indexes.
* <p>
* Example: For the index "idx" in keyspace "ks" on table "tb", calling this method with the raw message
* "Flushing new index segment..." will produce...
* <p>
* "[ks.tb.idx] Flushing new index segment..."
*
* @param message The raw content of a logging message, without information identifying it with an index.
*
* @return A log message with the proper keyspace, table and index name prepended to it.
*/
public String logMessage(String message)
{
// Index names are unique only within a keyspace.
return String.format("[%s.%s.%s] %s", keyspaceName, tableName, indexName, message);
}
@Override
public String toString()
{
return String.format("%s.%s", keyspaceName, indexName);
}
@Override
public int hashCode()
{
return Objects.hashCode(keyspaceName, tableName, indexName);
}
@Override
public boolean equals(Object obj)
{
if (this == obj) return true;
if (obj == null || getClass() != obj.getClass()) return false;
IndexIdentifier other = (IndexIdentifier) obj;
return Objects.equal(keyspaceName, other.keyspaceName) &&
Objects.equal(tableName, other.tableName) &&
Objects.equal(indexName, other.indexName);
}
}

868
src/java/org/apache/cassandra/index/sai/utils/IndexTermType.java Normal file
View File

@ -0,0 +1,868 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.index.sai.utils;
import java.math.BigInteger;
import java.net.InetAddress;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.EnumSet;
import java.util.Iterator;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Stream;
import java.util.stream.StreamSupport;
import com.google.common.base.MoreObjects;
import com.google.common.collect.ImmutableSet;
import com.googlecode.concurrenttrees.radix.ConcurrentRadixTree;
import org.apache.cassandra.cql3.CQL3Type;
import org.apache.cassandra.cql3.Operator;
import org.apache.cassandra.cql3.statements.schema.IndexTarget;
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.db.filter.RowFilter;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.db.marshal.AsciiType;
import org.apache.cassandra.db.marshal.BooleanType;
import org.apache.cassandra.db.marshal.ByteBufferAccessor;
import org.apache.cassandra.db.marshal.CollectionType;
import org.apache.cassandra.db.marshal.CompositeType;
import org.apache.cassandra.db.marshal.DecimalType;
import org.apache.cassandra.db.marshal.InetAddressType;
import org.apache.cassandra.db.marshal.IntegerType;
import org.apache.cassandra.db.marshal.LongType;
import org.apache.cassandra.db.marshal.StringType;
import org.apache.cassandra.db.marshal.UTF8Type;
import org.apache.cassandra.db.marshal.UUIDType;
import org.apache.cassandra.db.marshal.VectorType;
import org.apache.cassandra.db.rows.Cell;
import org.apache.cassandra.db.rows.ComplexColumnData;
import org.apache.cassandra.db.rows.Row;
import org.apache.cassandra.index.sai.plan.Expression;
import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.serializers.MarshalException;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.FastByteOperations;
import org.apache.cassandra.utils.bytecomparable.ByteComparable;
import org.apache.cassandra.utils.bytecomparable.ByteSource;
import org.apache.cassandra.utils.bytecomparable.ByteSourceInverse;
/**
* This class is a representation of an {@link AbstractType} as an indexable type. It is responsible for determining the
* capabilities of the type and provides helper methods for handling term values associated with the type.
*/
public class IndexTermType
{
private static final Set<AbstractType<?>> EQ_ONLY_TYPES = ImmutableSet.of(UTF8Type.instance,
AsciiType.instance,
BooleanType.instance,
UUIDType.instance);
private static final byte[] IPV4_PREFIX = new byte[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, -1 };
/**
* DecimalType / BigDecimal values are indexed by truncating their asComparableBytes representation to this size,
* padding on the right with zero-value-bytes until this size is reached (if necessary). This causes
* false-positives that must be filtered in a separate step after hitting the index and reading the associated
* (full) values.
*/
private static final int DECIMAL_APPROXIMATION_BYTES = 24;
private static final int BIG_INTEGER_APPROXIMATION_BYTES = 20;
private static final int INET_ADDRESS_SIZE = 16;
private static final int DEFAULT_FIXED_LENGTH = 16;
private enum Capability
{
STRING,
VECTOR,
INET_ADDRESS,
BIG_INTEGER,
BIG_DECIMAL,
LONG,
BOOLEAN,
LITERAL,
REVERSED,
FROZEN,
COLLECTION,
NON_FROZEN_COLLECTION,
COMPOSITE,
COMPOSITE_PARTITION
}
private final ColumnMetadata columnMetadata;
private final IndexTarget.Type indexTargetType;
private final AbstractType<?> indexType;
private final List<IndexTermType> subTypes;
private final AbstractType<?> vectorElementType;
private final int vectorDimension;
private final EnumSet<Capability> capabilities;
/**
* Create an {@link IndexTermType} from a {@link ColumnMetadata} and {@link IndexTarget.Type}.
*
* @param columnMetadata the {@link ColumnMetadata} for the column being indexed
* @param partitionColumns the partition columns for the table this column belongs to. This is used for identifying
* if the {@code columnMetadata} is a partition column and if it belongs to a composite
* partition
* @param indexTargetType the {@link IndexTarget.Type} for the index
*
* @return the {@link IndexTermType}
*/
public static IndexTermType create(ColumnMetadata columnMetadata, List<ColumnMetadata> partitionColumns, IndexTarget.Type indexTargetType)
{
return new IndexTermType(columnMetadata, partitionColumns, indexTargetType);
}
private IndexTermType(ColumnMetadata columnMetadata, List<ColumnMetadata> partitionColumns, IndexTarget.Type indexTargetType)
{
this.columnMetadata = columnMetadata;
this.indexTargetType = indexTargetType;
this.capabilities = calculateCapabilities(columnMetadata, partitionColumns, indexTargetType);
this.indexType = calculateIndexType(columnMetadata.type, capabilities, indexTargetType);
if (indexType.subTypes().isEmpty())
{
this.subTypes = Collections.emptyList();
}
else
{
List<IndexTermType> subTypes = new ArrayList<>(indexType.subTypes().size());
for (AbstractType<?> subType : indexType.subTypes())
subTypes.add(new IndexTermType(columnMetadata.withNewType(subType), partitionColumns, indexTargetType));
this.subTypes = Collections.unmodifiableList(subTypes);
}
if (isVector())
{
VectorType<?> vectorType = (VectorType<?>) indexType;
vectorElementType = vectorType.elementType;
vectorDimension = vectorType.dimension;
}
else
{
vectorElementType = null;
vectorDimension = -1;
}
}
/**
* Returns {@code true} if the index type is a literal type and will use a literal index. This applies to
* string types, frozen types, composite types and boolean type.
*/
public boolean isLiteral()
{
return capabilities.contains(Capability.LITERAL);
}
/**
* Returns {@code true} if the index type is a string type. This is used to determine if the type supports
* analysis.
*/
public boolean isString()
{
return capabilities.contains(Capability.STRING);
}
/**
* Returns {@code true} if the index type is a vector type. Note: being a vector type does not mean that the type
* is valid for indexing in that we don't check the element type and dimension constraints here.
*/
public boolean isVector()
{
return capabilities.contains(Capability.VECTOR);
}
/**
* Returns {@code true} if the index type is reversed. This is only the case (currently) for clustering keys with
* descending ordering.
*/
public boolean isReversed()
{
return capabilities.contains(Capability.REVERSED);
}
/**
* Returns {@code true} if the index type is frozen, e.g. the type is wrapped with {@code frozen<type>}.
*/
public boolean isFrozen()
{
return capabilities.contains(Capability.FROZEN);
}
/**
* Returns {@code true} if the index type is a non-frozen collection
*/
public boolean isNonFrozenCollection()
{
return capabilities.contains(Capability.NON_FROZEN_COLLECTION);
}
/**
* Returns {@code true} if the index type is a frozen collection. This is the inverse of a non-frozen collection
* but this method is here for clarity.
*/
public boolean isFrozenCollection()
{
return capabilities.contains(Capability.COLLECTION) && capabilities.contains(Capability.FROZEN);
}
/**
* Returns {@code true} if the index type is a composite type, e.g. it has the form {@code Composite<typea, typeb>}
*/
public boolean isComposite()
{
return capabilities.contains(Capability.COMPOSITE);
}
/**
* Returns {@code true} if the {@link RowFilter.Expression} passed is backed by a non-frozen collection and the
* {@code Operator} is one that cannot be merged together.
*/
public boolean isMultiExpression(RowFilter.Expression expression)
{
boolean multiExpression = false;
switch (expression.operator())
{
case EQ:
multiExpression = isNonFrozenCollection();
break;
case CONTAINS:
case CONTAINS_KEY:
multiExpression = true;
break;
}
return multiExpression;
}
/**
* Returns <code>true</code> if given buffer would pass the {@link AbstractType#validate(ByteBuffer)}
* check. False otherwise.
*/
public boolean isValid(ByteBuffer term)
{
try
{
indexType.validate(term);
return true;
}
catch (MarshalException e)
{
return false;
}
}
public AbstractType<?> indexType()
{
return indexType;
}
public Collection<IndexTermType> subTypes()
{
return subTypes;
}
public CQL3Type asCQL3Type()
{
return indexType.asCQL3Type();
}
public ColumnMetadata columnMetadata()
{
return columnMetadata;
}
public String columnName()
{
return columnMetadata.name.toString();
}
public AbstractType<?> vectorElementType()
{
assert isVector();
return vectorElementType;
}
public int vectorDimension()
{
assert isVector();
return vectorDimension;
}
public boolean dependsOn(ColumnMetadata columnMetadata)
{
return this.columnMetadata.compareTo(columnMetadata) == 0;
}
/**
* Indicates if the type encoding supports rounding of the raw value.
* <p>
* This is significant in range searches where we have to make all range
* queries inclusive when searching the indexes in order to avoid excluding
* rounded values. Excluded values are removed by post-filtering.
*/
public boolean supportsRounding()
{
return isBigInteger() || isBigDecimal();
}
/**
* Returns the value length for the given {@link AbstractType}, selecting 16 for types
* that officially use VARIABLE_LENGTH but are, in fact, of a fixed length.
*/
public int fixedSizeOf()
{
if (indexType.isValueLengthFixed())
return indexType.valueLengthIfFixed();
else if (isInetAddress())
return INET_ADDRESS_SIZE;
else if (isBigInteger())
return BIG_INTEGER_APPROXIMATION_BYTES;
else if (isBigDecimal())
return DECIMAL_APPROXIMATION_BYTES;
return DEFAULT_FIXED_LENGTH;
}
/**
* Allows overriding the default getString method for {@link CompositeType}. It is
* a requirement of the {@link ConcurrentRadixTree} that the keys are strings but
* the getString method of {@link CompositeType} does not return a string that compares
* in the same order as the underlying {@link ByteBuffer}. To get round this we convert
* the {@link CompositeType} bytes to a hex string.
*/
public String asString(ByteBuffer value)
{
if (isComposite())
return ByteBufferUtil.bytesToHex(value);
return indexType.getString(value);
}
/**
* The inverse of the above method. Overrides the fromString method on {@link CompositeType}
* in order to convert the hex string to bytes.
*/
public ByteBuffer fromString(String value)
{
if (isComposite())
return ByteBufferUtil.hexToBytes(value);
return indexType.fromString(value);
}
/**
* Returns the cell value from the {@link DecoratedKey} or {@link Row} for the {@link IndexTermType} based on the
* kind of column this {@link IndexTermType} is based on.
*
* @param key the {@link DecoratedKey} of the row
* @param row the {@link Row} containing the non-partition column data
* @param nowInSecs the time that the index write operation started
*
* @return a {@link ByteBuffer} containing the cell value
*/
public ByteBuffer valueOf(DecoratedKey key, Row row, long nowInSecs)
{
if (row == null)
return null;
switch (columnMetadata.kind)
{
case PARTITION_KEY:
return isCompositePartition() ? CompositeType.extractComponent(key.getKey(), columnMetadata.position())
: key.getKey();
case CLUSTERING:
// skip indexing of static clustering when regular column is indexed
return row.isStatic() ? null : row.clustering().bufferAt(columnMetadata.position());
// treat static cell retrieval the same was as regular
// only if row kind is STATIC otherwise return null
case STATIC:
if (!row.isStatic())
return null;
case REGULAR:
Cell<?> cell = row.getCell(columnMetadata);
return cell == null || !cell.isLive(nowInSecs) ? null : cell.buffer();
default:
return null;
}
}
/**
* Returns a value iterator for collection type {@link IndexTermType}s.
*
* @param row the {@link Row} containing the column data
* @param nowInSecs the time that the index write operation started
*
* @return an {@link Iterator} of the collection values
*/
public Iterator<ByteBuffer> valuesOf(Row row, long nowInSecs)
{
if (row == null)
return null;
switch (columnMetadata.kind)
{
// treat static cell retrieval the same was as regular
// only if row kind is STATIC otherwise return null
case STATIC:
if (!row.isStatic())
return null;
case REGULAR:
return collectionIterator(row.getComplexColumnData(columnMetadata), nowInSecs);
default:
return null;
}
}
public Comparator<ByteBuffer> comparator()
{
// Override the comparator for BigInteger, frozen collections and composite types
if (isBigInteger() || isBigDecimal() || isComposite() || isFrozen())
return FastByteOperations::compareUnsigned;
return indexType;
}
/**
* Compare two terms based on their type. This is used in place of {@link AbstractType#compare(ByteBuffer, ByteBuffer)}
* so that the default comparison can be overridden for specific types.
* <p>
* Note: This should be used for all term comparison
*/
public int compare(ByteBuffer b1, ByteBuffer b2)
{
if (isInetAddress())
return compareInet(b1, b2);
// BigInteger values, frozen types and composite types (map entries) use compareUnsigned to maintain
// a consistent order between the in-memory index and the on-disk index.
else if (isBigInteger() || isBigDecimal() || isComposite() || isFrozen())
return FastByteOperations.compareUnsigned(b1, b2);
return indexType.compare(b1, b2 );
}
/**
* Returns the smaller of two {@code ByteBuffer} values, based on the result of {@link
* #compare(ByteBuffer, ByteBuffer)} comparision.
*/
public ByteBuffer min(ByteBuffer a, ByteBuffer b)
{
return a == null ? b : (b == null || compare(b, a) > 0) ? a : b;
}
/**
* Returns the greater of two {@code ByteBuffer} values, based on the result of {@link
* #compare(ByteBuffer, ByteBuffer)} comparision.
*/
public ByteBuffer max(ByteBuffer a, ByteBuffer b)
{
return a == null ? b : (b == null || compare(b, a) < 0) ? a : b;
}
/**
* This is used for value comparison in post-filtering - {@link Expression#isSatisfiedBy(ByteBuffer)}.
* <p>
* This allows types to decide whether they should be compared based on their encoded value or their
* raw value. At present only {@link InetAddressType} values are compared by their encoded values to
* allow for ipv4 -> ipv6 equivalency in searches.
*/
public int comparePostFilter(Expression.Value requestedValue, Expression.Value columnValue)
{
if (isInetAddress())
return compareInet(requestedValue.encoded, columnValue.encoded);
// Override comparisons for frozen collections and composite types (map entries)
else if (isComposite() || isFrozen())
return FastByteOperations.compareUnsigned(requestedValue.raw, columnValue.raw);
return indexType.compare(requestedValue.raw, columnValue.raw);
}
/**
* Fills a byte array with the comparable bytes for a type.
* <p>
* This method expects a {@code value} parameter generated by calling {@link #asIndexBytes(ByteBuffer)}.
* It is not generally safe to pass the output of other serialization methods to this method. For instance, it is
* not generally safe to pass the output of {@link AbstractType#decompose(Object)} as the {@code value} parameter
* (there are certain types for which this is technically OK, but that doesn't hold for all types).
*
* @param value a value buffer returned by {@link #asIndexBytes(ByteBuffer)}
* @param bytes this method's output
*/
public void toComparableBytes(ByteBuffer value, byte[] bytes)
{
if (isInetAddress())
ByteBufferUtil.copyBytes(value, value.hasArray() ? value.arrayOffset() + value.position() : value.position(), bytes, 0, INET_ADDRESS_SIZE);
else if (isBigInteger())
ByteBufferUtil.copyBytes(value, value.hasArray() ? value.arrayOffset() + value.position() : value.position(), bytes, 0, BIG_INTEGER_APPROXIMATION_BYTES);
else if (isBigDecimal())
ByteBufferUtil.copyBytes(value, value.hasArray() ? value.arrayOffset() + value.position() : value.position(), bytes, 0, DECIMAL_APPROXIMATION_BYTES);
else
ByteSourceInverse.copyBytes(asComparableBytes(value, ByteComparable.Version.OSS50), bytes);
}
public ByteSource asComparableBytes(ByteBuffer value, ByteComparable.Version version)
{
if (isInetAddress() || isBigInteger() || isBigDecimal())
return ByteSource.optionalFixedLength(ByteBufferAccessor.instance, value);
// The LongType.asComparableBytes uses variableLengthInteger which doesn't play well with
// the balanced tree because it is expecting fixed length data. So for SAI we use a optionalSignedFixedLengthNumber
// to keep all comparable values the same length
else if (isLong())
return ByteSource.optionalSignedFixedLengthNumber(ByteBufferAccessor.instance, value);
return indexType.asComparableBytes(value, version);
}
/**
* Translates the external value of specific types into a format used by the index.
*/
public ByteBuffer asIndexBytes(ByteBuffer value)
{
if (value == null)
return null;
if (isInetAddress())
return encodeInetAddress(value);
else if (isBigInteger())
return encodeBigInteger(value);
else if (isBigDecimal())
return encodeDecimal(value);
return value;
}
public float[] decomposeVector(ByteBuffer byteBuffer)
{
assert isVector();
return ((VectorType<?>) indexType).composeAsFloat(byteBuffer);
}
public boolean supports(Operator operator)
{
if (operator == Operator.LIKE ||
operator == Operator.LIKE_CONTAINS ||
operator == Operator.LIKE_PREFIX ||
operator == Operator.LIKE_MATCHES ||
operator == Operator.LIKE_SUFFIX) return false;
// ANN is only supported against vectors, and vector indexes only support ANN
if (operator == Operator.ANN)
return isVector();
Expression.IndexOperator indexOperator = Expression.IndexOperator.valueOf(operator);
if (isNonFrozenCollection())
{
if (indexTargetType == IndexTarget.Type.KEYS) return indexOperator == Expression.IndexOperator.CONTAINS_KEY;
if (indexTargetType == IndexTarget.Type.VALUES) return indexOperator == Expression.IndexOperator.CONTAINS_VALUE;
return indexTargetType == IndexTarget.Type.KEYS_AND_VALUES && indexOperator == Expression.IndexOperator.EQ;
}
if (indexTargetType == IndexTarget.Type.FULL)
return indexOperator == Expression.IndexOperator.EQ;
if (indexOperator != Expression.IndexOperator.EQ && EQ_ONLY_TYPES.contains(indexType)) return false;
// RANGE only applicable to non-literal indexes
return (indexOperator != null) && !(isLiteral() && indexOperator == Expression.IndexOperator.RANGE);
}
@Override
public String toString()
{
return MoreObjects.toStringHelper(this)
.add("column", columnMetadata)
.add("type", indexType)
.add("indexType", indexTargetType)
.toString();
}
@Override
public boolean equals(Object obj)
{
if (obj == this)
return true;
if (!(obj instanceof IndexTermType))
return false;
IndexTermType other = (IndexTermType) obj;
return Objects.equals(columnMetadata, other.columnMetadata) && (indexTargetType == other.indexTargetType);
}
@Override
public int hashCode()
{
return Objects.hash(columnMetadata, indexTargetType);
}
private EnumSet<Capability> calculateCapabilities(ColumnMetadata columnMetadata, List<ColumnMetadata> partitionKeyColumns, IndexTarget.Type indexTargetType)
{
EnumSet<Capability> capabilities = EnumSet.noneOf(Capability.class);
if (partitionKeyColumns.contains(columnMetadata) && partitionKeyColumns.size() > 1)
capabilities.add(Capability.COMPOSITE_PARTITION);
AbstractType<?> type = columnMetadata.type;
if (type.isReversed())
capabilities.add(Capability.REVERSED);
AbstractType<?> baseType = type.unwrap();
if (baseType.isCollection())
capabilities.add(Capability.COLLECTION);
if (baseType.isCollection() && baseType.isMultiCell())
capabilities.add(Capability.NON_FROZEN_COLLECTION);
if (!baseType.subTypes().isEmpty() && !baseType.isMultiCell())
capabilities.add(Capability.FROZEN);
AbstractType<?> indexType = calculateIndexType(baseType, capabilities, indexTargetType);
if (indexType instanceof CompositeType)
capabilities.add(Capability.COMPOSITE);
else if (!indexType.subTypes().isEmpty() && !indexType.isMultiCell())
capabilities.add(Capability.FROZEN);
if (indexType instanceof StringType)
capabilities.add(Capability.STRING);
if (indexType instanceof BooleanType)
capabilities.add(Capability.BOOLEAN);
if (capabilities.contains(Capability.STRING) ||
capabilities.contains(Capability.BOOLEAN) ||
capabilities.contains(Capability.FROZEN) ||
capabilities.contains(Capability.COMPOSITE))
capabilities.add(Capability.LITERAL);
if (indexType instanceof VectorType<?>)
capabilities.add(Capability.VECTOR);
if (indexType instanceof InetAddressType)
capabilities.add(Capability.INET_ADDRESS);
if (indexType instanceof IntegerType)
capabilities.add(Capability.BIG_INTEGER);
if (indexType instanceof DecimalType)
capabilities.add(Capability.BIG_DECIMAL);
if (indexType instanceof LongType)
capabilities.add(Capability.LONG);
return capabilities;
}
private AbstractType<?> calculateIndexType(AbstractType<?> baseType, EnumSet<Capability> capabilities, IndexTarget.Type indexTargetType)
{
return capabilities.contains(Capability.NON_FROZEN_COLLECTION) ? collectionCellValueType(baseType, indexTargetType) : baseType;
}
private Iterator<ByteBuffer> collectionIterator(ComplexColumnData cellData, long nowInSecs)
{
if (cellData == null)
return null;
Stream<ByteBuffer> stream = StreamSupport.stream(cellData.spliterator(), false)
.filter(cell -> cell != null && cell.isLive(nowInSecs))
.map(this::cellValue);
if (isInetAddress())
stream = stream.sorted((c1, c2) -> compareInet(encodeInetAddress(c1), encodeInetAddress(c2)));
return stream.iterator();
}
private ByteBuffer cellValue(Cell<?> cell)
{
if (isNonFrozenCollection())
{
switch (((CollectionType<?>) columnMetadata.type).kind)
{
case LIST:
return cell.buffer();
case SET:
return cell.path().get(0);
case MAP:
switch (indexTargetType)
{
case KEYS:
return cell.path().get(0);
case VALUES:
return cell.buffer();
case KEYS_AND_VALUES:
return CompositeType.build(ByteBufferAccessor.instance, cell.path().get(0), cell.buffer());
}
}
}
return cell.buffer();
}
private AbstractType<?> collectionCellValueType(AbstractType<?> type, IndexTarget.Type indexType)
{
CollectionType<?> collection = ((CollectionType<?>) type);
switch (collection.kind)
{
case LIST:
return collection.valueComparator();
case SET:
return collection.nameComparator();
case MAP:
switch (indexType)
{
case KEYS:
return collection.nameComparator();
case VALUES:
return collection.valueComparator();
case KEYS_AND_VALUES:
return CompositeType.getInstance(collection.nameComparator(), collection.valueComparator());
}
default:
throw new IllegalArgumentException("Unsupported collection type: " + collection.kind);
}
}
private boolean isCompositePartition()
{
return capabilities.contains(Capability.COMPOSITE_PARTITION);
}
/**
* Returns <code>true</code> if given {@link AbstractType} is {@link InetAddressType}
*/
private boolean isInetAddress()
{
return capabilities.contains(Capability.INET_ADDRESS);
}
/**
* Returns <code>true</code> if given {@link AbstractType} is {@link IntegerType}
*/
private boolean isBigInteger()
{
return capabilities.contains(Capability.BIG_INTEGER);
}
/**
* Returns <code>true</code> if given {@link AbstractType} is {@link DecimalType}
*/
private boolean isBigDecimal()
{
return capabilities.contains(Capability.BIG_DECIMAL);
}
private boolean isLong()
{
return capabilities.contains(Capability.LONG);
}
/**
* Compares 2 InetAddress terms by ensuring that both addresses are represented as
* ipv6 addresses.
*/
private static int compareInet(ByteBuffer b1, ByteBuffer b2)
{
assert isIPv6(b1) && isIPv6(b2);
return FastByteOperations.compareUnsigned(b1, b2);
}
private static boolean isIPv6(ByteBuffer address)
{
return address.remaining() == INET_ADDRESS_SIZE;
}
/**
* Encode a {@link InetAddress} into a fixed width 16 byte encoded value.
* <p>
* The encoded value is byte comparable and prefix compressible.
* <p>
* The encoding is done by converting ipv4 addresses to their ipv6 equivalent.
*/
private static ByteBuffer encodeInetAddress(ByteBuffer value)
{
if (value.remaining() == 4)
{
int position = value.hasArray() ? value.arrayOffset() + value.position() : value.position();
ByteBuffer mapped = ByteBuffer.allocate(INET_ADDRESS_SIZE);
System.arraycopy(IPV4_PREFIX, 0, mapped.array(), 0, IPV4_PREFIX.length);
ByteBufferUtil.copyBytes(value, position, mapped, IPV4_PREFIX.length, value.remaining());
return mapped;
}
return value;
}
/**
* Encode a {@link BigInteger} into a fixed width 20 byte encoded value. The encoded value is byte comparable
* and prefix compressible.
* <p>
* The format of the encoding is:
* <p>
* The first 4 bytes contain the integer length of the {@link BigInteger} byte array
* with the top bit flipped for positive values.
* <p>
* The remaining 16 bytes contain the 16 most significant bytes of the
* {@link BigInteger} byte array.
* <p>
* For {@link BigInteger} values whose underlying byte array is less than
* 16 bytes, the encoded value is sign extended.
*/
public static ByteBuffer encodeBigInteger(ByteBuffer value)
{
int size = value.remaining();
int position = value.hasArray() ? value.arrayOffset() + value.position() : value.position();
byte[] bytes = new byte[BIG_INTEGER_APPROXIMATION_BYTES];
if (size < BIG_INTEGER_APPROXIMATION_BYTES - Integer.BYTES)
{
ByteBufferUtil.copyBytes(value, position, bytes, bytes.length - size, size);
if ((bytes[bytes.length - size] & 0x80) != 0)
Arrays.fill(bytes, Integer.BYTES, bytes.length - size, (byte)0xff);
else
Arrays.fill(bytes, Integer.BYTES, bytes.length - size, (byte)0x00);
}
else
{
ByteBufferUtil.copyBytes(value, position, bytes, Integer.BYTES, BIG_INTEGER_APPROXIMATION_BYTES - Integer.BYTES);
}
if ((bytes[4] & 0x80) != 0)
{
size = -size;
}
bytes[0] = (byte)(size >> 24 & 0xff);
bytes[1] = (byte)(size >> 16 & 0xff);
bytes[2] = (byte)(size >> 8 & 0xff);
bytes[3] = (byte)(size & 0xff);
bytes[0] ^= 0x80;
return ByteBuffer.wrap(bytes);
}
public static ByteBuffer encodeDecimal(ByteBuffer value)
{
ByteSource bs = DecimalType.instance.asComparableBytes(value, ByteComparable.Version.OSS50);
bs = ByteSource.cutOrRightPad(bs, DECIMAL_APPROXIMATION_BYTES, 0);
return ByteBuffer.wrap(ByteSourceInverse.readBytes(bs, DECIMAL_APPROXIMATION_BYTES));
}
}

522
src/java/org/apache/cassandra/index/sai/utils/TypeUtil.java
View File

@ -1,522 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.index.sai.utils;
import java.math.BigInteger;
import java.net.InetAddress;
import java.nio.ByteBuffer;
import java.util.Arrays;
import java.util.Comparator;
import java.util.Iterator;
import java.util.stream.Stream;
import java.util.stream.StreamSupport;
import com.googlecode.concurrenttrees.radix.ConcurrentRadixTree;
import org.apache.cassandra.cql3.statements.schema.IndexTarget;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.db.marshal.BooleanType;
import org.apache.cassandra.db.marshal.ByteBufferAccessor;
import org.apache.cassandra.db.marshal.CollectionType;
import org.apache.cassandra.db.marshal.CompositeType;
import org.apache.cassandra.db.marshal.DecimalType;
import org.apache.cassandra.db.marshal.InetAddressType;
import org.apache.cassandra.db.marshal.IntegerType;
import org.apache.cassandra.db.marshal.LongType;
import org.apache.cassandra.db.marshal.ReversedType;
import org.apache.cassandra.db.marshal.StringType;
import org.apache.cassandra.db.marshal.VectorType;
import org.apache.cassandra.db.rows.Cell;
import org.apache.cassandra.db.rows.ComplexColumnData;
import org.apache.cassandra.index.sai.IndexContext;
import org.apache.cassandra.index.sai.plan.Expression;
import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.serializers.MarshalException;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.FastByteOperations;
import org.apache.cassandra.utils.bytecomparable.ByteComparable;
import org.apache.cassandra.utils.bytecomparable.ByteSource;
import org.apache.cassandra.utils.bytecomparable.ByteSourceInverse;
public class TypeUtil
{
private static final byte[] IPV4_PREFIX = new byte[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, -1 };
/**
* DecimalType / BigDecimal values are indexed by truncating their asComparableBytes representation to this size,
* padding on the right with zero-value-bytes until this size is reached (if necessary). This causes
* false-positives that must be filtered in a separate step after hitting the index and reading the associated
* (full) values.
*/
public static final int DECIMAL_APPROXIMATION_BYTES = 24;
public static final int BIG_INTEGER_APPROXIMATION_BYTES = 20;
public static final int INET_ADDRESS_SIZE = 16;
public static final int DEFAULT_FIXED_LENGTH = 16;
private TypeUtil() {}
/**
* Returns <code>true</code> if given buffer would pass the {@link AbstractType#validate(ByteBuffer)}
* check. False otherwise.
*/
public static boolean isValid(ByteBuffer term, AbstractType<?> validator)
{
try
{
validator.validate(term);
return true;
}
catch (MarshalException e)
{
return false;
}
}
/**
* Indicates if the type encoding supports rounding of the raw value.
*
* This is significant in range searches where we have to make all range
* queries inclusive when searching the indexes in order to avoid excluding
* rounded values. Excluded values are removed by post-filtering.
*/
public static boolean supportsRounding(AbstractType<?> type)
{
return isBigInteger(type) || isBigDecimal(type);
}
/**
* Returns the smaller of two {@code ByteBuffer} values, based on the result of {@link
* #compare(ByteBuffer, ByteBuffer, AbstractType)} comparision.
*/
public static ByteBuffer min(ByteBuffer a, ByteBuffer b, AbstractType<?> type)
{
return a == null ? b : (b == null || compare(b, a, type) > 0) ? a : b;
}
/**
* Returns the greater of two {@code ByteBuffer} values, based on the result of {@link
* #compare(ByteBuffer, ByteBuffer, AbstractType)} comparision.
*/
public static ByteBuffer max(ByteBuffer a, ByteBuffer b, AbstractType<?> type)
{
return a == null ? b : (b == null || compare(b, a, type) < 0) ? a : b;
}
/**
* Returns the value length for the given {@link AbstractType}, selecting 16 for types
* that officially use VARIABLE_LENGTH but are, in fact, of a fixed length.
*/
public static int fixedSizeOf(AbstractType<?> type)
{
if (type.isValueLengthFixed())
return type.valueLengthIfFixed();
else if (isInetAddress(type))
return INET_ADDRESS_SIZE;
else if (isBigInteger(type))
return BIG_INTEGER_APPROXIMATION_BYTES;
else if (isBigDecimal(type))
return DECIMAL_APPROXIMATION_BYTES;
return DEFAULT_FIXED_LENGTH;
}
public static AbstractType<?> cellValueType(ColumnMetadata columnMetadata, IndexTarget.Type indexType)
{
AbstractType<?> type = columnMetadata.type;
if (isNonFrozenCollection(type))
{
CollectionType<?> collection = ((CollectionType<?>) type);
switch (collection.kind)
{
case LIST:
return collection.valueComparator();
case SET:
return collection.nameComparator();
case MAP:
switch (indexType)
{
case KEYS:
return collection.nameComparator();
case VALUES:
return collection.valueComparator();
case KEYS_AND_VALUES:
return CompositeType.getInstance(collection.nameComparator(), collection.valueComparator());
}
}
}
return type;
}
/**
* Allows overriding the default getString method for {@link CompositeType}. It is
* a requirement of the {@link ConcurrentRadixTree} that the keys are strings but
* the getString method of {@link CompositeType} does not return a string that compares
* in the same order as the underlying {@link ByteBuffer}. To get round this we convert
* the {@link CompositeType} bytes to a hex string.
*/
public static String getString(ByteBuffer value, AbstractType<?> type)
{
if (isComposite(type))
return ByteBufferUtil.bytesToHex(value);
return type.getString(value);
}
/**
* The inverse of the above method. Overrides the fromString method on {@link CompositeType}
* in order to convert the hex string to bytes.
*/
public static ByteBuffer fromString(String value, AbstractType<?> type)
{
if (isComposite(type))
return ByteBufferUtil.hexToBytes(value);
return type.fromString(value);
}
public static ByteSource asComparableBytes(ByteBuffer value, AbstractType<?> type, ByteComparable.Version version)
{
if (type instanceof InetAddressType || type instanceof IntegerType || type instanceof DecimalType)
return ByteSource.optionalFixedLength(ByteBufferAccessor.instance, value);
// The LongType.asComparableBytes uses variableLengthInteger which doesn't play well with
// the balanced tree because it is expecting fixed length data. So for SAI we use a optionalSignedFixedLengthNumber
// to keep all comparable values the same length
else if (type instanceof LongType)
return ByteSource.optionalSignedFixedLengthNumber(ByteBufferAccessor.instance, value);
return type.asComparableBytes(value, version);
}
/**
* Fills a byte array with the comparable bytes for a type.
* <p>
* This method expects a {@code value} parameter generated by calling {@link #asIndexBytes(ByteBuffer, AbstractType)}.
* It is not generally safe to pass the output of other serialization methods to this method. For instance, it is
* not generally safe to pass the output of {@link AbstractType#decompose(Object)} as the {@code value} parameter
* (there are certain types for which this is technically OK, but that doesn't hold for all types).
*
* @param value a value buffer returned by {@link #asIndexBytes(ByteBuffer, AbstractType)}
* @param type the type associated with the encoded {@code value} parameter
* @param bytes this method's output
*/
public static void toComparableBytes(ByteBuffer value, AbstractType<?> type, byte[] bytes)
{
if (isInetAddress(type))
ByteBufferUtil.copyBytes(value, value.hasArray() ? value.arrayOffset() + value.position() : value.position(), bytes, 0, INET_ADDRESS_SIZE);
else if (isBigInteger(type))
ByteBufferUtil.copyBytes(value, value.hasArray() ? value.arrayOffset() + value.position() : value.position(), bytes, 0, BIG_INTEGER_APPROXIMATION_BYTES);
else if (isBigDecimal(type))
ByteBufferUtil.copyBytes(value, value.hasArray() ? value.arrayOffset() + value.position() : value.position(), bytes, 0, DECIMAL_APPROXIMATION_BYTES);
else
ByteSourceInverse.copyBytes(asComparableBytes(value, type, ByteComparable.Version.OSS50), bytes);
}
/**
* Translates the external value of specific types into a format used by the index.
*/
public static ByteBuffer asIndexBytes(ByteBuffer value, AbstractType<?> type)
{
if (value == null)
return null;
if (isInetAddress(type))
return encodeInetAddress(value);
else if (isBigInteger(type))
return encodeBigInteger(value);
else if (type instanceof DecimalType)
return encodeDecimal(value);
return value;
}
public static float[] decomposeVector(IndexContext indexContext, ByteBuffer byteBuffer)
{
return ((VectorType<?>)indexContext.getValidator()).composeAsFloat(byteBuffer);
}
/**
* Compare two terms based on their type. This is used in place of {@link AbstractType#compare(ByteBuffer, ByteBuffer)}
* so that the default comparison can be overridden for specific types.
*
* Note: This should be used for all term comparison
*/
public static int compare(ByteBuffer b1, ByteBuffer b2, AbstractType<?> type)
{
if (isInetAddress(type))
return compareInet(b1, b2);
// BigInteger values, frozen types and composite types (map entries) use compareUnsigned to maintain
// a consistent order between the in-memory index and the on-disk index.
else if (isBigInteger(type) || isBigDecimal(type) || isCompositeOrFrozen(type))
return FastByteOperations.compareUnsigned(b1, b2);
return type.compare(b1, b2 );
}
/**
* This is used for value comparison in post-filtering - {@link Expression#isSatisfiedBy(ByteBuffer)}.
*
* This allows types to decide whether they should be compared based on their encoded value or their
* raw value. At present only {@link InetAddressType} values are compared by their encoded values to
* allow for ipv4 -> ipv6 equivalency in searches.
*/
public static int comparePostFilter(Expression.Value requestedValue, Expression.Value columnValue, AbstractType<?> type)
{
if (isInetAddress(type))
return compareInet(requestedValue.encoded, columnValue.encoded);
// Override comparisons for frozen collections and composite types (map entries)
else if (isCompositeOrFrozen(type))
return FastByteOperations.compareUnsigned(requestedValue.raw, columnValue.raw);
return type.compare(requestedValue.raw, columnValue.raw);
}
public static Iterator<ByteBuffer> collectionIterator(AbstractType<?> validator,
ComplexColumnData cellData,
ColumnMetadata columnMetadata,
IndexTarget.Type indexType,
long nowInSecs)
{
if (cellData == null)
return null;
Stream<ByteBuffer> stream = StreamSupport.stream(cellData.spliterator(), false).filter(cell -> cell != null && cell.isLive(nowInSecs))
.map(cell -> cellValue(columnMetadata, indexType, cell));
if (isInetAddress(validator))
stream = stream.sorted((c1, c2) -> compareInet(encodeInetAddress(c1), encodeInetAddress(c2)));
return stream.iterator();
}
public static Comparator<ByteBuffer> comparator(AbstractType<?> type)
{
// Override the comparator for BigInteger, frozen collections and composite types
if (isBigInteger(type) || isBigDecimal(type) || isCompositeOrFrozen(type))
return FastByteOperations::compareUnsigned;
return type;
}
private static ByteBuffer cellValue(ColumnMetadata columnMetadata, IndexTarget.Type indexType, Cell<?> cell)
{
if (columnMetadata.type.isCollection() && columnMetadata.type.isMultiCell())
{
switch (((CollectionType<?>) columnMetadata.type).kind)
{
case LIST:
return cell.buffer();
case SET:
return cell.path().get(0);
case MAP:
switch (indexType)
{
case KEYS:
return cell.path().get(0);
case VALUES:
return cell.buffer();
case KEYS_AND_VALUES:
return CompositeType.build(ByteBufferAccessor.instance, cell.path().get(0), cell.buffer());
}
}
}
return cell.buffer();
}
/**
* Compares 2 InetAddress terms by ensuring that both addresses are represented as
* ipv6 addresses.
*/
private static int compareInet(ByteBuffer b1, ByteBuffer b2)
{
assert isIPv6(b1) && isIPv6(b2);
return FastByteOperations.compareUnsigned(b1, b2);
}
private static boolean isIPv6(ByteBuffer address)
{
return address.remaining() == INET_ADDRESS_SIZE;
}
/**
* Encode a {@link InetAddress} into a fixed width 16 byte encoded value.
*
* The encoded value is byte comparable and prefix compressible.
*
* The encoding is done by converting ipv4 addresses to their ipv6 equivalent.
*/
private static ByteBuffer encodeInetAddress(ByteBuffer value)
{
if (value.remaining() == 4)
{
int position = value.hasArray() ? value.arrayOffset() + value.position() : value.position();
ByteBuffer mapped = ByteBuffer.allocate(INET_ADDRESS_SIZE);
System.arraycopy(IPV4_PREFIX, 0, mapped.array(), 0, IPV4_PREFIX.length);
ByteBufferUtil.copyBytes(value, position, mapped, IPV4_PREFIX.length, value.remaining());
return mapped;
}
return value;
}
/**
* Encode a {@link BigInteger} into a fixed width 20 byte encoded value.
*
* The encoded value is byte comparable and prefix compressible.
*
* The format of the encoding is:
*
* The first 4 bytes contain the integer length of the {@link BigInteger} byte array
* with the top bit flipped for positive values.
*
* The remaining 16 bytes contain the 16 most significant bytes of the
* {@link BigInteger} byte array.
*
* For {@link BigInteger} values whose underlying byte array is less than
* 16 bytes, the encoded value is sign extended.
*/
public static ByteBuffer encodeBigInteger(ByteBuffer value)
{
int size = value.remaining();
int position = value.hasArray() ? value.arrayOffset() + value.position() : value.position();
byte[] bytes = new byte[BIG_INTEGER_APPROXIMATION_BYTES];
if (size < BIG_INTEGER_APPROXIMATION_BYTES - Integer.BYTES)
{
ByteBufferUtil.copyBytes(value, position, bytes, bytes.length - size, size);
if ((bytes[bytes.length - size] & 0x80) != 0)
Arrays.fill(bytes, Integer.BYTES, bytes.length - size, (byte)0xff);
else
Arrays.fill(bytes, Integer.BYTES, bytes.length - size, (byte)0x00);
}
else
{
ByteBufferUtil.copyBytes(value, position, bytes, Integer.BYTES, BIG_INTEGER_APPROXIMATION_BYTES - Integer.BYTES);
}
if ((bytes[4] & 0x80) != 0)
{
size = -size;
}
bytes[0] = (byte)(size >> 24 & 0xff);
bytes[1] = (byte)(size >> 16 & 0xff);
bytes[2] = (byte)(size >> 8 & 0xff);
bytes[3] = (byte)(size & 0xff);
bytes[0] ^= 0x80;
return ByteBuffer.wrap(bytes);
}
/**
* Returns <code>true</code> if values of the given {@link AbstractType} should be indexed as literals.
*/
public static boolean isLiteral(AbstractType<?> type)
{
return isString(type) || isCompositeOrFrozen(type) || baseType(type) instanceof BooleanType;
}
/**
* Returns <code>true</code> if given {@link AbstractType} is based on a string, e.g. UTF8 or Ascii
*/
public static boolean isString(AbstractType<?> type)
{
type = baseType(type);
return type instanceof StringType;
}
/**
* Returns <code>true</code> if given {@link AbstractType} is a Composite(map entry) or frozen.
*/
public static boolean isCompositeOrFrozen(AbstractType<?> type)
{
type = baseType(type);
return type instanceof CompositeType || isFrozen(type);
}
/**
* Returns <code>true</code> if given {@link AbstractType} is frozen.
*/
public static boolean isFrozen(AbstractType<?> type)
{
type = baseType(type);
return !type.subTypes().isEmpty() && !type.isMultiCell();
}
/**
* Returns <code>true</code> if given {@link AbstractType} is a frozen collection.
*/
public static boolean isFrozenCollection(AbstractType<?> type)
{
type = baseType(type);
return type.isCollection() && !type.isMultiCell();
}
/**
* Returns <code>true</code> if given {@link AbstractType} is a non-frozen collection.
*/
public static boolean isNonFrozenCollection(AbstractType<?> type)
{
type = baseType(type);
return type.isCollection() && type.isMultiCell();
}
/**
* Returns <code>true</code> if given {@link AbstractType} is {@link InetAddressType}
*/
private static boolean isInetAddress(AbstractType<?> type)
{
type = baseType(type);
return type instanceof InetAddressType;
}
/**
* Returns <code>true</code> if given {@link AbstractType} is {@link IntegerType}
*/
private static boolean isBigInteger(AbstractType<?> type)
{
type = baseType(type);
return type instanceof IntegerType;
}
/**
* Returns <code>true</code> if given {@link AbstractType} is {@link DecimalType}
*/
private static boolean isBigDecimal(AbstractType<?> type)
{
type = baseType(type);
return type instanceof DecimalType;
}
/**
* Returns <code>true</code> if given {@link AbstractType} is {@link CompositeType}
*/
public static boolean isComposite(AbstractType<?> type)
{
type = baseType(type);
return type instanceof CompositeType;
}
/**
* @return base type if given type is reversed, otherwise return itself
*/
private static AbstractType<?> baseType(AbstractType<?> type)
{
return type.isReversed() ? ((ReversedType<?>) type).baseType : type;
}
public static ByteBuffer encodeDecimal(ByteBuffer value)
{
ByteSource bs = DecimalType.instance.asComparableBytes(value, ByteComparable.Version.OSS50);
bs = ByteSource.cutOrRightPad(bs, DECIMAL_APPROXIMATION_BYTES, 0);
return ByteBuffer.wrap(ByteSourceInverse.readBytes(bs, DECIMAL_APPROXIMATION_BYTES));
}
}

81
src/java/org/apache/cassandra/index/sai/view/IndexViewManager.java
View File

@ -28,9 +28,9 @@ import java.util.concurrent.atomic.AtomicReference;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.index.sai.IndexContext;
import org.apache.cassandra.index.sai.IndexValidation;
import org.apache.cassandra.index.sai.SSTableContext;
import org.apache.cassandra.index.sai.StorageAttachedIndex;
import org.apache.cassandra.index.sai.disk.SSTableIndex;
import org.apache.cassandra.index.sai.StorageAttachedIndexGroup;
import org.apache.cassandra.io.sstable.format.SSTableReader;
@ -38,7 +38,7 @@ import org.apache.cassandra.utils.Pair;
/**
* Maintain an atomic view for read requests, so that requests can read all data during concurrent compactions.
*
* <p>
* All per-column {@link SSTableIndex} updates should be proxied by {@link StorageAttachedIndexGroup} to make
* sure per-sstable {@link SSTableContext} are in-sync.
*/
@ -46,16 +46,16 @@ public class IndexViewManager
{
private static final Logger logger = LoggerFactory.getLogger(IndexViewManager.class);
private final IndexContext context;
private final StorageAttachedIndex index;
private final AtomicReference<View> view = new AtomicReference<>();
public IndexViewManager(IndexContext context)
public IndexViewManager(StorageAttachedIndex index)
{
this.context = context;
this.view.set(new View(context, Collections.emptySet()));
this.index = index;
this.view.set(new View(index.termType(), Collections.emptySet()));
}
public View getView()
public View view()
{
return view.get();
}
@ -72,7 +72,7 @@ public class IndexViewManager
public Collection<SSTableContext> update(Collection<SSTableReader> oldSSTables, Collection<SSTableContext> newSSTableContexts, IndexValidation validation)
{
// Valid indexes on the left and invalid SSTable contexts on the right...
Pair<Collection<SSTableIndex>, Collection<SSTableContext>> indexes = context.getBuiltIndexes(newSSTableContexts, validation);
Pair<Collection<SSTableIndex>, Collection<SSTableContext>> indexes = getBuiltIndexes(newSSTableContexts, validation);
View currentView, newView;
Collection<SSTableIndex> newViewIndexes = new HashSet<>();
@ -104,14 +104,14 @@ public class IndexViewManager
newViewIndexes.add(sstableIndex);
}
newView = new View(context, newViewIndexes);
newView = new View(index.termType(), newViewIndexes);
}
while (!view.compareAndSet(currentView, newView));
releasableIndexes.forEach(SSTableIndex::release);
if (logger.isTraceEnabled())
logger.trace(context.logMessage("There are now {} active SSTable indexes."), view.get().getIndexes().size());
logger.trace(index.identifier().logMessage("There are now {} active SSTable indexes."), view.get().getIndexes().size());
return indexes.right;
}
@ -139,11 +139,70 @@ public class IndexViewManager
*/
public void invalidate()
{
View previousView = view.getAndSet(new View(context, Collections.emptyList()));
View previousView = view.getAndSet(new View(index.termType(), Collections.emptyList()));
for (SSTableIndex index : previousView)
{
index.markObsolete();
}
}
/**
* @return the indexes that are built on the given SSTables on the left and corrupted indexes'
* corresponding contexts on the right
*/
private Pair<Collection<SSTableIndex>, Collection<SSTableContext>> getBuiltIndexes(Collection<SSTableContext> sstableContexts, IndexValidation validation)
{
Set<SSTableIndex> valid = new HashSet<>(sstableContexts.size());
Set<SSTableContext> invalid = new HashSet<>();
for (SSTableContext sstableContext : sstableContexts)
{
if (sstableContext.sstable.isMarkedCompacted())
continue;
if (!sstableContext.indexDescriptor.isPerColumnIndexBuildComplete(index.identifier()))
{
logger.debug(index.identifier().logMessage("An on-disk index build for SSTable {} has not completed."), sstableContext.descriptor());
continue;
}
if (sstableContext.indexDescriptor.isIndexEmpty(index.termType(), index.identifier()))
{
logger.debug(index.identifier().logMessage("No on-disk index was built for SSTable {} because the SSTable " +
"had no indexable rows for the index."), sstableContext.descriptor());
continue;
}
try
{
if (validation != IndexValidation.NONE)
{
if (!sstableContext.indexDescriptor.validatePerIndexComponents(index.termType(), index.identifier(), validation))
{
invalid.add(sstableContext);
continue;
}
}
SSTableIndex ssTableIndex = sstableContext.newSSTableIndex(index);
logger.debug(index.identifier().logMessage("Successfully created index for SSTable {}."), sstableContext.descriptor());
// Try to add new index to the set, if set already has such index, we'll simply release and move on.
// This covers situation when SSTable collection has the same SSTable multiple
// times because we don't know what kind of collection it actually is.
if (!valid.add(ssTableIndex))
{
ssTableIndex.release();
}
}
catch (Throwable e)
{
logger.warn(index.identifier().logMessage("Failed to update per-column components for SSTable {}"), sstableContext.descriptor(), e);
invalid.add(sstableContext);
}
}
return Pair.create(valid, invalid);
}
}

51
src/java/org/apache/cassandra/index/sai/view/RangeTermTree.java
View File

@ -26,11 +26,9 @@ import com.google.common.base.MoreObjects;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.index.sai.IndexContext;
import org.apache.cassandra.index.sai.disk.SSTableIndex;
import org.apache.cassandra.index.sai.plan.Expression;
import org.apache.cassandra.index.sai.utils.TypeUtil;
import org.apache.cassandra.index.sai.utils.IndexTermType;
import org.apache.cassandra.utils.Interval;
import org.apache.cassandra.utils.IntervalTree;
@ -39,60 +37,59 @@ public class RangeTermTree
private static final Logger logger = LoggerFactory.getLogger(RangeTermTree.class);
protected final ByteBuffer min, max;
protected final AbstractType<?> comparator;
protected final IndexTermType indexTermType;
private final IntervalTree<Term, SSTableIndex, Interval<Term, SSTableIndex>> rangeTree;
private RangeTermTree(ByteBuffer min, ByteBuffer max, IntervalTree<Term, SSTableIndex, Interval<Term, SSTableIndex>> rangeTree, AbstractType<?> comparator)
private RangeTermTree(ByteBuffer min, ByteBuffer max, IntervalTree<Term, SSTableIndex, Interval<Term, SSTableIndex>> rangeTree, IndexTermType indexTermType)
{
this.min = min;
this.max = max;
this.rangeTree = rangeTree;
this.comparator = comparator;
this.indexTermType = indexTermType;
}
public List<SSTableIndex> search(Expression e)
{
ByteBuffer minTerm = e.lower == null ? min : e.lower.value.encoded;
ByteBuffer maxTerm = e.upper == null ? max : e.upper.value.encoded;
ByteBuffer minTerm = e.lower() == null ? min : e.lower().value.encoded;
ByteBuffer maxTerm = e.upper() == null ? max : e.upper().value.encoded;
return rangeTree.search(Interval.create(new Term(minTerm, comparator),
new Term(maxTerm, comparator),
return rangeTree.search(Interval.create(new Term(minTerm, indexTermType),
new Term(maxTerm, indexTermType),
null));
}
static class Builder
{
private final AbstractType<?> comparator;
private final IndexTermType indexTermType;
private ByteBuffer min, max;
final List<Interval<Term, SSTableIndex>> intervals = new ArrayList<>();
protected Builder(AbstractType<?> comparator)
protected Builder(IndexTermType indexTermType)
{
this.comparator = comparator;
this.indexTermType = indexTermType;
}
public final void add(SSTableIndex index)
{
addIndex(index);
min = min == null || TypeUtil.compare(min, index.minTerm(), comparator) > 0 ? index.minTerm() : min;
max = max == null || TypeUtil.compare(max, index.maxTerm(), comparator) < 0 ? index.maxTerm() : max;
min = min == null || index.getIndexTermType().compare(min, index.minTerm()) > 0 ? index.minTerm() : min;
max = max == null || index.getIndexTermType().compare(max, index.maxTerm()) < 0 ? index.maxTerm() : max;
}
public void addIndex(SSTableIndex index)
{
Interval<Term, SSTableIndex> interval =
Interval.create(new Term(index.minTerm(), comparator), new Term(index.maxTerm(), comparator), index);
Interval.create(new Term(index.minTerm(), indexTermType), new Term(index.maxTerm(), indexTermType), index);
if (logger.isTraceEnabled())
{
IndexContext context = index.getIndexContext();
logger.trace(context.logMessage("Adding index for SSTable {} with minTerm={} and maxTerm={}..."),
index.getSSTable().descriptor,
comparator.compose(index.minTerm()),
comparator.compose(index.maxTerm()));
logger.trace(index.getIndexIdentifier().logMessage("Adding index for SSTable {} with minTerm={} and maxTerm={}..."),
index.getSSTable().descriptor,
indexTermType.indexType().compose(index.minTerm()),
indexTermType.indexType().compose(index.maxTerm()));
}
intervals.add(interval);
@ -100,7 +97,7 @@ public class RangeTermTree
public RangeTermTree build()
{
return new RangeTermTree(min, max, IntervalTree.build(intervals), comparator);
return new RangeTermTree(min, max, IntervalTree.build(intervals), indexTermType);
}
}
@ -111,24 +108,24 @@ public class RangeTermTree
protected static class Term implements Comparable<Term>
{
private final ByteBuffer term;
private final AbstractType<?> comparator;
private final IndexTermType indexTermType;
Term(ByteBuffer term, AbstractType<?> comparator)
Term(ByteBuffer term, IndexTermType indexTermType)
{
this.term = term;
this.comparator = comparator;
this.indexTermType = indexTermType;
}
@Override
public int compareTo(Term o)
{
return TypeUtil.compare(term, o.term, comparator);
return indexTermType.compare(term, o.term);
}
@Override
public String toString()
{
return MoreObjects.toStringHelper(this).add("term", comparator.getString(term)).toString();
return MoreObjects.toStringHelper(this).add("term", indexTermType.asString(term)).toString();
}
}
}

13
src/java/org/apache/cassandra/index/sai/view/View.java
View File

@ -23,16 +23,15 @@ import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.index.sai.IndexContext;
import org.apache.cassandra.index.sai.disk.SSTableIndex;
import org.apache.cassandra.index.sai.plan.Expression;
import org.apache.cassandra.index.sai.utils.IndexTermType;
import org.apache.cassandra.io.sstable.Descriptor;
import org.apache.cassandra.io.sstable.format.SSTableReader;
/**
* The View is an immutable, point in time, view of the avalailable {@link SSTableIndex}es for an index.
*
* <p>
* The view maintains a {@link RangeTermTree} for querying the view by value range. This is used by the
* {@link org.apache.cassandra.index.sai.plan.QueryViewBuilder} to select the set of {@link SSTableIndex}es
* to perform a query without needing to query indexes that are known not to contain to the requested
@ -44,13 +43,11 @@ public class View implements Iterable<SSTableIndex>
private final RangeTermTree rangeTermTree;
public View(IndexContext context, Collection<SSTableIndex> indexes)
public View(IndexTermType indexTermType, Collection<SSTableIndex> indexes)
{
this.view = new HashMap<>();
AbstractType<?> termValidator = context.getValidator();
RangeTermTree.Builder rangeTermTreeBuilder = new RangeTermTree.Builder(termValidator);
RangeTermTree.Builder rangeTermTreeBuilder = new RangeTermTree.Builder(indexTermType);
for (SSTableIndex sstableIndex : indexes)
{
@ -67,7 +64,7 @@ public class View implements Iterable<SSTableIndex>
*/
public Collection<SSTableIndex> match(Expression expression)
{
if (expression.getOp() == Expression.IndexOperator.ANN)
if (expression.getIndexOperator() == Expression.IndexOperator.ANN)
return getIndexes();
return rangeTermTree.search(expression);

17
src/java/org/apache/cassandra/index/sai/virtual/ColumnIndexesSystemView.java
View File

@ -27,9 +27,7 @@ import org.apache.cassandra.db.virtual.SimpleDataSet;
import org.apache.cassandra.db.virtual.VirtualTable;
import org.apache.cassandra.dht.LocalPartitioner;
import org.apache.cassandra.exceptions.InvalidRequestException;
import org.apache.cassandra.index.Index;
import org.apache.cassandra.index.SecondaryIndexManager;
import org.apache.cassandra.index.sai.IndexContext;
import org.apache.cassandra.index.sai.StorageAttachedIndex;
import org.apache.cassandra.index.sai.StorageAttachedIndexGroup;
import org.apache.cassandra.schema.Schema;
@ -92,19 +90,18 @@ public class ColumnIndexesSystemView extends AbstractVirtualTable
if (group != null)
{
for (Index index : group.getIndexes())
{
IndexContext context = ((StorageAttachedIndex) index).getIndexContext();
String indexName = context.getIndexName();
group.getIndexes().forEach(i -> {
StorageAttachedIndex index = (StorageAttachedIndex) i;
String indexName = index.identifier().indexName;
dataset.row(ks, indexName)
.column(TABLE_NAME, cfs.name)
.column(COLUMN_NAME, context.getColumnName())
.column(COLUMN_NAME, index.termType().columnName())
.column(IS_QUERYABLE, manager.isIndexQueryable(index))
.column(IS_BUILDING, manager.isIndexBuilding(indexName))
.column(IS_STRING, context.isLiteral())
.column(ANALYZER, context.getAnalyzerFactory().toString());
}
.column(IS_STRING, index.termType().isLiteral())
.column(ANALYZER, index.hasAnalyzer() ? index.analyzer().toString() : "NoOpAnalyzer");
});
}
}
}

15
src/java/org/apache/cassandra/index/sai/virtual/SSTableIndexesSystemView.java
View File

@ -27,8 +27,6 @@ import org.apache.cassandra.db.virtual.VirtualTable;
import org.apache.cassandra.dht.AbstractBounds;
import org.apache.cassandra.dht.LocalPartitioner;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.index.Index;
import org.apache.cassandra.index.sai.IndexContext;
import org.apache.cassandra.index.sai.StorageAttachedIndex;
import org.apache.cassandra.index.sai.StorageAttachedIndexGroup;
import org.apache.cassandra.index.sai.disk.SSTableIndex;
@ -99,19 +97,18 @@ public class SSTableIndexesSystemView extends AbstractVirtualTable
{
Token.TokenFactory tokenFactory = cfs.metadata().partitioner.getTokenFactory();
for (Index index : group.getIndexes())
{
IndexContext indexContext = ((StorageAttachedIndex)index).getIndexContext();
group.getIndexes().forEach(i -> {
StorageAttachedIndex index = (StorageAttachedIndex)i;
for (SSTableIndex sstableIndex : indexContext.getView())
for (SSTableIndex sstableIndex : index.view())
{
SSTableReader sstable = sstableIndex.getSSTable();
Descriptor descriptor = sstable.descriptor;
AbstractBounds<Token> bounds = sstable.getBounds();
dataset.row(ks, indexContext.getIndexName(), sstable.getFilename())
dataset.row(ks, index.identifier().indexName, sstable.getFilename())
.column(TABLE_NAME, descriptor.cfname)
.column(COLUMN_NAME, indexContext.getColumnName())
.column(COLUMN_NAME, index.termType().columnName())
.column(FORMAT_VERSION, sstableIndex.getVersion().toString())
.column(CELL_COUNT, sstableIndex.getRowCount())
.column(MIN_ROW_ID, sstableIndex.minSSTableRowId())
@ -121,7 +118,7 @@ public class SSTableIndexesSystemView extends AbstractVirtualTable
.column(PER_TABLE_DISK_SIZE, sstableIndex.getSSTableContext().diskUsage())
.column(PER_COLUMN_DISK_SIZE, sstableIndex.sizeOfPerColumnComponents());
}
}
});
}
}
}

15
src/java/org/apache/cassandra/index/sai/virtual/SegmentsSystemView.java
View File

@ -28,8 +28,6 @@ import org.apache.cassandra.db.virtual.AbstractVirtualTable;
import org.apache.cassandra.db.virtual.SimpleDataSet;
import org.apache.cassandra.db.virtual.VirtualTable;
import org.apache.cassandra.dht.LocalPartitioner;
import org.apache.cassandra.index.Index;
import org.apache.cassandra.index.sai.IndexContext;
import org.apache.cassandra.index.sai.StorageAttachedIndex;
import org.apache.cassandra.index.sai.StorageAttachedIndexGroup;
import org.apache.cassandra.index.sai.disk.SSTableIndex;
@ -89,8 +87,8 @@ public class SegmentsSystemView extends AbstractVirtualTable
{
SimpleDataSet dataset = new SimpleDataSet(metadata());
forEachIndex(indexContext -> {
for (SSTableIndex sstableIndex : indexContext.getView())
forEachIndex(index -> {
for (SSTableIndex sstableIndex : index.view())
{
sstableIndex.populateSegmentView(dataset);
}
@ -99,7 +97,7 @@ public class SegmentsSystemView extends AbstractVirtualTable
return dataset;
}
private void forEachIndex(Consumer<IndexContext> process)
private void forEachIndex(Consumer<StorageAttachedIndex> process)
{
for (String ks : Schema.instance.getUserKeyspaces())
{
@ -112,12 +110,7 @@ public class SegmentsSystemView extends AbstractVirtualTable
StorageAttachedIndexGroup group = StorageAttachedIndexGroup.getIndexGroup(cfs);
if (group != null)
{
for (Index index : group.getIndexes())
{
process.accept(((StorageAttachedIndex)index).getIndexContext());
}
}
group.getIndexes().stream().map(index -> (StorageAttachedIndex) index).forEach(process);
}
}
}

17
test/distributed/org/apache/cassandra/distributed/test/sai/IndexStreamingFailureTest.java
View File

@ -21,33 +21,32 @@ package org.apache.cassandra.distributed.test.sai;
import java.io.IOException;
import java.util.concurrent.Callable;
import net.bytebuddy.ByteBuddy;
import net.bytebuddy.dynamic.loading.ClassLoadingStrategy;
import net.bytebuddy.implementation.MethodDelegation;
import net.bytebuddy.implementation.bind.annotation.SuperCall;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
import net.bytebuddy.ByteBuddy;
import net.bytebuddy.dynamic.loading.ClassLoadingStrategy;
import net.bytebuddy.implementation.MethodDelegation;
import net.bytebuddy.implementation.bind.annotation.SuperCall;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.distributed.Cluster;
import org.apache.cassandra.distributed.api.IInvokableInstance;
import org.apache.cassandra.distributed.test.TestBaseImpl;
import org.apache.cassandra.index.sai.IndexContext;
import org.apache.cassandra.index.sai.disk.format.IndexDescriptor;
import org.apache.cassandra.index.sai.disk.v1.SAICodecUtils;
import org.apache.cassandra.index.sai.disk.v1.segment.SegmentBuilder;
import org.apache.cassandra.index.sai.disk.v1.segment.SegmentMetadata;
import org.apache.cassandra.index.sai.utils.IndexIdentifier;
import org.apache.cassandra.utils.Throwables;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.store.IndexInput;
import static net.bytebuddy.matcher.ElementMatchers.named;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.Assert.assertFalse;
import static org.apache.cassandra.distributed.api.Feature.GOSSIP;
import static org.apache.cassandra.distributed.api.Feature.NETWORK;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.Assert.assertFalse;
public class IndexStreamingFailureTest extends TestBaseImpl
{
@ -164,7 +163,7 @@ public class IndexStreamingFailureTest extends TestBaseImpl
}
@SuppressWarnings("unused")
public static SegmentMetadata flush(IndexDescriptor indexDescriptor, IndexContext indexContext, @SuperCall Callable<SegmentMetadata> zuper) throws IOException
public static SegmentMetadata flush(IndexDescriptor indexDescriptor, IndexIdentifier indexIdentifier, @SuperCall Callable<SegmentMetadata> zuper) throws IOException
{
if (failFlush)
throw new IOException(TEST_ERROR_MESSAGE);

3
test/microbench/org/apache/cassandra/test/microbench/sai/KeyLookupBench.java
View File

@ -37,7 +37,6 @@ import org.apache.cassandra.index.sai.disk.format.IndexDescriptor;
import org.apache.cassandra.index.sai.disk.v1.SSTableComponentsWriter;
import org.apache.cassandra.index.sai.disk.v1.WidePrimaryKeyMap;
import org.apache.cassandra.index.sai.utils.PrimaryKey;
import org.apache.cassandra.index.sai.utils.TypeUtil;
import org.apache.cassandra.io.sstable.Descriptor;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.util.File;
@ -173,7 +172,7 @@ public class KeyLookupBench
private static DecoratedKey makeKey(TableMetadata table, Object...partitionKeys)
{
ByteBuffer key;
if (TypeUtil.isComposite(table.partitionKeyType))
if (table.partitionKeyType instanceof CompositeType)
key = ((CompositeType)table.partitionKeyType).decompose(partitionKeys);
else
key = table.partitionKeyType.fromString((String)partitionKeys[0]);

156
test/unit/org/apache/cassandra/index/sai/SAITester.java
View File

@ -31,8 +31,10 @@ import java.nio.file.StandardOpenOption;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Random;
import java.util.Set;
@ -70,16 +72,18 @@ import org.apache.cassandra.db.Directories;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.db.compaction.CompactionManager;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.db.marshal.UTF8Type;
import org.apache.cassandra.db.marshal.Int32Type;
import org.apache.cassandra.dht.Murmur3Partitioner;
import org.apache.cassandra.index.Index;
import org.apache.cassandra.index.sai.disk.SSTableIndex;
import org.apache.cassandra.index.sai.disk.format.IndexComponent;
import org.apache.cassandra.index.sai.disk.format.IndexDescriptor;
import org.apache.cassandra.index.sai.utils.IndexIdentifier;
import org.apache.cassandra.index.sai.disk.format.OnDiskFormat;
import org.apache.cassandra.index.sai.disk.format.Version;
import org.apache.cassandra.index.sai.disk.v1.V1OnDiskFormat;
import org.apache.cassandra.index.sai.disk.v1.segment.SegmentBuilder;
import org.apache.cassandra.index.sai.utils.IndexTermType;
import org.apache.cassandra.index.sai.utils.PrimaryKey;
import org.apache.cassandra.index.sai.utils.ResourceLeakDetector;
import org.apache.cassandra.inject.Injection;
@ -90,9 +94,12 @@ import org.apache.cassandra.io.sstable.format.SSTableFormat;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.sstable.format.TOCComponent;
import org.apache.cassandra.io.util.File;
import org.apache.cassandra.schema.CachingParams;
import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.schema.IndexMetadata;
import org.apache.cassandra.schema.MockSchema;
import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.service.snapshot.TableSnapshot;
import org.apache.cassandra.utils.JVMStabilityInspector;
@ -250,16 +257,59 @@ public abstract class SAITester extends CQLTester
public abstract void corrupt(File file) throws IOException;
}
public static IndexContext createIndexContext(String name, AbstractType<?> validator)
public static StorageAttachedIndex createMockIndex(ColumnMetadata column)
{
return new IndexContext("test_ks",
"test_cf",
UTF8Type.instance,
Murmur3Partitioner.instance,
new ClusteringComparator(),
ColumnMetadata.regularColumn("sai", "internal", name, validator),
IndexTarget.Type.SIMPLE,
IndexMetadata.fromSchemaMetadata(name, IndexMetadata.Kind.CUSTOM, null));
TableMetadata table = TableMetadata.builder(column.ksName, column.cfName)
.addPartitionKeyColumn("pk", Int32Type.instance)
.addRegularColumn(column.name, column.type)
.partitioner(Murmur3Partitioner.instance)
.caching(CachingParams.CACHE_NOTHING)
.build();
Map<String, String> options = new HashMap<>();
options.put(IndexTarget.CUSTOM_INDEX_OPTION_NAME, StorageAttachedIndex.class.getCanonicalName());
options.put("target", column.name.toString());
IndexMetadata indexMetadata = IndexMetadata.fromSchemaMetadata(column.name.toString(), IndexMetadata.Kind.CUSTOM, options);
ColumnFamilyStore cfs = MockSchema.newCFS(table);
return new StorageAttachedIndex(cfs, indexMetadata);
}
public static StorageAttachedIndex createMockIndex(String columnName, AbstractType<?> cellType)
{
TableMetadata table = TableMetadata.builder("test", "test")
.addPartitionKeyColumn("pk", Int32Type.instance)
.addRegularColumn(columnName, cellType)
.partitioner(Murmur3Partitioner.instance)
.caching(CachingParams.CACHE_NOTHING)
.build();
Map<String, String> options = new HashMap<>();
options.put(IndexTarget.CUSTOM_INDEX_OPTION_NAME, StorageAttachedIndex.class.getCanonicalName());
options.put("target", columnName);
IndexMetadata indexMetadata = IndexMetadata.fromSchemaMetadata(columnName, IndexMetadata.Kind.CUSTOM, options);
ColumnFamilyStore cfs = MockSchema.newCFS(table);
return new StorageAttachedIndex(cfs, indexMetadata);
}
public static IndexTermType createIndexTermType(AbstractType<?> cellType)
{
return IndexTermType.create(ColumnMetadata.regularColumn("sai", "internal", "val", cellType), Collections.emptyList(), IndexTarget.Type.SIMPLE);
}
public IndexIdentifier createIndexIdentifier(String indexName)
{
return createIndexIdentifier(keyspace(), currentTable(), indexName);
}
public static IndexIdentifier createIndexIdentifier(String keyspaceName, String tableName, String indexName)
{
return new IndexIdentifier(keyspaceName, tableName, indexName);
}
protected StorageAttachedIndexGroup getCurrentIndexGroup()
@ -267,6 +317,11 @@ public abstract class SAITester extends CQLTester
return StorageAttachedIndexGroup.getIndexGroup(getCurrentColumnFamilyStore());
}
protected void dropIndex(IndexIdentifier indexIdentifier) throws Throwable
{
dropIndex("DROP INDEX %s." + indexIdentifier.indexName);
}
protected void simulateNodeRestart()
{
simulateNodeRestart(true);
@ -294,12 +349,12 @@ public abstract class SAITester extends CQLTester
}
}
protected void corruptIndexComponent(IndexComponent indexComponent, IndexContext indexContext, CorruptionType corruptionType) throws Exception
protected void corruptIndexComponent(IndexComponent indexComponent, IndexIdentifier indexIdentifier, CorruptionType corruptionType) throws Exception
{
ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
for (SSTableReader sstable : cfs.getLiveSSTables())
{
File file = IndexDescriptor.create(sstable).fileFor(indexComponent, indexContext);
File file = IndexDescriptor.create(sstable).fileFor(indexComponent, indexIdentifier);
corruptionType.corrupt(file);
}
}
@ -316,7 +371,7 @@ public abstract class SAITester extends CQLTester
waitForAssert(() -> assertTrue(indexNeedsFullRebuild(indexName)));
}
protected boolean verifyChecksum(IndexContext indexContext)
protected boolean verifyChecksum(IndexTermType indexContext, IndexIdentifier indexIdentifier)
{
ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
@ -324,21 +379,22 @@ public abstract class SAITester extends CQLTester
{
IndexDescriptor indexDescriptor = IndexDescriptor.create(sstable);
if (!indexDescriptor.validatePerSSTableComponents(IndexValidation.CHECKSUM)
|| !indexDescriptor.validatePerIndexComponents(indexContext, IndexValidation.CHECKSUM))
|| !indexDescriptor.validatePerIndexComponents(indexContext, indexIdentifier, IndexValidation.CHECKSUM))
return false;
}
return true;
}
protected boolean validateComponents(IndexContext indexContext)
protected boolean validateComponents(IndexTermType indexTermType, String indexName)
{
ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
IndexIdentifier indexIdentifier = createIndexIdentifier(cfs.getKeyspaceName(), cfs.getTableName(), indexName);
for (SSTableReader sstable : cfs.getLiveSSTables())
{
IndexDescriptor indexDescriptor = IndexDescriptor.create(sstable);
if (!indexDescriptor.validatePerSSTableComponents(IndexValidation.HEADER_FOOTER)
|| !indexDescriptor.validatePerIndexComponents(indexContext, IndexValidation.HEADER_FOOTER))
|| !indexDescriptor.validatePerIndexComponents(indexTermType, indexIdentifier, IndexValidation.HEADER_FOOTER))
return false;
}
return true;
@ -467,67 +523,45 @@ public abstract class SAITester extends CQLTester
assertTrue(indexFiles().isEmpty());
}
protected void verifyIndexFiles(IndexContext numericIndexContext, IndexContext literalIndexContext, int numericFiles, int literalFiles)
protected void verifyIndexFiles(IndexTermType indexTermType,
IndexIdentifier indexIdentifier,
int indexFiles)
{
verifyIndexFiles(numericIndexContext,
literalIndexContext,
Math.max(numericFiles, literalFiles),
numericFiles,
literalFiles,
numericFiles,
literalFiles);
verifyIndexFiles(indexTermType, indexIdentifier, indexFiles, indexFiles, indexFiles);
}
protected void verifyIndexFiles(IndexContext numericIndexContext,
IndexContext literalIndexContext,
protected void verifyIndexFiles(IndexTermType indexTermType,
IndexIdentifier indexIdentifier,
int perSSTableFiles,
int numericFiles,
int literalFiles,
int numericCompletionMarkers,
int literalCompletionMarkers)
int perColumnFiles,
int completionMarkers)
{
Set<File> indexFiles = indexFiles();
for (IndexComponent indexComponent : Version.LATEST.onDiskFormat().perSSTableIndexComponents(false))
{
Set<File> tableFiles = componentFiles(indexFiles, SSTableFormat.Components.Types.CUSTOM.createComponent(Version.LATEST.fileNameFormatter().format(indexComponent, null)));
Component component = SSTableFormat.Components.Types.CUSTOM.createComponent(Version.LATEST.fileNameFormatter().format(indexComponent, null));
Set<File> tableFiles = componentFiles(indexFiles, component);
assertEquals(tableFiles.toString(), perSSTableFiles, tableFiles.size());
}
if (literalIndexContext != null)
for (IndexComponent indexComponent : Version.LATEST.onDiskFormat().perColumnIndexComponents(indexTermType))
{
for (IndexComponent indexComponent : Version.LATEST.onDiskFormat().perColumnIndexComponents(literalIndexContext))
{
Set<File> stringIndexFiles = componentFiles(indexFiles,
SSTableFormat.Components.Types.CUSTOM.createComponent(Version.LATEST.fileNameFormatter().format(indexComponent,
literalIndexContext)));
if (isBuildCompletionMarker(indexComponent))
assertEquals(literalCompletionMarkers, stringIndexFiles.size());
else
assertEquals(stringIndexFiles.toString(), literalFiles, stringIndexFiles.size());
}
}
if (numericIndexContext != null)
{
for (IndexComponent indexComponent : Version.LATEST.onDiskFormat().perColumnIndexComponents(numericIndexContext))
{
Set<File> numericIndexFiles = componentFiles(indexFiles,
SSTableFormat.Components.Types.CUSTOM.createComponent(Version.LATEST.fileNameFormatter().format(indexComponent,
numericIndexContext)));
if (isBuildCompletionMarker(indexComponent))
assertEquals(numericCompletionMarkers, numericIndexFiles.size());
else
assertEquals(numericIndexFiles.toString(), numericFiles, numericIndexFiles.size());
}
String componentName = Version.LATEST.fileNameFormatter().format(indexComponent, indexIdentifier);
Component component = SSTableFormat.Components.Types.CUSTOM.createComponent(componentName);
Set<File> stringIndexFiles = componentFiles(indexFiles, component);
if (isBuildCompletionMarker(indexComponent))
assertEquals(completionMarkers, stringIndexFiles.size());
else
assertEquals(stringIndexFiles.toString(), perColumnFiles, stringIndexFiles.size());
}
}
protected void verifySSTableIndexes(String indexName, int count)
protected void verifySSTableIndexes(IndexIdentifier indexIdentifier, int count)
{
try
{
verifySSTableIndexes(indexName, count, count);
verifySSTableIndexes(indexIdentifier, count, count);
}
catch (Exception e)
{
@ -535,15 +569,15 @@ public abstract class SAITester extends CQLTester
}
}
protected void verifySSTableIndexes(String indexName, int sstableContextCount, int sstableIndexCount)
protected void verifySSTableIndexes(IndexIdentifier indexIdentifier, int sstableContextCount, int sstableIndexCount)
{
ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
StorageAttachedIndexGroup indexGroup = getCurrentIndexGroup();
int contextCount = indexGroup.sstableContextManager().size();
assertEquals("Expected " + sstableContextCount +" SSTableContexts, but got " + contextCount, sstableContextCount, contextCount);
StorageAttachedIndex sai = (StorageAttachedIndex) cfs.indexManager.getIndexByName(indexName);
Collection<SSTableIndex> sstableIndexes = sai == null ? Collections.emptyList() : sai.getIndexContext().getView().getIndexes();
StorageAttachedIndex sai = (StorageAttachedIndex) cfs.indexManager.getIndexByName(indexIdentifier.indexName);
Collection<SSTableIndex> sstableIndexes = sai == null ? Collections.emptyList() : sai.view().getIndexes();
assertEquals("Expected " + sstableIndexCount +" SSTableIndexes, but got " + sstableIndexes.toString(), sstableIndexCount, sstableIndexes.size());
}

3
test/unit/org/apache/cassandra/index/sai/analyzer/NonTokenizingAnalyzerTest.java
View File

@ -23,6 +23,7 @@ import java.nio.ByteBuffer;
import org.junit.Test;
import org.apache.cassandra.db.marshal.UTF8Type;
import org.apache.cassandra.index.sai.SAITester;
import org.apache.cassandra.utils.ByteBufferUtil;
import static org.junit.Assert.assertEquals;
@ -68,7 +69,7 @@ public class NonTokenizingAnalyzerTest
private String getAnalyzedString(String input, NonTokenizingOptions options) throws Exception
{
NonTokenizingAnalyzer analyzer = new NonTokenizingAnalyzer(UTF8Type.instance, options);
NonTokenizingAnalyzer analyzer = new NonTokenizingAnalyzer(SAITester.createIndexTermType(UTF8Type.instance), options);
analyzer.reset(ByteBuffer.wrap(input.getBytes()));
return analyzer.hasNext() ? ByteBufferUtil.string(analyzer.next) : null;
}

245
test/unit/org/apache/cassandra/index/sai/cql/StorageAttachedIndexDDLTest.java
View File

@ -54,20 +54,20 @@ import org.apache.cassandra.db.compaction.CompactionManager;
import org.apache.cassandra.db.compaction.OperationType;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.db.marshal.Int32Type;
import org.apache.cassandra.db.marshal.ReversedType;
import org.apache.cassandra.db.marshal.UTF8Type;
import org.apache.cassandra.exceptions.InvalidRequestException;
import org.apache.cassandra.index.Index;
import org.apache.cassandra.index.SecondaryIndexManager;
import org.apache.cassandra.index.sai.IndexContext;
import org.apache.cassandra.index.sai.SAITester;
import org.apache.cassandra.index.sai.StorageAttachedIndex;
import org.apache.cassandra.index.sai.StorageAttachedIndexBuilder;
import org.apache.cassandra.index.sai.analyzer.NonTokenizingOptions;
import org.apache.cassandra.index.sai.disk.format.IndexComponent;
import org.apache.cassandra.index.sai.disk.format.Version;
import org.apache.cassandra.index.sai.disk.v1.segment.SegmentBuilder;
import org.apache.cassandra.index.sai.disk.v1.bitpack.NumericValuesWriter;
import org.apache.cassandra.index.sai.disk.v1.segment.SegmentBuilder;
import org.apache.cassandra.index.sai.utils.IndexIdentifier;
import org.apache.cassandra.index.sai.utils.IndexTermType;
import org.apache.cassandra.index.sai.view.View;
import org.apache.cassandra.inject.ActionBuilder;
import org.apache.cassandra.inject.Expression;
@ -75,7 +75,6 @@ import org.apache.cassandra.inject.Injection;
import org.apache.cassandra.inject.Injections;
import org.apache.cassandra.inject.InvokePointBuilder;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.schema.IndexMetadata;
import org.apache.cassandra.schema.SchemaConstants;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.utils.FBUtilities;
@ -468,9 +467,9 @@ public class StorageAttachedIndexDDLTest extends SAITester
SecondaryIndexManager sim = getCurrentColumnFamilyStore().indexManager;
StorageAttachedIndex index = (StorageAttachedIndex) sim.getIndexByName(indexNameCk1);
IndexContext context = index.getIndexContext();
assertTrue(context.isLiteral());
assertTrue(context.getValidator() instanceof ReversedType);
IndexTermType indexTermType = index.termType();
assertTrue(indexTermType.isLiteral());
assertTrue(indexTermType.isReversed());
}
@Test
@ -676,15 +675,18 @@ public class StorageAttachedIndexDDLTest extends SAITester
{
createTable(CREATE_TABLE_TEMPLATE);
String numericIndexName = createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1"));
String literalIndexName = createIndex(String.format(CREATE_INDEX_TEMPLATE, "v2"));
IndexContext numericIndexContext = createIndexContext(numericIndexName, Int32Type.instance);
IndexContext literalIndexContext = createIndexContext(literalIndexName, UTF8Type.instance);
verifyIndexFiles(numericIndexContext, literalIndexContext, 0, 0);
IndexIdentifier numericIndexIdentifier = createIndexIdentifier(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1")));
IndexIdentifier literalIndexIdentifier = createIndexIdentifier(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v2")));
IndexTermType numericIndexTermType = createIndexTermType(Int32Type.instance);
IndexTermType literalIndexTermType = createIndexTermType(UTF8Type.instance);
verifyIndexFiles(numericIndexTermType, numericIndexIdentifier, 0);
verifyIndexFiles(literalIndexTermType, literalIndexIdentifier, 0);
execute("INSERT INTO %s (id1, v1, v2) VALUES ('0', 0, '0')");
flush();
verifyIndexFiles(numericIndexContext, literalIndexContext, 1, 1);
verifyIndexFiles(numericIndexTermType, numericIndexIdentifier, 1);
verifyIndexFiles(literalIndexTermType, literalIndexIdentifier, 1);
ResultSet rows = executeNet("SELECT id1 FROM %s WHERE v1>=0");
assertEquals(1, rows.all().size());
rows = executeNet("SELECT id1 FROM %s WHERE v2='0'");
@ -692,29 +694,34 @@ public class StorageAttachedIndexDDLTest extends SAITester
execute("INSERT INTO %s (id1, v1, v2) VALUES ('1', 1, '0')");
flush();
verifyIndexFiles(numericIndexContext, literalIndexContext, 2, 2);
verifySSTableIndexes(numericIndexName, 2, 2);
verifySSTableIndexes(literalIndexName, 2, 2);
verifyIndexFiles(numericIndexTermType, numericIndexIdentifier, 2);
verifyIndexFiles(literalIndexTermType, literalIndexIdentifier, 2);
verifySSTableIndexes(numericIndexIdentifier, 2, 2);
verifySSTableIndexes(literalIndexIdentifier, 2, 2);
rows = executeNet("SELECT id1 FROM %s WHERE v1>=0");
assertEquals(2, rows.all().size());
rows = executeNet("SELECT id1 FROM %s WHERE v2='0'");
assertEquals(2, rows.all().size());
dropIndex("DROP INDEX %s." + numericIndexName);
verifyIndexFiles(numericIndexContext, literalIndexContext, 0, 2);
verifySSTableIndexes(numericIndexName, 2, 0);
verifySSTableIndexes(literalIndexName, 2, 2);
dropIndex(numericIndexIdentifier);
verifyIndexFiles(numericIndexTermType, numericIndexIdentifier, 2, 0, 0);
verifyIndexFiles(literalIndexTermType, literalIndexIdentifier, 2, 2, 2);
verifySSTableIndexes(numericIndexIdentifier, 2, 0);
verifySSTableIndexes(literalIndexIdentifier, 2, 2);
rows = executeNet("SELECT id1 FROM %s WHERE v2='0'");
assertEquals(2, rows.all().size());
execute("INSERT INTO %s (id1, v1, v2) VALUES ('2', 2, '0')");
flush();
verifyIndexFiles(numericIndexContext, literalIndexContext, 0, 3);
verifyIndexFiles(numericIndexTermType, numericIndexIdentifier, 3, 0, 0);
verifyIndexFiles(literalIndexTermType, literalIndexIdentifier, 3, 3, 3);
rows = executeNet("SELECT id1 FROM %s WHERE v2='0'");
assertEquals(3, rows.all().size());
dropIndex("DROP INDEX %s." + literalIndexName);
verifyIndexFiles(numericIndexContext, literalIndexContext, 0, 0);
dropIndex(literalIndexIdentifier);
verifyIndexFiles(numericIndexTermType, numericIndexIdentifier, 0);
verifyIndexFiles(literalIndexTermType, literalIndexIdentifier, 0);
assertNull(getCurrentIndexGroup());
assertEquals("Segment memory limiter should revert to zero on drop.", 0L, getSegmentBufferUsedBytes());
@ -735,10 +742,14 @@ public class StorageAttachedIndexDDLTest extends SAITester
flush();
verifyNoIndexFiles();
IndexContext numericIndexContext = createIndexContext(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1")), Int32Type.instance);
IndexContext literalIndexContext = createIndexContext(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v2")), UTF8Type.instance);
waitForTableIndexesQueryable();
verifyIndexFiles(numericIndexContext, literalIndexContext, 2, 2);
IndexIdentifier numericIndexIdentifier = createIndexIdentifier(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1")));
IndexIdentifier literalIndexIdentifier = createIndexIdentifier(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v2")));
IndexTermType numericIndexTermType = createIndexTermType(Int32Type.instance);
IndexTermType literalIndexTermType = createIndexTermType(UTF8Type.instance);
verifyIndexFiles(numericIndexTermType, numericIndexIdentifier, 2);
verifyIndexFiles(literalIndexTermType, literalIndexIdentifier, 2);
ResultSet rows = executeNet("SELECT id1 FROM %s WHERE v1>=0");
assertEquals(2, rows.all().size());
rows = executeNet("SELECT id1 FROM %s WHERE v2='0'");
@ -762,15 +773,20 @@ public class StorageAttachedIndexDDLTest extends SAITester
flush();
verifyNoIndexFiles();
IndexContext numericIndexContext = createIndexContext(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1")), Int32Type.instance);
waitForTableIndexesQueryable();
verifyIndexFiles(numericIndexContext, null, 2, 0);
IndexIdentifier numericIndexIdentifier = createIndexIdentifier(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1")));
IndexTermType numericIndexTermType = createIndexTermType(Int32Type.instance);
verifyIndexFiles(numericIndexTermType, numericIndexIdentifier, 2);
ResultSet rows = executeNet("SELECT id1 FROM %s WHERE v1>=0");
assertEquals(2, rows.all().size());
IndexContext literalIndexContext = createIndexContext(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v2")), UTF8Type.instance);
waitForTableIndexesQueryable();
verifyIndexFiles(numericIndexContext, literalIndexContext, 2, 2);
IndexIdentifier literalIndexIdentifier = createIndexIdentifier(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v2")));
IndexTermType literalIndexTermType = createIndexTermType(UTF8Type.instance);
verifyIndexFiles(numericIndexTermType, numericIndexIdentifier, 2);
verifyIndexFiles(literalIndexTermType, literalIndexIdentifier, 2);
rows = executeNet("SELECT id1 FROM %s WHERE v1>=0");
assertEquals(2, rows.all().size());
rows = executeNet("SELECT id1 FROM %s WHERE v2='0'");
@ -786,13 +802,16 @@ public class StorageAttachedIndexDDLTest extends SAITester
createTable(CREATE_TABLE_TEMPLATE);
disableCompaction(KEYSPACE);
IndexContext numericIndexContext = createIndexContext(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1")), Int32Type.instance);
IndexContext literalIndexContext = createIndexContext(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v2")), UTF8Type.instance);
IndexIdentifier numericIndexIdentifier = createIndexIdentifier(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1")));
IndexIdentifier literalIndexIdentifier = createIndexIdentifier(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v2")));
IndexTermType numericIndexTermType = createIndexTermType(Int32Type.instance);
IndexTermType literalIndexTermType = createIndexTermType(UTF8Type.instance);
verifyNoIndexFiles();
execute("INSERT INTO %s (id1, v1, v2) VALUES ('0', 0, '0');");
flush();
verifyIndexFiles(numericIndexContext, literalIndexContext, 1, 1);
verifyIndexFiles(numericIndexTermType, numericIndexIdentifier, 1);
verifyIndexFiles(literalIndexTermType, literalIndexIdentifier, 1);
ResultSet rows = executeNet("SELECT id1 FROM %s WHERE v1>=0");
assertEquals(1, rows.all().size());
rows = executeNet("SELECT id1 FROM %s WHERE v2='0'");
@ -800,14 +819,16 @@ public class StorageAttachedIndexDDLTest extends SAITester
execute("INSERT INTO %s (id1, v1, v2) VALUES ('1', 1, '0');");
flush();
verifyIndexFiles(numericIndexContext, literalIndexContext, 2, 2);
verifyIndexFiles(numericIndexTermType, numericIndexIdentifier, 2);
verifyIndexFiles(literalIndexTermType, literalIndexIdentifier, 2);
rows = executeNet("SELECT id1 FROM %s WHERE v1>=0");
assertEquals(2, rows.all().size());
rows = executeNet("SELECT id1 FROM %s WHERE v2='0'");
assertEquals(2, rows.all().size());
compact();
waitForAssert(() -> verifyIndexFiles(numericIndexContext, literalIndexContext, 1, 1));
waitForAssert(() -> verifyIndexFiles(numericIndexTermType, numericIndexIdentifier, 1));
waitForAssert(() -> verifyIndexFiles(literalIndexTermType, literalIndexIdentifier, 1));
rows = executeNet("SELECT id1 FROM %s WHERE v1>=0");
assertEquals(2, rows.all().size());
@ -834,10 +855,13 @@ public class StorageAttachedIndexDDLTest extends SAITester
{
createTable(CREATE_TABLE_TEMPLATE);
IndexIdentifier numericIndexIdentifier = null;
IndexIdentifier literalIndexIdentifier = null;
if (!concurrentTruncate)
{
createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1"));
createIndex(String.format(CREATE_INDEX_TEMPLATE, "v2"));
numericIndexIdentifier = createIndexIdentifier(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1")));
literalIndexIdentifier = createIndexIdentifier(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v2")));
}
// create 100 rows, half in sstable and half in memtable
@ -851,8 +875,8 @@ public class StorageAttachedIndexDDLTest extends SAITester
if (concurrentTruncate)
{
createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1"));
createIndex(String.format(CREATE_INDEX_TEMPLATE, "v2"));
numericIndexIdentifier = createIndexIdentifier(createIndexAsync(String.format(CREATE_INDEX_TEMPLATE, "v1")));
literalIndexIdentifier = createIndexIdentifier(createIndexAsync(String.format(CREATE_INDEX_TEMPLATE, "v2")));
truncate(true);
waitForTableIndexesQueryable();
}
@ -864,8 +888,8 @@ public class StorageAttachedIndexDDLTest extends SAITester
waitForAssert(this::verifyNoIndexFiles);
// verify index-view-manager has been cleaned up
verifySSTableIndexes(IndexMetadata.generateDefaultIndexName(currentTable(), V1_COLUMN_IDENTIFIER), 0);
verifySSTableIndexes(IndexMetadata.generateDefaultIndexName(currentTable(), V2_COLUMN_IDENTIFIER), 0);
verifySSTableIndexes(numericIndexIdentifier, 0);
verifySSTableIndexes(literalIndexIdentifier, 0);
assertEquals("Segment memory limiter should revert to zero after truncate.", 0L, getSegmentBufferUsedBytes());
assertEquals("There should be no segment builders in progress.", 0L, getColumnIndexBuildsInProgress());
@ -876,8 +900,8 @@ public class StorageAttachedIndexDDLTest extends SAITester
{
// prepare schema and data
createTable(CREATE_TABLE_TEMPLATE);
String numericIndexName = createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1"));
String stringIndexName = createIndex(String.format(CREATE_INDEX_TEMPLATE, "v2"));
IndexIdentifier numericIndexIdentifier = createIndexIdentifier(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1")));
IndexIdentifier literalIndexIdentifier = createIndexIdentifier(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v2")));
execute("INSERT INTO %s (id1, v1, v2) VALUES ('0', 0, '0');");
execute("INSERT INTO %s (id1, v1, v2) VALUES ('1', 1, '0');");
@ -885,36 +909,39 @@ public class StorageAttachedIndexDDLTest extends SAITester
for (CorruptionType corruptionType : CorruptionType.values())
{
verifyRebuildCorruptedFiles(numericIndexName, stringIndexName, corruptionType, false);
verifyRebuildCorruptedFiles(numericIndexName, stringIndexName, corruptionType, true);
verifyRebuildCorruptedFiles(numericIndexIdentifier, literalIndexIdentifier, corruptionType, false);
verifyRebuildCorruptedFiles(numericIndexIdentifier, literalIndexIdentifier, corruptionType, true);
}
assertEquals("Segment memory limiter should revert to zero following rebuild.", 0L, getSegmentBufferUsedBytes());
assertEquals("There should be no segment builders in progress.", 0L, getColumnIndexBuildsInProgress());
}
private void verifyRebuildCorruptedFiles(String numericIndexName,
String stringIndexName,
private void verifyRebuildCorruptedFiles(IndexIdentifier numericIndexIdentifier,
IndexIdentifier literalIndexIdentifier,
CorruptionType corruptionType,
boolean rebuild) throws Throwable
{
IndexContext numericIndexContext = createIndexContext(numericIndexName, Int32Type.instance);
IndexContext stringIndexContext = createIndexContext(stringIndexName, UTF8Type.instance);
IndexTermType numericIndexTermType = createIndexTermType(Int32Type.instance);
IndexTermType literalIndexTermType = createIndexTermType(UTF8Type.instance);
for (IndexComponent component : Version.LATEST.onDiskFormat().perSSTableIndexComponents(false))
verifyRebuildIndexComponent(numericIndexContext, stringIndexContext, component, null, corruptionType, true, true, rebuild);
verifyRebuildIndexComponent(numericIndexTermType, numericIndexIdentifier, literalIndexTermType, literalIndexIdentifier, component, null, null, corruptionType, true, true, rebuild);
for (IndexComponent component : Version.LATEST.onDiskFormat().perColumnIndexComponents(numericIndexContext))
verifyRebuildIndexComponent(numericIndexContext, stringIndexContext, component, numericIndexContext, corruptionType, false, true, rebuild);
for (IndexComponent component : Version.LATEST.onDiskFormat().perColumnIndexComponents(numericIndexTermType))
verifyRebuildIndexComponent(numericIndexTermType, numericIndexIdentifier, literalIndexTermType, literalIndexIdentifier, component, numericIndexTermType, numericIndexIdentifier, corruptionType, false, true, rebuild);
for (IndexComponent component : Version.LATEST.onDiskFormat().perColumnIndexComponents(stringIndexContext))
verifyRebuildIndexComponent(numericIndexContext, stringIndexContext, component, stringIndexContext, corruptionType, true, false, rebuild);
for (IndexComponent component : Version.LATEST.onDiskFormat().perColumnIndexComponents(literalIndexTermType))
verifyRebuildIndexComponent(numericIndexTermType, numericIndexIdentifier, literalIndexTermType, literalIndexIdentifier, component, literalIndexTermType, literalIndexIdentifier, corruptionType, true, false, rebuild);
}
private void verifyRebuildIndexComponent(IndexContext numericIndexContext,
IndexContext stringIndexContext,
private void verifyRebuildIndexComponent(IndexTermType numericIndexTermType,
IndexIdentifier numericIndexIdentifier,
IndexTermType literalIndexTermType,
IndexIdentifier literalIndexIdentifier,
IndexComponent component,
IndexContext corruptionContext,
IndexTermType corruptionIndexTermType,
IndexIdentifier corruptionIndexIdentifier,
CorruptionType corruptionType,
boolean failedStringIndex,
boolean failedNumericIndex,
@ -938,11 +965,12 @@ public class StorageAttachedIndexDDLTest extends SAITester
int rowCount = 2;
// initial verification
verifySSTableIndexes(numericIndexContext.getIndexName(), 1);
verifySSTableIndexes(stringIndexContext.getIndexName(), 1);
verifyIndexFiles(numericIndexContext, stringIndexContext, 1, 1, 1, 1, 1);
assertTrue(verifyChecksum(numericIndexContext));
assertTrue(verifyChecksum(numericIndexContext));
verifySSTableIndexes(numericIndexIdentifier, 1);
verifySSTableIndexes(literalIndexIdentifier, 1);
verifyIndexFiles(numericIndexTermType, numericIndexIdentifier, 1);
verifyIndexFiles(literalIndexTermType, literalIndexIdentifier, 1);
assertTrue(verifyChecksum(numericIndexTermType, numericIndexIdentifier));
assertTrue(verifyChecksum(literalIndexTermType, literalIndexIdentifier));
ResultSet rows = executeNet("SELECT id1 FROM %s WHERE v1>=0");
assertEquals(rowCount, rows.all().size());
@ -950,8 +978,8 @@ public class StorageAttachedIndexDDLTest extends SAITester
assertEquals(rowCount, rows.all().size());
// corrupt file
if (corruptionContext != null)
corruptIndexComponent(component, corruptionContext, corruptionType);
if (corruptionIndexTermType != null)
corruptIndexComponent(component, corruptionIndexIdentifier, corruptionType);
else
corruptIndexComponent(component, corruptionType);
@ -961,12 +989,12 @@ public class StorageAttachedIndexDDLTest extends SAITester
boolean expectedLiteralState = !failedStringIndex || isBuildCompletionMarker(component);
assertEquals("Checksum verification for " + component + " should be " + expectedNumericState + " but was " + !expectedNumericState,
expectedNumericState, verifyChecksum(numericIndexContext));
assertEquals(expectedLiteralState, verifyChecksum(stringIndexContext));
expectedNumericState, verifyChecksum(numericIndexTermType, numericIndexIdentifier));
assertEquals(expectedLiteralState, verifyChecksum(literalIndexTermType, literalIndexIdentifier));
if (rebuild)
{
rebuildIndexes(numericIndexContext.getIndexName(), stringIndexContext.getIndexName());
rebuildIndexes(numericIndexIdentifier.indexName, literalIndexIdentifier.indexName);
}
else
{
@ -974,8 +1002,8 @@ public class StorageAttachedIndexDDLTest extends SAITester
reloadSSTableIndex();
// Verify the index cannot be read:
verifySSTableIndexes(numericIndexContext.getIndexName(), Version.LATEST.onDiskFormat().perSSTableIndexComponents(false).contains(component) ? 0 : 1, failedNumericIndex ? 0 : 1);
verifySSTableIndexes(stringIndexContext.getIndexName(), Version.LATEST.onDiskFormat().perSSTableIndexComponents(false).contains(component) ? 0 : 1, failedStringIndex ? 0 : 1);
verifySSTableIndexes(numericIndexIdentifier, Version.LATEST.onDiskFormat().perSSTableIndexComponents(false).contains(component) ? 0 : 1, failedNumericIndex ? 0 : 1);
verifySSTableIndexes(literalIndexIdentifier, Version.LATEST.onDiskFormat().perSSTableIndexComponents(false).contains(component) ? 0 : 1, failedStringIndex ? 0 : 1);
try
{
@ -1004,9 +1032,10 @@ public class StorageAttachedIndexDDLTest extends SAITester
}
// verify indexes are recovered
verifySSTableIndexes(numericIndexContext.getIndexName(), 1);
verifySSTableIndexes(numericIndexContext.getIndexName(), 1);
verifyIndexFiles(numericIndexContext, stringIndexContext, 1, 1, 1, 1, 1);
verifySSTableIndexes(numericIndexIdentifier, 1);
verifySSTableIndexes(numericIndexIdentifier, 1);
verifyIndexFiles(numericIndexTermType, numericIndexIdentifier, 1);
verifyIndexFiles(literalIndexTermType, literalIndexIdentifier, 1);
rows = executeNet("SELECT id1 FROM %s WHERE v1>=0");
assertEquals(rowCount, rows.all().size());
@ -1032,13 +1061,14 @@ public class StorageAttachedIndexDDLTest extends SAITester
try
{
// Create a new index, which will actuate a build compaction and fail, but leave the node running...
IndexContext numericIndexContext = createIndexContext(createIndexAsync(String.format(CREATE_INDEX_TEMPLATE, "v1")), Int32Type.instance);
IndexIdentifier numericIndexIdentifier = createIndexIdentifier(createIndexAsync(String.format(CREATE_INDEX_TEMPLATE, "v1")));
IndexTermType numericIndexTermType = createIndexTermType(Int32Type.instance);
// two index builders running in different compaction threads because of parallelised index initial build
waitForAssert(() -> assertEquals(2, indexBuildCounter.get()));
waitForCompactionsFinished();
// Only token/offset files for the first SSTable in the compaction task should exist, while column-specific files are blown away:
verifyIndexFiles(numericIndexContext, null, 2, 0, 0, 0, 0);
verifyIndexFiles(numericIndexTermType, numericIndexIdentifier, 2, 0, 0);
assertEquals("Segment memory limiter should revert to zero.", 0L, getSegmentBufferUsedBytes());
assertEquals("There should be no segment builders in progress.", 0L, getColumnIndexBuildsInProgress());
@ -1090,8 +1120,10 @@ public class StorageAttachedIndexDDLTest extends SAITester
createTable(CREATE_TABLE_TEMPLATE);
disableCompaction(KEYSPACE);
IndexContext numericIndexContext = createIndexContext(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1")), Int32Type.instance);
IndexContext literalIndexContext = createIndexContext(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v2")), UTF8Type.instance);
IndexIdentifier numericIndexIdentifier = createIndexIdentifier(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1")));
IndexIdentifier literalIndexIdentifier = createIndexIdentifier(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v2")));
IndexTermType numericIndexTermType = createIndexTermType(Int32Type.instance);
IndexTermType literalIndexTermType = createIndexTermType(UTF8Type.instance);
// flush empty index
execute("INSERT INTO %s (id1) VALUES ('0');");
@ -1100,7 +1132,8 @@ public class StorageAttachedIndexDDLTest extends SAITester
execute("INSERT INTO %s (id1) VALUES ('1');");
flush();
verifyIndexFiles(numericIndexContext, literalIndexContext, 2, 0, 0, 2, 2);
verifyIndexFiles(numericIndexTermType, numericIndexIdentifier, 2, 0, 2);
verifyIndexFiles(literalIndexTermType, literalIndexIdentifier, 2, 0, 2);
ResultSet rows = executeNet("SELECT id1 FROM %s WHERE v1>=0");
assertEquals(0, rows.all().size());
@ -1109,7 +1142,8 @@ public class StorageAttachedIndexDDLTest extends SAITester
// compact empty index
compact();
waitForAssert(() -> verifyIndexFiles(numericIndexContext, literalIndexContext, 1, 0, 0, 1, 1));
waitForAssert(() -> verifyIndexFiles(numericIndexTermType, numericIndexIdentifier, 1, 0, 1));
waitForAssert(() -> verifyIndexFiles(literalIndexTermType, literalIndexIdentifier, 1, 0, 1));
rows = executeNet("SELECT id1 FROM %s WHERE v1>=0");
assertEquals(0, rows.all().size());
@ -1170,15 +1204,15 @@ public class StorageAttachedIndexDDLTest extends SAITester
createTable(CREATE_TABLE_TEMPLATE);
disableCompaction(KEYSPACE);
IndexContext numericIndexContext = createIndexContext(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1")), Int32Type.instance);
IndexContext literalIndexContext = createIndexContext(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v2")), UTF8Type.instance);
waitForTableIndexesQueryable();
IndexIdentifier numericIndexIdentifier = createIndexIdentifier(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1")));
IndexIdentifier literalIndexIdentifier = createIndexIdentifier(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v2")));
IndexTermType numericIndexTermType = createIndexTermType(Int32Type.instance);
IndexTermType literalIndexTermType = createIndexTermType(UTF8Type.instance);
populateData.run();
verifySSTableIndexes(IndexMetadata.generateDefaultIndexName(currentTable(), V1_COLUMN_IDENTIFIER), 2, 0);
verifySSTableIndexes(IndexMetadata.generateDefaultIndexName(currentTable(), V2_COLUMN_IDENTIFIER), 2, 0);
verifyIndexFiles(numericIndexContext, literalIndexContext, 2, 0, 0, 2, 2);
verifySSTableIndexes(numericIndexIdentifier, 2, 0);
verifySSTableIndexes(literalIndexIdentifier, 2, 0);
verifyIndexFiles(numericIndexTermType, numericIndexIdentifier, 2, 0, 2);
verifyIndexFiles(literalIndexTermType, literalIndexIdentifier, 2, 0, 2);
ResultSet rows = executeNet("SELECT id1 FROM %s WHERE v1>=0");
assertEquals(0, rows.all().size());
@ -1187,9 +1221,10 @@ public class StorageAttachedIndexDDLTest extends SAITester
// compact empty index
compact();
verifySSTableIndexes(IndexMetadata.generateDefaultIndexName(currentTable(), V1_COLUMN_IDENTIFIER), 1, 0);
verifySSTableIndexes(IndexMetadata.generateDefaultIndexName(currentTable(), V2_COLUMN_IDENTIFIER), 1, 0);
waitForAssert(() -> verifyIndexFiles(numericIndexContext, literalIndexContext, 1, 0, 0, 1, 1));
verifySSTableIndexes(numericIndexIdentifier, 1, 0);
verifySSTableIndexes(literalIndexIdentifier, 1, 0);
waitForAssert(() -> verifyIndexFiles(numericIndexTermType, numericIndexIdentifier, 1, 0, 1));
waitForAssert(() -> verifyIndexFiles(literalIndexTermType, literalIndexIdentifier, 1, 0, 1));
rows = executeNet("SELECT id1 FROM %s WHERE v1>=0");
assertEquals(0, rows.all().size());
@ -1218,10 +1253,10 @@ public class StorageAttachedIndexDDLTest extends SAITester
.build();
Injections.inject(delayIndexBuilderCompletion);
String indexName = createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1"));
IndexIdentifier indexIdentifier = createIndexIdentifier(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1")));
waitForAssert(() -> assertEquals(1, delayIndexBuilderCompletion.getCount()));
dropIndex("DROP INDEX %s." + indexName);
dropIndex(indexIdentifier);
// let blocked builders to continue
delayIndexBuilderCompletion.countDown();
@ -1230,12 +1265,12 @@ public class StorageAttachedIndexDDLTest extends SAITester
delayIndexBuilderCompletion.disable();
assertNull(getCurrentIndexGroup());
assertFalse("Expect index not built", SystemKeyspace.isIndexBuilt(KEYSPACE, indexName));
assertFalse("Expect index not built", SystemKeyspace.isIndexBuilt(KEYSPACE, indexIdentifier.indexName));
// create index again, it should succeed
indexName = createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1"));
indexIdentifier = createIndexIdentifier(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1")));
waitForTableIndexesQueryable();
verifySSTableIndexes(indexName, 1);
verifySSTableIndexes(indexIdentifier, 1);
ResultSet rows = executeNet("SELECT id1 FROM %s WHERE v1>=0");
assertEquals(num, rows.all().size());
@ -1263,7 +1298,9 @@ public class StorageAttachedIndexDDLTest extends SAITester
Injections.inject(delayIndexBuilderCompletion);
IndexContext numericIndexContext = createIndexContext(createIndexAsync(String.format(CREATE_INDEX_TEMPLATE, "v1")), Int32Type.instance);
IndexIdentifier numericIndexIdentifier = createIndexIdentifier(createIndexAsync(String.format(CREATE_INDEX_TEMPLATE, "v1")));
IndexTermType numericIndexTermType = createIndexTermType(Int32Type.instance);
waitForAssert(() -> assertTrue(getCompactionTasks() > 0), 1000, TimeUnit.MILLISECONDS);
@ -1287,16 +1324,16 @@ public class StorageAttachedIndexDDLTest extends SAITester
delayIndexBuilderCompletion.disable();
// initial index builder should have stopped abruptly resulting in the index not being queryable
verifyInitialIndexFailed(numericIndexContext.getIndexName());
verifyInitialIndexFailed(numericIndexIdentifier.indexName);
Assertions.assertThat(getNotQueryableIndexes()).isNotEmpty();
ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(currentTable());
for (Index i : cfs.indexManager.listIndexes())
{
StorageAttachedIndex index = (StorageAttachedIndex) i;
assertEquals(0, index.getIndexContext().getMemtableIndexManager().size());
assertEquals(0, index.memtableIndexManager().size());
View view = index.getIndexContext().getView();
View view = index.view();
assertTrue("Expect index build stopped", view.getIndexes().isEmpty());
}
@ -1304,16 +1341,16 @@ public class StorageAttachedIndexDDLTest extends SAITester
assertEquals("There should be no segment builders in progress.", 0L, getColumnIndexBuildsInProgress());
// rebuild index
ColumnFamilyStore.rebuildSecondaryIndex(KEYSPACE, currentTable(), numericIndexContext.getIndexName());
ColumnFamilyStore.rebuildSecondaryIndex(KEYSPACE, currentTable(), numericIndexIdentifier.indexName);
verifyIndexFiles(numericIndexContext, null, sstable, 0);
verifyIndexFiles(numericIndexTermType, numericIndexIdentifier, sstable);
ResultSet rows = executeNet("SELECT id1 FROM %s WHERE v1>=0");
assertEquals(num, rows.all().size());
assertEquals("Segment memory limiter should revert to zero following rebuild.", 0L, getSegmentBufferUsedBytes());
assertEquals("There should be no segment builders in progress.", 0L, getColumnIndexBuildsInProgress());
assertTrue(verifyChecksum(numericIndexContext));
assertTrue(verifyChecksum(numericIndexTermType, numericIndexIdentifier));
}
@Test

106
test/unit/org/apache/cassandra/index/sai/cql/UnindexedExpressionsTest.java Normal file
View File

@ -0,0 +1,106 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.index.sai.cql;
import org.junit.Test;
import org.apache.cassandra.cql3.restrictions.StatementRestrictions;
import org.apache.cassandra.index.sai.SAITester;
import static org.junit.Assert.assertTrue;
public class UnindexedExpressionsTest extends SAITester
{
@Test
public void inOperatorIsNotHandledByIndexTest() throws Throwable
{
createTable("CREATE TABLE %s (k int PRIMARY KEY, v1 int, v2 int)");
createIndex("CREATE INDEX ON %s(v1) USING 'sai'");
createIndex("CREATE INDEX ON %s(v2) USING 'sai'");
execute("INSERT INTO %s (k, v1, v2) VALUES (1, 1, 1)");
execute("INSERT INTO %s (k, v1, v2) VALUES (2, 2, 2)");
execute("INSERT INTO %s (k, v1, v2) VALUES (3, 3, 1)");
execute("INSERT INTO %s (k, v1, v2) VALUES (4, 4, 2)");
assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE, "SELECT * FROM %s WHERE v1 IN (1, 2)");
assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE, "SELECT * FROM %s WHERE v1 IN (1, 2) AND v2 = 1");
assertRowsIgnoringOrder(execute("SELECT * FROM %s WHERE v1 IN (1, 2) AND v2 = 1 ALLOW FILTERING"), row(1, 1, 1));
assertRowsIgnoringOrder(execute("SELECT * FROM %s WHERE v1 IN (1, 2) AND v2 = 2 ALLOW FILTERING"), row(2, 2, 2));
assertTrue(execute("SELECT * FROM %s WHERE v1 IN (1, 3) AND v2 = 2 ALLOW FILTERING").isEmpty());
}
@Test
public void unsupportedOperatorsAreHandledTest() throws Throwable
{
createTable("CREATE TABLE %s (pk int primary key, val1 int, val2 text)");
createIndex("CREATE INDEX ON %s(val1) USING 'sai'");
createIndex("CREATE INDEX ON %s(val2) USING 'sai'");
execute("INSERT INTO %s (pk, val1, val2) VALUES (1, 1, '11')");
execute("INSERT INTO %s (pk, val1, val2) VALUES (2, 2, '22')");
execute("INSERT INTO %s (pk, val1, val2) VALUES (3, 3, '33')");
execute("INSERT INTO %s (pk, val1, val2) VALUES (4, 4, '44')");
// The LIKE operator is rejected because it needs to be handled by an index
assertInvalidMessage("LIKE restriction is only supported on properly indexed columns",
"SELECT pk FROM %s WHERE val1 = 1 AND val2 like '1%%'");
// The IS NOT operator is only valid on materialized views
assertInvalidMessage("Unsupported restriction:", "SELECT pk FROM %s WHERE val1 = 1 AND val2 is not null");
// The != operator is currently not supported at all
assertInvalidMessage("Unsupported \"!=\" relation:", "SELECT pk FROM %s WHERE val1 = 1 AND val2 != '22'");
assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE, "SELECT pk FROM %s WHERE val1 = 1 AND val2 < '22'");
assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE, "SELECT pk FROM %s WHERE val1 = 1 AND val2 <= '11'");
assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE, "SELECT pk FROM %s WHERE val1 = 1 AND val2 >= '11'");
assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE, "SELECT pk FROM %s WHERE val1 = 1 AND val2 > '00'");
assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE, "SELECT pk FROM %s WHERE val1 = 1 AND val2 in ('11', '22')");
assertRows(execute("SELECT pk FROM %s WHERE val1 >= 1 AND val2 < '22' ALLOW FILTERING"), row(1));
assertRows(execute("SELECT pk FROM %s WHERE val1 >= 1 AND val2 <= '11' ALLOW FILTERING"), row(1));
assertRows(execute("SELECT pk FROM %s WHERE val1 >= 1 AND val2 >= '11' AND val2 <= '22' ALLOW FILTERING"), row(1), row(2));
assertRows(execute("SELECT pk FROM %s WHERE val1 >= 1 AND val2 > '00' AND val2 <= '11' ALLOW FILTERING"), row(1));
assertRows(execute("SELECT pk FROM %s WHERE val1 >= 1 AND val2 in ('11', '22') ALLOW FILTERING"), row(1), row(2));
}
@Test
public void unindexedMapColumnTest() throws Throwable
{
createTable("CREATE TABLE %s (pk int primary key, val1 int, val2 map<int, text>)");
createIndex("CREATE INDEX ON %s(val1) USING 'sai'");
execute("INSERT INTO %s (pk, val1, val2) VALUES (1, 1, {1 : '1', 2 : '2', 3 : '3'})");
execute("INSERT INTO %s (pk, val1, val2) VALUES (2, 2, {2 : '2', 3 : '3', 4 : '4'})");
execute("INSERT INTO %s (pk, val1, val2) VALUES (3, 3, {3 : '3', 4 : '4', 5 : '5'})");
execute("INSERT INTO %s (pk, val1, val2) VALUES (4, 4, {4 : '4', 5 : '5', 6 : '6'})");
assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE, "SELECT pk FROM %s WHERE val1 = 1 AND val2 CONTAINS KEY 1");
assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE, "SELECT pk FROM %s WHERE val1 = 1 AND val2 CONTAINS '2'");
assertRows(execute("SELECT pk FROM %s WHERE val1 >= 1 AND val2 CONTAINS KEY 2 ALLOW FILTERING"), row(1), row(2));
assertRows(execute("SELECT pk FROM %s WHERE val1 >= 1 AND val2 CONTAINS '2' ALLOW FILTERING"), row(1), row(2));
assertRows(execute("SELECT pk FROM %s WHERE val1 >= 1 AND val2[2] = '2' ALLOW FILTERING"), row(1), row(2));
}
}

8
test/unit/org/apache/cassandra/index/sai/cql/types/DataSet.java
View File

@ -34,7 +34,7 @@ import java.util.concurrent.TimeUnit;
import org.apache.cassandra.db.marshal.InetAddressType;
import org.apache.cassandra.index.sai.SAITester;
import org.apache.cassandra.index.sai.utils.TypeUtil;
import org.apache.cassandra.index.sai.utils.IndexTermType;
import org.apache.cassandra.serializers.SimpleDateSerializer;
import org.apache.cassandra.serializers.TimeSerializer;
import org.apache.cassandra.utils.TimeUUID;
@ -678,9 +678,9 @@ public abstract class DataSet<T> extends SAITester
while (list.contains(value));
values[index] = value;
}
Arrays.sort(values, (o1, o2) -> TypeUtil.compare(TypeUtil.asIndexBytes(ByteBuffer.wrap(o1.getAddress()), InetAddressType.instance),
TypeUtil.asIndexBytes(ByteBuffer.wrap(o2.getAddress()), InetAddressType.instance),
InetAddressType.instance));
IndexTermType indexTermType = createIndexTermType(InetAddressType.instance);
Arrays.sort(values, (o1, o2) -> indexTermType.compare(indexTermType.asIndexBytes(ByteBuffer.wrap(o1.getAddress())),
indexTermType.asIndexBytes(ByteBuffer.wrap(o2.getAddress()))));
}
@Override

18
test/unit/org/apache/cassandra/index/sai/cql/types/NumericTypeSortingTest.java
View File

@ -27,8 +27,8 @@ import org.junit.Test;
import org.apache.cassandra.db.marshal.DecimalType;
import org.apache.cassandra.db.marshal.IntegerType;
import org.apache.cassandra.index.sai.utils.IndexTermType;
import org.apache.cassandra.index.sai.utils.SAIRandomizedTester;
import org.apache.cassandra.index.sai.utils.TypeUtil;
import static org.junit.Assert.assertTrue;
@ -48,6 +48,8 @@ public class NumericTypeSortingTest extends SAIRandomizedTester
data[i] = randomNumber;
}
IndexTermType indexTermType = createIndexTermType(DecimalType.instance);
Arrays.sort(data, BigDecimal::compareTo);
for (int i = 1; i < data.length; i++)
@ -56,11 +58,11 @@ public class NumericTypeSortingTest extends SAIRandomizedTester
BigDecimal i1 = data[i];
assertTrue(i0 + " <= " + i1, i0.compareTo(i1) <= 0);
ByteBuffer b0 = TypeUtil.asIndexBytes(DecimalType.instance.decompose(i0), DecimalType.instance);
ByteBuffer b0 = indexTermType.asIndexBytes(DecimalType.instance.decompose(i0));
ByteBuffer b1 = TypeUtil.asIndexBytes(DecimalType.instance.decompose(i1), DecimalType.instance);
ByteBuffer b1 = indexTermType.asIndexBytes(DecimalType.instance.decompose(i1));
assertTrue(i0 + " <= " + i1, TypeUtil.compare(b0, b1, DecimalType.instance) <= 0);
assertTrue(i0 + " <= " + i1, indexTermType.compare(b0, b1) <= 0);
}
}
@ -80,17 +82,19 @@ public class NumericTypeSortingTest extends SAIRandomizedTester
Arrays.sort(data, BigInteger::compareTo);
IndexTermType indexTermType = createIndexTermType(IntegerType.instance);
for (int i = 1; i < data.length; i++)
{
BigInteger i0 = data[i - 1];
BigInteger i1 = data[i];
assertTrue(i0 + " <= " + i1, i0.compareTo(i1) <= 0);
ByteBuffer b0 = TypeUtil.asIndexBytes(IntegerType.instance.decompose(i0), IntegerType.instance);
ByteBuffer b0 = indexTermType.asIndexBytes(IntegerType.instance.decompose(i0));
ByteBuffer b1 = TypeUtil.asIndexBytes(IntegerType.instance.decompose(i1), IntegerType.instance);
ByteBuffer b1 = indexTermType.asIndexBytes(IntegerType.instance.decompose(i1));
assertTrue(i0 + " <= " + i1, TypeUtil.compare(b0, b1, IntegerType.instance) <= 0);
assertTrue(i0 + " <= " + i1, indexTermType.compare(b0, b1) <= 0);
}
}
}

28
test/unit/org/apache/cassandra/index/sai/disk/NodeStartupTest.java
View File

@ -35,14 +35,15 @@ import org.junit.runners.Parameterized;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.marshal.Int32Type;
import org.apache.cassandra.index.SecondaryIndexManager;
import org.apache.cassandra.index.sai.IndexContext;
import org.apache.cassandra.index.sai.SAITester;
import org.apache.cassandra.index.sai.StorageAttachedIndex;
import org.apache.cassandra.index.sai.StorageAttachedIndexBuilder;
import org.apache.cassandra.index.sai.disk.format.IndexComponent;
import org.apache.cassandra.index.sai.disk.format.IndexDescriptor;
import org.apache.cassandra.index.sai.utils.IndexIdentifier;
import org.apache.cassandra.index.sai.disk.format.Version;
import org.apache.cassandra.index.sai.disk.v1.SSTableIndexWriter;
import org.apache.cassandra.index.sai.utils.IndexTermType;
import org.apache.cassandra.inject.Injection;
import org.apache.cassandra.inject.Injections;
import org.apache.cassandra.inject.InvokePointBuilder;
@ -115,7 +116,8 @@ public class NodeStartupTest extends SAITester
private static Throwable error = null;
private IndexContext indexContext = null;
private IndexIdentifier indexIdentifier = null;
private IndexTermType indexTermType = null;
enum Populator
{
@ -181,8 +183,8 @@ public class NodeStartupTest extends SAITester
public void setup() throws Throwable
{
createTable("CREATE TABLE %s (id text PRIMARY KEY, v1 text)");
String indexName = createIndex(String.format("CREATE CUSTOM INDEX ON %%s(v1) USING '%s'", StorageAttachedIndex.class.getName()));
indexContext = createIndexContext(indexName, Int32Type.instance);
indexIdentifier = createIndexIdentifier(createIndex(String.format("CREATE CUSTOM INDEX ON %%s(v1) USING '%s'", StorageAttachedIndex.class.getName())));
indexTermType = createIndexTermType(Int32Type.instance);
Injections.inject(ObjectArrays.concat(barriers, counters, Injection.class));
Stream.of(barriers).forEach(Injections.Barrier::reset);
Stream.of(barriers).forEach(Injections.Barrier::disable);
@ -238,7 +240,7 @@ public class NodeStartupTest extends SAITester
}
@Test
public void startupOrderingTest() throws Throwable
public void startupOrderingTest()
{
populator.populate(this);
@ -326,7 +328,7 @@ public class NodeStartupTest extends SAITester
private boolean isColumnIndexComplete()
{
ColumnFamilyStore cfs = Objects.requireNonNull(Schema.instance.getKeyspaceInstance(KEYSPACE)).getColumnFamilyStore(currentTable());
return cfs.getLiveSSTables().stream().allMatch(sstable -> IndexDescriptor.create(sstable).isPerColumnIndexBuildComplete(indexContext));
return cfs.getLiveSSTables().stream().allMatch(sstable -> IndexDescriptor.create(sstable).isPerColumnIndexBuildComplete(indexIdentifier));
}
private void setState(IndexStateOnRestart state)
@ -337,19 +339,19 @@ public class NodeStartupTest extends SAITester
break;
case ALL_EMPTY:
Version.LATEST.onDiskFormat().perSSTableIndexComponents(false).forEach(this::remove);
Version.LATEST.onDiskFormat().perColumnIndexComponents(indexContext).forEach(c -> remove(c, indexContext));
Version.LATEST.onDiskFormat().perColumnIndexComponents(indexTermType).forEach(c -> remove(c, indexIdentifier));
break;
case PER_SSTABLE_INCOMPLETE:
remove(IndexComponent.GROUP_COMPLETION_MARKER);
break;
case PER_COLUMN_INCOMPLETE:
remove(IndexComponent.COLUMN_COMPLETION_MARKER, indexContext);
remove(IndexComponent.COLUMN_COMPLETION_MARKER, indexIdentifier);
break;
case PER_SSTABLE_CORRUPT:
corrupt();
break;
case PER_COLUMN_CORRUPT:
corrupt(indexContext);
corrupt(indexIdentifier);
break;
}
}
@ -367,11 +369,11 @@ public class NodeStartupTest extends SAITester
}
}
private void remove(IndexComponent component, IndexContext indexContext)
private void remove(IndexComponent component, IndexIdentifier indexIdentifier)
{
try
{
corruptIndexComponent(component, indexContext, CorruptionType.REMOVED);
corruptIndexComponent(component, indexIdentifier, CorruptionType.REMOVED);
}
catch (Exception e)
{
@ -393,11 +395,11 @@ public class NodeStartupTest extends SAITester
}
}
private void corrupt(IndexContext indexContext)
private void corrupt(IndexIdentifier indexIdentifier)
{
try
{
corruptIndexComponent(IndexComponent.META, indexContext, CorruptionType.TRUNCATED_HEADER);
corruptIndexComponent(IndexComponent.META, indexIdentifier, CorruptionType.TRUNCATED_HEADER);
}
catch (Exception e)
{

9
test/unit/org/apache/cassandra/index/sai/disk/format/IndexDescriptorTest.java
View File

@ -30,10 +30,9 @@ import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.marshal.UTF8Type;
import org.apache.cassandra.dht.Murmur3Partitioner;
import org.apache.cassandra.index.sai.IndexContext;
import org.apache.cassandra.index.sai.SAITester;
import org.apache.cassandra.index.sai.utils.IndexIdentifier;
import org.apache.cassandra.io.sstable.Descriptor;
import org.apache.cassandra.io.util.File;
@ -48,7 +47,7 @@ public class IndexDescriptorTest
@BeforeClass
public static void initialise()
{
DatabaseDescriptor.daemonInitialization();
DatabaseDescriptor.toolInitialization();
}
@Before
@ -81,10 +80,10 @@ public class IndexDescriptorTest
createFileOnDisk("-SAI+aa+test_index+ColumnComplete.db");
IndexDescriptor indexDescriptor = IndexDescriptor.create(descriptor, Murmur3Partitioner.instance, SAITester.EMPTY_COMPARATOR);
IndexContext indexContext = SAITester.createIndexContext("test_index", UTF8Type.instance);
IndexIdentifier indexIdentifier = SAITester.createIndexIdentifier("test", "test", "test_index");
assertEquals(Version.AA, indexDescriptor.version);
assertTrue(indexDescriptor.hasComponent(IndexComponent.COLUMN_COMPLETION_MARKER, indexContext));
assertTrue(indexDescriptor.hasComponent(IndexComponent.COLUMN_COMPLETION_MARKER, indexIdentifier));
}
private void createFileOnDisk(String filename) throws Throwable

2
test/unit/org/apache/cassandra/index/sai/disk/format/VersionTest.java
View File

@ -30,7 +30,7 @@ public class VersionTest
@BeforeClass
public static void initialise()
{
DatabaseDescriptor.daemonInitialization();
DatabaseDescriptor.toolInitialization();
}
@Test

40
test/unit/org/apache/cassandra/index/sai/disk/v1/BalancedTreeIndexSearcherTest.java
View File

@ -28,6 +28,7 @@ import com.google.common.collect.Iterators;
import com.google.common.collect.Lists;
import org.junit.Test;
import org.apache.cassandra.cql3.Operator;
import org.apache.cassandra.db.marshal.DecimalType;
import org.apache.cassandra.db.marshal.Int32Type;
import org.apache.cassandra.db.marshal.IntegerType;
@ -111,7 +112,7 @@ public class BalancedTreeIndexSearcherTest extends SAIRandomizedTester
IndexSegmentSearcher indexSearcher = BlockBalancedTreeIndexBuilder.buildDecimalSearcher(newIndexDescriptor(),
BigDecimal.ZERO, BigDecimal.valueOf(10L));
testRangeQueries(indexSearcher, DecimalType.instance, DecimalType.instance, BigDecimal::valueOf,
getLongsOnInterval(21L, 70L));
getLongsOnInterval(20L, 70L));
}
private List<Long> getLongsOnInterval(long lowerInclusive, long upperInclusive)
@ -148,11 +149,9 @@ public class BalancedTreeIndexSearcherTest extends SAIRandomizedTester
final NumberType<T> rawType, final NumberType<?> encodedType,
final Function<Short, T> rawValueProducer) throws Exception
{
try (KeyRangeIterator results = indexSearcher.search(new Expression(SAITester.createIndexContext("meh", rawType))
{{
operator = IndexOperator.EQ;
lower = upper = new Bound(rawType.decompose(rawValueProducer.apply(EQ_TEST_LOWER_BOUND_INCLUSIVE)), encodedType, true);
}}, null, mock(QueryContext.class)))
try (KeyRangeIterator results = indexSearcher.search(Expression.create(SAITester.createIndexTermType(rawType))
.add(Operator.EQ, rawType.decompose(rawValueProducer.apply(EQ_TEST_LOWER_BOUND_INCLUSIVE)))
, null, mock(QueryContext.class)))
{
assertEquals(results.getMinimum(), results.getCurrent());
assertTrue(results.hasNext());
@ -160,11 +159,9 @@ public class BalancedTreeIndexSearcherTest extends SAIRandomizedTester
assertEquals(0L, results.next().token().getLongValue());
}
try (KeyRangeIterator results = indexSearcher.search(new Expression(SAITester.createIndexContext("meh", rawType))
{{
operator = IndexOperator.EQ;
lower = upper = new Bound(rawType.decompose(rawValueProducer.apply(EQ_TEST_UPPER_BOUND_EXCLUSIVE)), encodedType, true);
}}, null, mock(QueryContext.class)))
try (KeyRangeIterator results = indexSearcher.search(Expression.create(SAITester.createIndexTermType(rawType))
.add(Operator.EQ, rawType.decompose(rawValueProducer.apply(EQ_TEST_UPPER_BOUND_EXCLUSIVE))),
null, mock(QueryContext.class)))
{
assertFalse(results.hasNext());
indexSearcher.close();
@ -175,7 +172,7 @@ public class BalancedTreeIndexSearcherTest extends SAIRandomizedTester
final NumberType<T> rawType, final NumberType<?> encodedType,
final Function<Short, T> rawValueProducer) throws Exception
{
List<Long> expectedTokenList = getLongsOnInterval(3L, 7L);
List<Long> expectedTokenList = getLongsOnInterval(2L, 7L);
testRangeQueries(indexSearcher, rawType, encodedType, rawValueProducer, expectedTokenList);
}
@ -184,13 +181,10 @@ public class BalancedTreeIndexSearcherTest extends SAIRandomizedTester
final NumberType<T> rawType, final NumberType<?> encodedType,
final Function<Short, T> rawValueProducer, List<Long> expectedTokenList) throws Exception
{
try (KeyRangeIterator results = indexSearcher.search(new Expression(SAITester.createIndexContext("meh", rawType))
{{
operator = IndexOperator.RANGE;
lower = new Bound(rawType.decompose(rawValueProducer.apply((short)2)), encodedType, false);
upper = new Bound(rawType.decompose(rawValueProducer.apply((short)7)), encodedType, true);
}}, null, mock(QueryContext.class)))
try (KeyRangeIterator results = indexSearcher.search(Expression.create(SAITester.createIndexTermType(rawType))
.add(Operator.GTE, rawType.decompose(rawValueProducer.apply((short)2)))
.add(Operator.LTE, rawType.decompose(rawValueProducer.apply((short)7))),
null, mock(QueryContext.class)))
{
assertEquals(results.getMinimum(), results.getCurrent());
assertTrue(results.hasNext());
@ -199,19 +193,19 @@ public class BalancedTreeIndexSearcherTest extends SAIRandomizedTester
assertEquals(expectedTokenList, actualTokenList);
}
try (KeyRangeIterator results = indexSearcher.search(new Expression(SAITester.createIndexContext("meh", rawType))
try (KeyRangeIterator results = indexSearcher.search(new Expression.IndexedExpression(SAITester.createMockIndex("meh", rawType))
{{
operator = IndexOperator.RANGE;
lower = new Bound(rawType.decompose(rawValueProducer.apply(RANGE_TEST_UPPER_BOUND_EXCLUSIVE)), encodedType, true);
lower = new Bound(rawType.decompose(rawValueProducer.apply(RANGE_TEST_UPPER_BOUND_EXCLUSIVE)), getIndexTermType(), true);
}}, null, mock(QueryContext.class)))
{
assertFalse(results.hasNext());
}
try (KeyRangeIterator results = indexSearcher.search(new Expression(SAITester.createIndexContext("meh", rawType))
try (KeyRangeIterator results = indexSearcher.search(new Expression.IndexedExpression(SAITester.createMockIndex("meh", rawType))
{{
operator = IndexOperator.RANGE;
upper = new Bound(rawType.decompose(rawValueProducer.apply(RANGE_TEST_LOWER_BOUND_INCLUSIVE)), encodedType, false);
upper = new Bound(rawType.decompose(rawValueProducer.apply(RANGE_TEST_LOWER_BOUND_INCLUSIVE)), getIndexTermType(), false);
}}, null, mock(QueryContext.class)))
{
assertFalse(results.hasNext());

42
test/unit/org/apache/cassandra/index/sai/disk/v1/InvertedIndexSearcherTest.java
View File

@ -31,9 +31,9 @@ import org.apache.cassandra.db.ClusteringComparator;
import org.apache.cassandra.db.marshal.UTF8Type;
import org.apache.cassandra.dht.Murmur3Partitioner;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.index.sai.IndexContext;
import org.apache.cassandra.index.sai.QueryContext;
import org.apache.cassandra.index.sai.SAITester;
import org.apache.cassandra.index.sai.StorageAttachedIndex;
import org.apache.cassandra.index.sai.iterators.KeyRangeIterator;
import org.apache.cassandra.index.sai.memory.MemtableTermsIterator;
import org.apache.cassandra.index.sai.disk.PrimaryKeyMap;
@ -100,22 +100,18 @@ public class InvertedIndexSearcherTest extends SAIRandomizedTester
@Test
public void testEqQueriesAgainstStringIndex() throws Exception
{
doTestEqQueriesAgainstStringIndex();
}
private void doTestEqQueriesAgainstStringIndex() throws Exception
{
QueryContext context = mock(QueryContext.class);
final StorageAttachedIndex index = createMockIndex(newIndex(), UTF8Type.instance);
final int numTerms = getRandom().nextIntBetween(64, 512), numPostings = getRandom().nextIntBetween(256, 1024);
final List<Pair<ByteComparable, LongArrayList>> termsEnum = buildTermsEnum(numTerms, numPostings);
try (IndexSegmentSearcher searcher = buildIndexAndOpenSearcher(numTerms, numPostings, termsEnum))
try (IndexSegmentSearcher searcher = buildIndexAndOpenSearcher(index, numTerms, numPostings, termsEnum))
{
for (int t = 0; t < numTerms; ++t)
{
try (KeyRangeIterator results = searcher.search(new Expression(SAITester.createIndexContext("meh", UTF8Type.instance))
.add(Operator.EQ, wrap(termsEnum.get(t).left)), null, context))
try (KeyRangeIterator results = searcher.search(Expression.create(index).add(Operator.EQ, wrap(termsEnum.get(t).left)), null, context))
{
assertEquals(results.getMinimum(), results.getCurrent());
assertTrue(results.hasNext());
@ -130,8 +126,7 @@ public class InvertedIndexSearcherTest extends SAIRandomizedTester
assertFalse(results.hasNext());
}
try (KeyRangeIterator results = searcher.search(new Expression(SAITester.createIndexContext("meh", UTF8Type.instance))
.add(Operator.EQ, wrap(termsEnum.get(t).left)), null, context))
try (KeyRangeIterator results = searcher.search(Expression.create(index).add(Operator.EQ, wrap(termsEnum.get(t).left)), null, context))
{
assertEquals(results.getMinimum(), results.getCurrent());
assertTrue(results.hasNext());
@ -153,13 +148,11 @@ public class InvertedIndexSearcherTest extends SAIRandomizedTester
// try searching for terms that weren't indexed
final String tooLongTerm = randomSimpleString(10, 12);
KeyRangeIterator results = searcher.search(new Expression(SAITester.createIndexContext("meh", UTF8Type.instance))
.add(Operator.EQ, UTF8Type.instance.decompose(tooLongTerm)), null, context);
KeyRangeIterator results = searcher.search(Expression.create(index).add(Operator.EQ, UTF8Type.instance.decompose(tooLongTerm)), null, context);
assertFalse(results.hasNext());
final String tooShortTerm = randomSimpleString(1, 2);
results = searcher.search(new Expression(SAITester.createIndexContext("meh", UTF8Type.instance))
.add(Operator.EQ, UTF8Type.instance.decompose(tooShortTerm)), null, context);
results = searcher.search(Expression.create(index).add(Operator.EQ, UTF8Type.instance.decompose(tooShortTerm)), null, context);
assertFalse(results.hasNext());
}
}
@ -168,14 +161,14 @@ public class InvertedIndexSearcherTest extends SAIRandomizedTester
public void testUnsupportedOperator() throws Exception
{
QueryContext context = mock(QueryContext.class);
final StorageAttachedIndex index = createMockIndex(newIndex(), UTF8Type.instance);
final int numTerms = getRandom().nextIntBetween(5, 15), numPostings = getRandom().nextIntBetween(5, 20);
final List<Pair<ByteComparable, LongArrayList>> termsEnum = buildTermsEnum(numTerms, numPostings);
try (IndexSegmentSearcher searcher = buildIndexAndOpenSearcher(numTerms, numPostings, termsEnum))
try (IndexSegmentSearcher searcher = buildIndexAndOpenSearcher(index, numTerms, numPostings, termsEnum))
{
searcher.search(new Expression(SAITester.createIndexContext("meh", UTF8Type.instance))
.add(Operator.GT, UTF8Type.instance.decompose("a")), null, context);
searcher.search(Expression.create(index).add(Operator.GT, UTF8Type.instance.decompose("a")), null, context);
fail("Expect IllegalArgumentException thrown, but didn't");
}
@ -185,15 +178,16 @@ public class InvertedIndexSearcherTest extends SAIRandomizedTester
}
}
private IndexSegmentSearcher buildIndexAndOpenSearcher(int terms, int postings, List<Pair<ByteComparable, LongArrayList>> termsEnum) throws IOException
private IndexSegmentSearcher buildIndexAndOpenSearcher(StorageAttachedIndex index,
int terms,
int postings,
List<Pair<ByteComparable, LongArrayList>> termsEnum) throws IOException
{
final int size = terms * postings;
final IndexDescriptor indexDescriptor = newIndexDescriptor();
final String index = newIndex();
final IndexContext indexContext = SAITester.createIndexContext(index, UTF8Type.instance);
SegmentMetadata.ComponentMetadataMap indexMetas;
try (LiteralIndexWriter writer = new LiteralIndexWriter(indexDescriptor, indexContext))
try (LiteralIndexWriter writer = new LiteralIndexWriter(indexDescriptor, index.identifier()))
{
indexMetas = writer.writeCompleteSegment(new MemtableTermsIterator(null, null, termsEnum.iterator()));
}
@ -208,12 +202,12 @@ public class InvertedIndexSearcherTest extends SAIRandomizedTester
wrap(termsEnum.get(terms - 1).left),
indexMetas);
try (PerColumnIndexFiles indexFiles = new PerColumnIndexFiles(indexDescriptor, indexContext))
try (PerColumnIndexFiles indexFiles = new PerColumnIndexFiles(indexDescriptor, index.termType(), index.identifier()))
{
final IndexSegmentSearcher searcher = IndexSegmentSearcher.open(TEST_PRIMARY_KEY_MAP_FACTORY,
indexFiles,
segmentMetadata,
SAITester.createIndexContext(index, UTF8Type.instance));
index);
assertThat(searcher, is(instanceOf(LiteralIndexSegmentSearcher.class)));
return searcher;
}

23
test/unit/org/apache/cassandra/index/sai/disk/v1/MetadataTest.java
View File

@ -29,11 +29,9 @@ import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
import org.apache.cassandra.db.marshal.UTF8Type;
import org.apache.cassandra.index.sai.IndexContext;
import org.apache.cassandra.index.sai.SAITester;
import org.apache.cassandra.index.sai.disk.format.IndexComponent;
import org.apache.cassandra.index.sai.disk.format.IndexDescriptor;
import org.apache.cassandra.index.sai.utils.IndexIdentifier;
import org.apache.cassandra.index.sai.disk.io.IndexOutputWriter;
import org.apache.cassandra.index.sai.utils.SAIRandomizedTester;
import org.apache.cassandra.io.util.File;
@ -52,21 +50,20 @@ public class MetadataTest extends SAIRandomizedTester
public final ExpectedException expectedException = ExpectedException.none();
private IndexDescriptor indexDescriptor;
private IndexContext indexContext;
private IndexIdentifier indexIdentifier;
@Before
public void setup() throws Throwable
{
indexDescriptor = newIndexDescriptor();
String index = newIndex();
indexContext = SAITester.createIndexContext(index, UTF8Type.instance);
indexIdentifier = createIndexIdentifier("test", "test", newIndex());
}
@Test
public void shouldReadWrittenMetadata() throws Exception
{
final Map<String, byte[]> data = new HashMap<>();
try (MetadataWriter writer = new MetadataWriter(indexDescriptor.openPerIndexOutput(IndexComponent.META, indexContext)))
try (MetadataWriter writer = new MetadataWriter(indexDescriptor.openPerIndexOutput(IndexComponent.META, indexIdentifier)))
{
int num = nextInt(1, 50);
for (int x = 0; x < num; x++)
@ -82,7 +79,7 @@ public class MetadataTest extends SAIRandomizedTester
}
}
}
MetadataSource reader = MetadataSource.loadColumnMetadata(indexDescriptor, indexContext);
MetadataSource reader = MetadataSource.loadColumnMetadata(indexDescriptor, indexIdentifier);
for (Map.Entry<String, byte[]> entry : data.entrySet())
{
@ -98,7 +95,7 @@ public class MetadataTest extends SAIRandomizedTester
@Test
public void shouldFailWhenFileHasNoHeader() throws IOException
{
try (IndexOutputWriter out = indexDescriptor.openPerIndexOutput(IndexComponent.META, indexContext))
try (IndexOutputWriter out = indexDescriptor.openPerIndexOutput(IndexComponent.META, indexIdentifier))
{
final byte[] bytes = nextBytes(13, 29);
out.writeBytes(bytes, bytes.length);
@ -106,7 +103,7 @@ public class MetadataTest extends SAIRandomizedTester
expectedException.expect(CorruptIndexException.class);
expectedException.expectMessage("codec header mismatch");
MetadataSource.loadColumnMetadata(indexDescriptor, indexContext);
MetadataSource.loadColumnMetadata(indexDescriptor, indexIdentifier);
}
@Test
@ -130,7 +127,7 @@ public class MetadataTest extends SAIRandomizedTester
expectedException.expect(CorruptIndexException.class);
expectedException.expectMessage("misplaced codec footer (file truncated?)");
MetadataSource.loadColumnMetadata(indexDescriptor, indexContext);
MetadataSource.loadColumnMetadata(indexDescriptor, indexIdentifier);
}
}
@ -166,13 +163,13 @@ public class MetadataTest extends SAIRandomizedTester
expectedException.expect(CorruptIndexException.class);
expectedException.expectMessage("checksum failed");
MetadataSource.loadColumnMetadata(indexDescriptor, indexContext);
MetadataSource.loadColumnMetadata(indexDescriptor, indexIdentifier);
}
}
private IndexOutputWriter writeRandomBytes() throws IOException
{
final IndexOutputWriter output = indexDescriptor.openPerIndexOutput(IndexComponent.META, indexContext);
final IndexOutputWriter output = indexDescriptor.openPerIndexOutput(IndexComponent.META, indexIdentifier);
try (MetadataWriter writer = new MetadataWriter(output))
{
byte[] bytes = nextBytes(11, 1024);

32
test/unit/org/apache/cassandra/index/sai/disk/v1/SegmentFlushTest.java
View File

@ -31,19 +31,18 @@ import org.junit.BeforeClass;
import org.junit.Test;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.cql3.statements.schema.IndexTarget;
import org.apache.cassandra.db.Clustering;
import org.apache.cassandra.db.ClusteringComparator;
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.db.marshal.UTF8Type;
import org.apache.cassandra.db.rows.BTreeRow;
import org.apache.cassandra.db.rows.BufferCell;
import org.apache.cassandra.db.rows.Row;
import org.apache.cassandra.dht.Murmur3Partitioner;
import org.apache.cassandra.index.sai.IndexContext;
import org.apache.cassandra.index.sai.SAITester;
import org.apache.cassandra.index.sai.StorageAttachedIndex;
import org.apache.cassandra.index.sai.disk.format.IndexComponent;
import org.apache.cassandra.index.sai.disk.format.IndexDescriptor;
import org.apache.cassandra.index.sai.utils.IndexIdentifier;
import org.apache.cassandra.index.sai.disk.v1.segment.SegmentBuilder;
import org.apache.cassandra.index.sai.disk.v1.segment.SegmentMetadata;
import org.apache.cassandra.index.sai.postings.PostingList;
@ -54,7 +53,6 @@ import org.apache.cassandra.io.sstable.SequenceBasedSSTableId;
import org.apache.cassandra.io.util.File;
import org.apache.cassandra.io.util.FileHandle;
import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.schema.IndexMetadata;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.utils.bytecomparable.ByteComparable;
@ -76,7 +74,7 @@ public class SegmentFlushTest
@BeforeClass
public static void init()
{
DatabaseDescriptor.daemonInitialization();
DatabaseDescriptor.toolInitialization();
DatabaseDescriptor.setPartitionerUnsafe(Murmur3Partitioner.instance);
StorageService.instance.setPartitionerUnsafe(Murmur3Partitioner.instance);
}
@ -114,18 +112,10 @@ public class SegmentFlushTest
SAITester.EMPTY_COMPARATOR);
ColumnMetadata column = ColumnMetadata.regularColumn("sai", "internal", "column", UTF8Type.instance);
IndexMetadata config = IndexMetadata.fromSchemaMetadata("index_name", IndexMetadata.Kind.CUSTOM, null);
IndexContext indexContext = new IndexContext("ks",
"cf",
UTF8Type.instance,
Murmur3Partitioner.instance,
new ClusteringComparator(),
column,
IndexTarget.Type.SIMPLE,
config);
StorageAttachedIndex index = SAITester.createMockIndex(column);
SSTableIndexWriter writer = new SSTableIndexWriter(indexDescriptor, indexContext, V1OnDiskFormat.SEGMENT_BUILD_MEMORY_LIMITER, () -> true);
SSTableIndexWriter writer = new SSTableIndexWriter(indexDescriptor, index, V1OnDiskFormat.SEGMENT_BUILD_MEMORY_LIMITER, () -> true);
List<DecoratedKey> keys = Arrays.asList(dk("1"), dk("2"));
Collections.sort(keys);
@ -143,7 +133,7 @@ public class SegmentFlushTest
writer.complete(Stopwatch.createStarted());
MetadataSource source = MetadataSource.loadColumnMetadata(indexDescriptor, indexContext);
MetadataSource source = MetadataSource.loadColumnMetadata(indexDescriptor, index.identifier());
List<SegmentMetadata> segmentMetadatas = SegmentMetadata.load(source, indexDescriptor.primaryKeyFactory);
assertEquals(segments, segmentMetadatas.size());
@ -159,7 +149,7 @@ public class SegmentFlushTest
maxTerm = segments == 1 ? term2 : term1;
numRows = segments == 1 ? 2 : 1;
verifySegmentMetadata(segmentMetadata);
verifyStringIndex(indexDescriptor, indexContext, segmentMetadata);
verifyStringIndex(indexDescriptor, index.identifier(), segmentMetadata);
if (segments > 1)
{
@ -174,7 +164,7 @@ public class SegmentFlushTest
segmentMetadata = segmentMetadatas.get(1);
verifySegmentMetadata(segmentMetadata);
verifyStringIndex(indexDescriptor, indexContext, segmentMetadata);
verifyStringIndex(indexDescriptor, index.identifier(), segmentMetadata);
}
}
@ -188,10 +178,10 @@ public class SegmentFlushTest
assertEquals(numRows, segmentMetadata.numRows);
}
private void verifyStringIndex(IndexDescriptor indexDescriptor, IndexContext indexContext, SegmentMetadata segmentMetadata) throws IOException
private void verifyStringIndex(IndexDescriptor indexDescriptor, IndexIdentifier indexIdentifier, SegmentMetadata segmentMetadata) throws IOException
{
FileHandle termsData = indexDescriptor.createPerIndexFileHandle(IndexComponent.TERMS_DATA, indexContext, null);
FileHandle postingLists = indexDescriptor.createPerIndexFileHandle(IndexComponent.POSTING_LISTS, indexContext, null);
FileHandle termsData = indexDescriptor.createPerIndexFileHandle(IndexComponent.TERMS_DATA, indexIdentifier, null);
FileHandle postingLists = indexDescriptor.createPerIndexFileHandle(IndexComponent.POSTING_LISTS, indexIdentifier, null);
try (TermsIterator iterator = new TermsScanner(termsData, postingLists, segmentMetadata.componentMetadatas.get(IndexComponent.TERMS_DATA).root))
{

2
test/unit/org/apache/cassandra/index/sai/disk/v1/SegmentTest.java
View File

@ -46,7 +46,7 @@ public class SegmentTest
@BeforeClass
public static void init()
{
DatabaseDescriptor.daemonInitialization();
DatabaseDescriptor.toolInitialization();
DatabaseDescriptor.setPartitionerUnsafe(Murmur3Partitioner.instance);
partitioner = DatabaseDescriptor.getPartitioner();
min = partitioner.getMinimumToken();

30
test/unit/org/apache/cassandra/index/sai/disk/v1/TermsReaderTest.java
View File

@ -23,20 +23,18 @@ import java.util.List;
import org.junit.Test;
import com.carrotsearch.hppc.LongArrayList;
import org.apache.cassandra.db.marshal.UTF8Type;
import org.apache.cassandra.index.sai.IndexContext;
import org.apache.cassandra.index.sai.QueryContext;
import org.apache.cassandra.index.sai.SAITester;
import org.apache.cassandra.index.sai.memory.MemtableTermsIterator;
import org.apache.cassandra.index.sai.postings.PostingList;
import org.apache.cassandra.index.sai.utils.TermsIterator;
import org.apache.cassandra.index.sai.disk.format.IndexComponent;
import org.apache.cassandra.index.sai.disk.format.IndexDescriptor;
import org.apache.cassandra.index.sai.utils.IndexIdentifier;
import org.apache.cassandra.index.sai.disk.v1.segment.LiteralIndexSegmentTermsReader;
import org.apache.cassandra.index.sai.disk.v1.segment.SegmentMetadata;
import org.apache.cassandra.index.sai.disk.v1.trie.LiteralIndexWriter;
import org.apache.cassandra.index.sai.memory.MemtableTermsIterator;
import org.apache.cassandra.index.sai.metrics.QueryEventListener;
import org.apache.cassandra.index.sai.postings.PostingList;
import org.apache.cassandra.index.sai.utils.SAIRandomizedTester;
import org.apache.cassandra.index.sai.utils.TermsIterator;
import org.apache.cassandra.io.util.FileHandle;
import org.apache.cassandra.utils.Pair;
import org.apache.cassandra.utils.bytecomparable.ByteComparable;
@ -72,18 +70,17 @@ public class TermsReaderTest extends SAIRandomizedTester
{
final int terms = 70, postings = 2;
final IndexDescriptor indexDescriptor = newIndexDescriptor();
final String index = newIndex();
final IndexContext indexContext = SAITester.createIndexContext(index, UTF8Type.instance);
final IndexIdentifier indexIdentifier = createIndexIdentifier("test", "test", newIndex());
final List<Pair<ByteComparable, LongArrayList>> termsEnum = buildTermsEnum(terms, postings);
SegmentMetadata.ComponentMetadataMap indexMetas;
try (LiteralIndexWriter writer = new LiteralIndexWriter(indexDescriptor, indexContext))
try (LiteralIndexWriter writer = new LiteralIndexWriter(indexDescriptor, indexIdentifier))
{
indexMetas = writer.writeCompleteSegment(new MemtableTermsIterator(null, null, termsEnum.iterator()));
}
FileHandle termsData = indexDescriptor.createPerIndexFileHandle(IndexComponent.TERMS_DATA, indexContext, null);
FileHandle postingLists = indexDescriptor.createPerIndexFileHandle(IndexComponent.POSTING_LISTS, indexContext, null);
FileHandle termsData = indexDescriptor.createPerIndexFileHandle(IndexComponent.TERMS_DATA, indexIdentifier, null);
FileHandle postingLists = indexDescriptor.createPerIndexFileHandle(IndexComponent.POSTING_LISTS, indexIdentifier, null);
try (TermsIterator iterator = new TermsScanner(termsData, postingLists, indexMetas.get(IndexComponent.TERMS_DATA).root))
{
@ -99,22 +96,21 @@ public class TermsReaderTest extends SAIRandomizedTester
private void testTermQueries(int numTerms, int numPostings) throws IOException
{
final IndexDescriptor indexDescriptor = newIndexDescriptor();
final String index = newIndex();
final IndexContext indexContext = SAITester.createIndexContext(index, UTF8Type.instance);
final IndexIdentifier indexIdentifier = createIndexIdentifier("test", "test", newIndex());
final List<Pair<ByteComparable, LongArrayList>> termsEnum = buildTermsEnum(numTerms, numPostings);
SegmentMetadata.ComponentMetadataMap indexMetas;
try (LiteralIndexWriter writer = new LiteralIndexWriter(indexDescriptor, indexContext))
try (LiteralIndexWriter writer = new LiteralIndexWriter(indexDescriptor, indexIdentifier))
{
indexMetas = writer.writeCompleteSegment(new MemtableTermsIterator(null, null, termsEnum.iterator()));
}
FileHandle termsData = indexDescriptor.createPerIndexFileHandle(IndexComponent.TERMS_DATA, indexContext, null);
FileHandle postingLists = indexDescriptor.createPerIndexFileHandle(IndexComponent.POSTING_LISTS, indexContext, null);
FileHandle termsData = indexDescriptor.createPerIndexFileHandle(IndexComponent.TERMS_DATA, indexIdentifier, null);
FileHandle postingLists = indexDescriptor.createPerIndexFileHandle(IndexComponent.POSTING_LISTS, indexIdentifier, null);
long termsFooterPointer = Long.parseLong(indexMetas.get(IndexComponent.TERMS_DATA).attributes.get(SAICodecUtils.FOOTER_POINTER));
try (LiteralIndexSegmentTermsReader reader = new LiteralIndexSegmentTermsReader(indexContext,
try (LiteralIndexSegmentTermsReader reader = new LiteralIndexSegmentTermsReader(indexIdentifier,
termsData,
postingLists,
indexMetas.get(IndexComponent.TERMS_DATA).root,

48
test/unit/org/apache/cassandra/index/sai/disk/v1/bbtree/BlockBalancedTreeIndexBuilder.java
View File

@ -40,18 +40,18 @@ import org.apache.cassandra.db.marshal.ShortType;
import org.apache.cassandra.db.marshal.UTF8Type;
import org.apache.cassandra.dht.Murmur3Partitioner;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.index.sai.IndexContext;
import org.apache.cassandra.index.sai.SAITester;
import org.apache.cassandra.index.sai.StorageAttachedIndex;
import org.apache.cassandra.index.sai.disk.PrimaryKeyMap;
import org.apache.cassandra.index.sai.disk.format.IndexDescriptor;
import org.apache.cassandra.index.sai.disk.v1.segment.NumericIndexSegmentSearcher;
import org.apache.cassandra.index.sai.disk.v1.PerColumnIndexFiles;
import org.apache.cassandra.index.sai.disk.v1.segment.IndexSegmentSearcher;
import org.apache.cassandra.index.sai.disk.v1.segment.NumericIndexSegmentSearcher;
import org.apache.cassandra.index.sai.disk.v1.segment.SegmentMetadata;
import org.apache.cassandra.index.sai.memory.MemtableTermsIterator;
import org.apache.cassandra.index.sai.utils.IndexTermType;
import org.apache.cassandra.index.sai.utils.PrimaryKey;
import org.apache.cassandra.index.sai.utils.TermsIterator;
import org.apache.cassandra.index.sai.utils.TypeUtil;
import org.apache.cassandra.utils.AbstractGuavaIterator;
import org.apache.cassandra.utils.Pair;
import org.apache.cassandra.utils.bytecomparable.ByteComparable;
@ -98,38 +98,36 @@ public class BlockBalancedTreeIndexBuilder
private static final BigDecimal ONE_TENTH = BigDecimal.valueOf(1, 1);
private final IndexDescriptor indexDescriptor;
private final AbstractType<?> type;
private final AbstractGuavaIterator<Pair<ByteComparable, LongArrayList>> terms;
private final int size;
private final int minSegmentRowId;
private final int maxSegmentRowId;
public BlockBalancedTreeIndexBuilder(IndexDescriptor indexDescriptor,
AbstractType<?> type,
AbstractGuavaIterator<Pair<ByteComparable, LongArrayList>> terms,
int size,
int minSegmentRowId,
int maxSegmentRowId)
{
this.indexDescriptor = indexDescriptor;
this.type = type;
this.terms = terms;
this.size = size;
this.minSegmentRowId = minSegmentRowId;
this.maxSegmentRowId = maxSegmentRowId;
}
NumericIndexSegmentSearcher flushAndOpen() throws IOException
NumericIndexSegmentSearcher flushAndOpen(AbstractType<?> type) throws IOException
{
final TermsIterator termEnum = new MemtableTermsIterator(null, null, terms);
final SegmentMetadata metadata;
IndexContext columnContext = SAITester.createIndexContext("test", Int32Type.instance);
StorageAttachedIndex index = SAITester.createMockIndex("test", type);
NumericIndexWriter writer = new NumericIndexWriter(indexDescriptor,
columnContext,
TypeUtil.fixedSizeOf(type),
index.identifier(),
index.termType().fixedSizeOf(),
maxSegmentRowId);
final SegmentMetadata.ComponentMetadataMap indexMetas = writer.writeCompleteSegment(BlockBalancedTreeIterator.fromTermsIterator(termEnum, type));
final SegmentMetadata.ComponentMetadataMap indexMetas = writer.writeCompleteSegment(BlockBalancedTreeIterator.fromTermsIterator(termEnum, index.termType()));
metadata = new SegmentMetadata(0,
size,
minSegmentRowId,
@ -141,9 +139,9 @@ public class BlockBalancedTreeIndexBuilder
UTF8Type.instance.fromString("d"),
indexMetas);
try (PerColumnIndexFiles indexFiles = new PerColumnIndexFiles(indexDescriptor, SAITester.createIndexContext("test", Int32Type.instance)))
try (PerColumnIndexFiles indexFiles = new PerColumnIndexFiles(indexDescriptor, index.termType(), index.identifier()))
{
IndexSegmentSearcher searcher = IndexSegmentSearcher.open(TEST_PRIMARY_KEY_MAP_FACTORY, indexFiles, metadata, columnContext);
IndexSegmentSearcher searcher = IndexSegmentSearcher.open(TEST_PRIMARY_KEY_MAP_FACTORY, indexFiles, metadata, index);
assertThat(searcher, is(instanceOf(NumericIndexSegmentSearcher.class)));
return (NumericIndexSegmentSearcher) searcher;
}
@ -161,12 +159,11 @@ public class BlockBalancedTreeIndexBuilder
final int size = endTermExclusive - startTermInclusive;
Assert.assertTrue(size > 0);
BlockBalancedTreeIndexBuilder indexBuilder = new BlockBalancedTreeIndexBuilder(indexDescriptor,
Int32Type.instance,
singleOrd(int32Range(startTermInclusive, endTermExclusive), Int32Type.instance, startTermInclusive, size),
size,
startTermInclusive,
endTermExclusive);
return indexBuilder.flushAndOpen();
return indexBuilder.flushAndOpen(Int32Type.instance);
}
public static IndexSegmentSearcher buildDecimalSearcher(IndexDescriptor indexDescriptor, BigDecimal startTermInclusive, BigDecimal endTermExclusive)
@ -176,12 +173,11 @@ public class BlockBalancedTreeIndexBuilder
int size = bigDifference.intValueExact() * 10;
Assert.assertTrue(size > 0);
BlockBalancedTreeIndexBuilder indexBuilder = new BlockBalancedTreeIndexBuilder(indexDescriptor,
DecimalType.instance,
singleOrd(decimalRange(startTermInclusive, endTermExclusive), DecimalType.instance, startTermInclusive.intValueExact() * 10, size),
size,
startTermInclusive.intValueExact() * 10,
endTermExclusive.intValueExact() * 10);
return indexBuilder.flushAndOpen();
return indexBuilder.flushAndOpen(DecimalType.instance);
}
public static IndexSegmentSearcher buildBigIntegerSearcher(IndexDescriptor indexDescriptor, BigInteger startTermInclusive, BigInteger endTermExclusive)
@ -191,12 +187,11 @@ public class BlockBalancedTreeIndexBuilder
int size = bigDifference.intValueExact();
Assert.assertTrue(size > 0);
BlockBalancedTreeIndexBuilder indexBuilder = new BlockBalancedTreeIndexBuilder(indexDescriptor,
IntegerType.instance,
singleOrd(bigIntegerRange(startTermInclusive, endTermExclusive), IntegerType.instance, startTermInclusive.intValueExact(), size),
size,
startTermInclusive.intValueExact(),
endTermExclusive.intValueExact());
return indexBuilder.flushAndOpen();
return indexBuilder.flushAndOpen(IntegerType.instance);
}
/**
@ -211,12 +206,11 @@ public class BlockBalancedTreeIndexBuilder
final long size = endTermExclusive - startTermInclusive;
Assert.assertTrue(size > 0);
BlockBalancedTreeIndexBuilder indexBuilder = new BlockBalancedTreeIndexBuilder(indexDescriptor,
LongType.instance,
singleOrd(longRange(startTermInclusive, endTermExclusive), LongType.instance, Math.toIntExact(startTermInclusive), Math.toIntExact(size)),
Math.toIntExact(size),
Math.toIntExact(startTermInclusive),
Math.toIntExact(endTermExclusive));
return indexBuilder.flushAndOpen();
return indexBuilder.flushAndOpen(LongType.instance);
}
/**
@ -231,12 +225,11 @@ public class BlockBalancedTreeIndexBuilder
final int size = endTermExclusive - startTermInclusive;
Assert.assertTrue(size > 0);
BlockBalancedTreeIndexBuilder indexBuilder = new BlockBalancedTreeIndexBuilder(indexDescriptor,
ShortType.instance,
singleOrd(shortRange(startTermInclusive, endTermExclusive), ShortType.instance, startTermInclusive, size),
size,
startTermInclusive,
endTermExclusive);
return indexBuilder.flushAndOpen();
return indexBuilder.flushAndOpen(ShortType.instance);
}
/**
@ -245,6 +238,7 @@ public class BlockBalancedTreeIndexBuilder
*/
public static AbstractGuavaIterator<Pair<ByteComparable, LongArrayList>> singleOrd(Iterator<ByteBuffer> terms, AbstractType<?> type, int segmentRowIdOffset, int size)
{
IndexTermType indexTermType = SAITester.createIndexTermType(type);
return new AbstractGuavaIterator<Pair<ByteComparable, LongArrayList>>()
{
private long currentTerm = 0;
@ -262,7 +256,7 @@ public class BlockBalancedTreeIndexBuilder
postings.add(currentSegmentRowId++);
assertTrue(terms.hasNext());
final ByteSource encoded = TypeUtil.asComparableBytes(terms.next(), type, ByteComparable.Version.OSS50);
final ByteSource encoded = indexTermType.asComparableBytes(terms.next(), ByteComparable.Version.OSS50);
return Pair.create(v -> encoded, postings);
}
};
@ -305,9 +299,10 @@ public class BlockBalancedTreeIndexBuilder
return result;
}
};
IndexTermType indexTermType = SAITester.createIndexTermType(DecimalType.instance);
return Stream.generate(generator)
.limit(n)
.map(bd -> TypeUtil.asIndexBytes(DecimalType.instance.decompose(bd), DecimalType.instance))
.map(bd -> indexTermType.asIndexBytes(DecimalType.instance.decompose(bd)))
.collect(Collectors.toList())
.iterator();
}
@ -325,9 +320,10 @@ public class BlockBalancedTreeIndexBuilder
return result;
}
};
IndexTermType indexTermType = SAITester.createIndexTermType(IntegerType.instance);
return Stream.generate(generator)
.limit(n)
.map(bd -> TypeUtil.asIndexBytes(IntegerType.instance.decompose(bd), IntegerType.instance))
.map(bd -> indexTermType.asIndexBytes(IntegerType.instance.decompose(bd)))
.collect(Collectors.toList())
.iterator();
}

27
test/unit/org/apache/cassandra/index/sai/disk/v1/bbtree/BlockBalancedTreePostingsWriterTest.java
View File

@ -26,12 +26,11 @@ import org.junit.Before;
import org.junit.Test;
import org.agrona.collections.IntArrayList;
import org.apache.cassandra.db.marshal.Int32Type;
import org.apache.cassandra.index.sai.IndexContext;
import org.apache.cassandra.index.sai.SAITester;
import org.apache.cassandra.index.sai.disk.ArrayPostingList;
import org.apache.cassandra.index.sai.disk.format.IndexComponent;
import org.apache.cassandra.index.sai.disk.format.IndexDescriptor;
import org.apache.cassandra.index.sai.utils.IndexIdentifier;
import org.apache.cassandra.index.sai.disk.io.IndexOutputWriter;
import org.apache.cassandra.index.sai.disk.v1.postings.PostingsReader;
import org.apache.cassandra.index.sai.metrics.QueryEventListener;
@ -48,13 +47,13 @@ import static org.mockito.Mockito.mock;
public class BlockBalancedTreePostingsWriterTest extends SAIRandomizedTester
{
private IndexDescriptor indexDescriptor;
private IndexContext indexContext;
private IndexIdentifier indexIdentifier;
@Before
public void setup() throws Throwable
{
indexDescriptor = newIndexDescriptor();
indexContext = SAITester.createIndexContext(newIndex(), Int32Type.instance);
indexIdentifier = SAITester.createIndexIdentifier("test", "test", newIndex());
}
@Test
@ -73,12 +72,12 @@ public class BlockBalancedTreePostingsWriterTest extends SAIRandomizedTester
writer.onLeaf(112, 4, pathToRoot(1, 3, 7, 14, 28));
long fp;
try (IndexOutputWriter output = indexDescriptor.openPerIndexOutput(IndexComponent.POSTING_LISTS, indexContext))
try (IndexOutputWriter output = indexDescriptor.openPerIndexOutput(IndexComponent.POSTING_LISTS, indexIdentifier))
{
fp = writer.finish(output, leaves, indexContext);
fp = writer.finish(output, leaves, indexIdentifier);
}
BlockBalancedTreePostingsIndex postingsIndex = new BlockBalancedTreePostingsIndex(indexDescriptor.createPerIndexFileHandle(IndexComponent.POSTING_LISTS, indexContext, null), fp);
BlockBalancedTreePostingsIndex postingsIndex = new BlockBalancedTreePostingsIndex(indexDescriptor.createPerIndexFileHandle(IndexComponent.POSTING_LISTS, indexIdentifier, null), fp);
assertEquals(10, postingsIndex.size());
// Internal postings...
@ -120,13 +119,13 @@ public class BlockBalancedTreePostingsWriterTest extends SAIRandomizedTester
writer.onLeaf(16, 1, pathToRoot(1, 2, 4, 8));
long fp;
try (IndexOutputWriter output = indexDescriptor.openPerIndexOutput(IndexComponent.POSTING_LISTS, indexContext))
try (IndexOutputWriter output = indexDescriptor.openPerIndexOutput(IndexComponent.POSTING_LISTS, indexIdentifier))
{
fp = writer.finish(output, leaves, indexContext);
fp = writer.finish(output, leaves, indexIdentifier);
}
// There is only a single posting list...the leaf posting list.
BlockBalancedTreePostingsIndex postingsIndex = new BlockBalancedTreePostingsIndex(indexDescriptor.createPerIndexFileHandle(IndexComponent.POSTING_LISTS, indexContext, null), fp);
BlockBalancedTreePostingsIndex postingsIndex = new BlockBalancedTreePostingsIndex(indexDescriptor.createPerIndexFileHandle(IndexComponent.POSTING_LISTS, indexIdentifier, null), fp);
assertEquals(1, postingsIndex.size());
}
@ -142,19 +141,19 @@ public class BlockBalancedTreePostingsWriterTest extends SAIRandomizedTester
writer.onLeaf(16, 1, pathToRoot(1, 2, 4, 8));
long fp;
try (IndexOutputWriter output = indexDescriptor.openPerIndexOutput(IndexComponent.POSTING_LISTS, indexContext))
try (IndexOutputWriter output = indexDescriptor.openPerIndexOutput(IndexComponent.POSTING_LISTS, indexIdentifier))
{
fp = writer.finish(output, leaves, indexContext);
fp = writer.finish(output, leaves, indexIdentifier);
}
// There is only a single posting list...the leaf posting list.
BlockBalancedTreePostingsIndex postingsIndex = new BlockBalancedTreePostingsIndex(indexDescriptor.createPerIndexFileHandle(IndexComponent.POSTING_LISTS, indexContext, null), fp);
BlockBalancedTreePostingsIndex postingsIndex = new BlockBalancedTreePostingsIndex(indexDescriptor.createPerIndexFileHandle(IndexComponent.POSTING_LISTS, indexIdentifier, null), fp);
assertEquals(1, postingsIndex.size());
}
private void assertPostingReaderEquals(BlockBalancedTreePostingsIndex postingsIndex, int nodeID, long... postings) throws IOException
{
assertPostingReaderEquals(indexDescriptor.openPerIndexInput(IndexComponent.POSTING_LISTS, indexContext),
assertPostingReaderEquals(indexDescriptor.openPerIndexInput(IndexComponent.POSTING_LISTS, indexIdentifier),
postingsIndex.getPostingsFilePointer(nodeID),
new ArrayPostingList(postings));
}

5
test/unit/org/apache/cassandra/index/sai/disk/v1/bbtree/BlockBalancedTreeQueriesTest.java
View File

@ -21,7 +21,6 @@ import org.junit.Test;
import org.apache.cassandra.cql3.Operator;
import org.apache.cassandra.db.marshal.Int32Type;
import org.apache.cassandra.index.sai.SAITester;
import org.apache.cassandra.index.sai.plan.Expression;
import org.apache.cassandra.index.sai.utils.SAIRandomizedTester;
import org.apache.cassandra.utils.bytecomparable.ByteComparable;
@ -40,7 +39,7 @@ public class BlockBalancedTreeQueriesTest extends SAIRandomizedTester
@Test
public void testMatchesAll()
{
Expression expression = new Expression(SAITester.createIndexContext("meh", Int32Type.instance));
Expression expression = Expression.create(createMockIndex("meh", Int32Type.instance));
BlockBalancedTreeReader.IntersectVisitor query = BlockBalancedTreeQueries.balancedTreeQueryFrom(expression, 4);
for (int visit = 0; visit < between(100, 1000); visit++)
@ -213,7 +212,7 @@ public class BlockBalancedTreeQueriesTest extends SAIRandomizedTester
private Expression buildExpression(Operator op, int value)
{
Expression expression = new Expression(SAITester.createIndexContext("meh", Int32Type.instance));
Expression expression = Expression.create(createMockIndex("meh", Int32Type.instance));
expression.add(op, Int32Type.instance.decompose(value));
return expression;
}

6
test/unit/org/apache/cassandra/index/sai/disk/v1/bbtree/BlockBalancedTreeRamBufferTest.java
View File

@ -27,8 +27,8 @@ import org.junit.Test;
import org.apache.cassandra.db.marshal.Int32Type;
import org.apache.cassandra.index.sai.postings.PostingList;
import org.apache.cassandra.index.sai.utils.IndexTermType;
import org.apache.cassandra.index.sai.utils.SAIRandomizedTester;
import org.apache.cassandra.index.sai.utils.TypeUtil;
import org.apache.cassandra.utils.Pair;
import org.apache.cassandra.utils.bytecomparable.ByteComparable;
import org.apache.cassandra.utils.bytecomparable.ByteSource;
@ -51,9 +51,11 @@ public class BlockBalancedTreeRamBufferTest extends SAIRandomizedTester
BlockBalancedTreeRamBuffer buffer = new BlockBalancedTreeRamBuffer(Integer.BYTES);
IndexTermType indexTermType = createIndexTermType(Int32Type.instance);
byte[] scratch = new byte[Integer.BYTES];
values.forEach(v -> {
TypeUtil.toComparableBytes(Int32Type.instance.decompose(v), Int32Type.instance, scratch);
indexTermType.toComparableBytes(Int32Type.instance.decompose(v), scratch);
buffer.add(0, scratch);
});

16
test/unit/org/apache/cassandra/index/sai/disk/v1/bbtree/BlockBalancedTreeReaderTest.java
View File

@ -29,9 +29,9 @@ import org.junit.Test;
import org.apache.cassandra.cql3.Operator;
import org.apache.cassandra.db.marshal.Int32Type;
import org.apache.cassandra.index.sai.IndexContext;
import org.apache.cassandra.index.sai.QueryContext;
import org.apache.cassandra.index.sai.SAITester;
import org.apache.cassandra.index.sai.StorageAttachedIndex;
import org.apache.cassandra.index.sai.disk.format.IndexComponent;
import org.apache.cassandra.index.sai.disk.format.IndexDescriptor;
import org.apache.cassandra.index.sai.disk.v1.segment.SegmentMetadata;
@ -90,13 +90,13 @@ public class BlockBalancedTreeReaderTest extends SAIRandomizedTester
};
private IndexDescriptor indexDescriptor;
private IndexContext indexContext;
private StorageAttachedIndex index;
@Before
public void setup() throws Throwable
{
indexDescriptor = newIndexDescriptor();
indexContext = SAITester.createIndexContext(newIndex(), Int32Type.instance);
index = SAITester.createMockIndex(newIndex(), Int32Type.instance);
}
@Test
@ -260,7 +260,7 @@ public class BlockBalancedTreeReaderTest extends SAIRandomizedTester
@SuppressWarnings("SameParameterValue")
private void assertRange(BlockBalancedTreeReader reader, long lowerBound, long upperBound)
{
Expression expression = new Expression(indexContext);
Expression expression = Expression.create(index);
expression.add(Operator.GT, Int32Type.instance.decompose(444));
expression.add(Operator.LT, Int32Type.instance.decompose(555));
@ -323,7 +323,7 @@ public class BlockBalancedTreeReaderTest extends SAIRandomizedTester
{
setBDKPostingsWriterSizing(8, 2);
final NumericIndexWriter writer = new NumericIndexWriter(indexDescriptor,
indexContext,
index.identifier(),
maxPointsPerLeaf,
Integer.BYTES,
Math.toIntExact(buffer.numRows()));
@ -334,9 +334,9 @@ public class BlockBalancedTreeReaderTest extends SAIRandomizedTester
final long postingsPosition = metadata.get(IndexComponent.POSTING_LISTS).root;
assertThat(postingsPosition, is(greaterThan(0L)));
FileHandle treeHandle = indexDescriptor.createPerIndexFileHandle(IndexComponent.BALANCED_TREE, indexContext, null);
FileHandle treePostingsHandle = indexDescriptor.createPerIndexFileHandle(IndexComponent.POSTING_LISTS, indexContext, null);
return new BlockBalancedTreeReader(indexContext,
FileHandle treeHandle = indexDescriptor.createPerIndexFileHandle(IndexComponent.BALANCED_TREE, index.identifier());
FileHandle treePostingsHandle = indexDescriptor.createPerIndexFileHandle(IndexComponent.POSTING_LISTS, index.identifier());
return new BlockBalancedTreeReader(index.identifier(),
treeHandle,
treePosition,
treePostingsHandle,

30
test/unit/org/apache/cassandra/index/sai/disk/v1/bbtree/NumericIndexWriterTest.java
View File

@ -24,18 +24,18 @@ import org.junit.Test;
import com.carrotsearch.hppc.LongArrayList;
import org.apache.cassandra.db.marshal.Int32Type;
import org.apache.cassandra.index.sai.IndexContext;
import org.apache.cassandra.index.sai.QueryContext;
import org.apache.cassandra.index.sai.SAITester;
import org.apache.cassandra.index.sai.disk.format.IndexComponent;
import org.apache.cassandra.index.sai.disk.format.IndexDescriptor;
import org.apache.cassandra.index.sai.utils.IndexIdentifier;
import org.apache.cassandra.index.sai.disk.v1.segment.SegmentMetadata;
import org.apache.cassandra.index.sai.memory.MemtableTermsIterator;
import org.apache.cassandra.index.sai.metrics.QueryEventListener;
import org.apache.cassandra.index.sai.postings.PostingList;
import org.apache.cassandra.index.sai.utils.IndexTermType;
import org.apache.cassandra.index.sai.utils.SAIRandomizedTester;
import org.apache.cassandra.index.sai.utils.TermsIterator;
import org.apache.cassandra.index.sai.utils.TypeUtil;
import org.apache.cassandra.io.util.FileHandle;
import org.apache.cassandra.utils.AbstractGuavaIterator;
import org.apache.cassandra.utils.Pair;
@ -52,13 +52,15 @@ import static org.mockito.Mockito.when;
public class NumericIndexWriterTest extends SAIRandomizedTester
{
private IndexDescriptor indexDescriptor;
private IndexContext indexContext;
private IndexTermType indexTermType;
private IndexIdentifier indexIdentifier;
@Before
public void setup() throws Throwable
{
indexDescriptor = newIndexDescriptor();
indexContext = SAITester.createIndexContext(newIndex(), Int32Type.instance);
indexTermType = SAITester.createIndexTermType(Int32Type.instance);
indexIdentifier = SAITester.createIndexIdentifier("test", "test", newIndex());
}
@Test
@ -79,15 +81,15 @@ public class NumericIndexWriterTest extends SAIRandomizedTester
SegmentMetadata.ComponentMetadataMap indexMetas;
NumericIndexWriter writer = new NumericIndexWriter(indexDescriptor,
indexContext,
indexIdentifier,
Integer.BYTES,
rowCount);
indexMetas = writer.writeCompleteSegment(ramBuffer.iterator());
final FileHandle treeHandle = indexDescriptor.createPerIndexFileHandle(IndexComponent.BALANCED_TREE, indexContext, null);
final FileHandle treePostingsHandle = indexDescriptor.createPerIndexFileHandle(IndexComponent.POSTING_LISTS, indexContext, null);
final FileHandle treeHandle = indexDescriptor.createPerIndexFileHandle(IndexComponent.BALANCED_TREE, indexIdentifier, null);
final FileHandle treePostingsHandle = indexDescriptor.createPerIndexFileHandle(IndexComponent.POSTING_LISTS, indexIdentifier, null);
try (BlockBalancedTreeReader reader = new BlockBalancedTreeReader(indexContext,
try (BlockBalancedTreeReader reader = new BlockBalancedTreeReader(indexIdentifier,
treeHandle,
indexMetas.get(IndexComponent.BALANCED_TREE).root,
treePostingsHandle,
@ -125,15 +127,15 @@ public class NumericIndexWriterTest extends SAIRandomizedTester
SegmentMetadata.ComponentMetadataMap indexMetas;
NumericIndexWriter writer = new NumericIndexWriter(indexDescriptor,
indexContext,
TypeUtil.fixedSizeOf(Int32Type.instance),
indexIdentifier,
indexTermType.fixedSizeOf(),
maxSegmentRowId);
indexMetas = writer.writeCompleteSegment(BlockBalancedTreeIterator.fromTermsIterator(termEnum, Int32Type.instance));
indexMetas = writer.writeCompleteSegment(BlockBalancedTreeIterator.fromTermsIterator(termEnum, indexTermType));
final FileHandle treeHandle = indexDescriptor.createPerIndexFileHandle(IndexComponent.BALANCED_TREE, indexContext, null);
final FileHandle treePostingsHandle = indexDescriptor.createPerIndexFileHandle(IndexComponent.POSTING_LISTS, indexContext, null);
final FileHandle treeHandle = indexDescriptor.createPerIndexFileHandle(IndexComponent.BALANCED_TREE, indexIdentifier, null);
final FileHandle treePostingsHandle = indexDescriptor.createPerIndexFileHandle(IndexComponent.POSTING_LISTS, indexIdentifier, null);
try (BlockBalancedTreeReader reader = new BlockBalancedTreeReader(indexContext,
try (BlockBalancedTreeReader reader = new BlockBalancedTreeReader(indexIdentifier,
treeHandle,
indexMetas.get(IndexComponent.BALANCED_TREE).root,
treePostingsHandle,

45
test/unit/org/apache/cassandra/index/sai/disk/v1/postings/PostingsTest.java
View File

@ -26,9 +26,8 @@ import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
import org.apache.cassandra.db.marshal.UTF8Type;
import org.apache.cassandra.index.sai.IndexContext;
import org.apache.cassandra.index.sai.SAITester;
import org.apache.cassandra.index.sai.utils.IndexIdentifier;
import org.apache.cassandra.index.sai.postings.PostingList;
import org.apache.cassandra.index.sai.disk.format.IndexComponent;
import org.apache.cassandra.index.sai.disk.format.IndexDescriptor;
@ -47,14 +46,16 @@ public class PostingsTest extends SAIRandomizedTester
public final ExpectedException expectedException = ExpectedException.none();
private IndexDescriptor indexDescriptor;
private IndexContext indexContext;
private IndexIdentifier indexIdentifier;
@Before
public void setup() throws Throwable
{
indexDescriptor = newIndexDescriptor();
String index = newIndex();
indexContext = SAITester.createIndexContext(index, UTF8Type.instance);
indexIdentifier = SAITester.createIndexIdentifier(indexDescriptor.sstableDescriptor.ksname,
indexDescriptor.sstableDescriptor.cfname,
index);
}
@ -65,13 +66,13 @@ public class PostingsTest extends SAIRandomizedTester
final ArrayPostingList expectedPostingList = new ArrayPostingList(10, 20, 30, 40, 50, 60);
long postingPointer;
try (PostingsWriter writer = new PostingsWriter(indexDescriptor, indexContext, blockSize))
try (PostingsWriter writer = new PostingsWriter(indexDescriptor, indexIdentifier, blockSize))
{
postingPointer = writer.write(expectedPostingList);
writer.complete();
}
IndexInput input = indexDescriptor.openPerIndexInput(IndexComponent.POSTING_LISTS, indexContext);
IndexInput input = indexDescriptor.openPerIndexInput(IndexComponent.POSTING_LISTS, indexIdentifier);
SAICodecUtils.validate(input);
input.seek(postingPointer);
@ -96,7 +97,7 @@ public class PostingsTest extends SAIRandomizedTester
reader.close();
assertEquals(reader.size(), listener.decodes);
input = indexDescriptor.openPerIndexInput(IndexComponent.POSTING_LISTS, indexContext);
input = indexDescriptor.openPerIndexInput(IndexComponent.POSTING_LISTS, indexIdentifier);
listener = new CountingPostingListEventListener();
reader = new PostingsReader(input, postingPointer, listener);
@ -118,7 +119,7 @@ public class PostingsTest extends SAIRandomizedTester
final ArrayPostingList[] expected = new ArrayPostingList[numPostingLists];
final long[] postingPointers = new long[numPostingLists];
try (PostingsWriter writer = new PostingsWriter(indexDescriptor, indexContext, blockSize))
try (PostingsWriter writer = new PostingsWriter(indexDescriptor, indexIdentifier, blockSize))
{
for (int i = 0; i < numPostingLists; ++i)
{
@ -130,14 +131,14 @@ public class PostingsTest extends SAIRandomizedTester
writer.complete();
}
try (IndexInput input = indexDescriptor.openPerIndexInput(IndexComponent.POSTING_LISTS, indexContext))
try (IndexInput input = indexDescriptor.openPerIndexInput(IndexComponent.POSTING_LISTS, indexIdentifier))
{
SAICodecUtils.validate(input);
}
for (int i = 0; i < numPostingLists; ++i)
{
IndexInput input = indexDescriptor.openPerIndexInput(IndexComponent.POSTING_LISTS, indexContext);
IndexInput input = indexDescriptor.openPerIndexInput(IndexComponent.POSTING_LISTS, indexIdentifier);
input.seek(postingPointers[i]);
ArrayPostingList expectedPostingList = expected[i];
PostingsReader.BlocksSummary summary = assertBlockSummary(blockSize, expectedPostingList, input);
@ -156,7 +157,7 @@ public class PostingsTest extends SAIRandomizedTester
// test random advances through the posting list
listener = new CountingPostingListEventListener();
input = indexDescriptor.openPerIndexInput(IndexComponent.POSTING_LISTS, indexContext);
input = indexDescriptor.openPerIndexInput(IndexComponent.POSTING_LISTS, indexIdentifier);
try (PostingsReader reader = new PostingsReader(input, postingPointers[i], listener))
{
expectedPostingList.reset();
@ -171,7 +172,7 @@ public class PostingsTest extends SAIRandomizedTester
// test skipping to the last block
listener = new CountingPostingListEventListener();
input = indexDescriptor.openPerIndexInput(IndexComponent.POSTING_LISTS, indexContext);
input = indexDescriptor.openPerIndexInput(IndexComponent.POSTING_LISTS, indexIdentifier);
try (PostingsReader reader = new PostingsReader(input, postingPointers[i], listener))
{
long tokenToAdvance = -1;
@ -200,13 +201,13 @@ public class PostingsTest extends SAIRandomizedTester
final ArrayPostingList expectedPostingList = new ArrayPostingList(0, 1, 1, 3, 3, 5, 5, 7, 7, 7, 7, 7, 7, 9, 9, 10, 11, 12);
long postingPointer;
try (PostingsWriter writer = new PostingsWriter(indexDescriptor, indexContext, blockSize))
try (PostingsWriter writer = new PostingsWriter(indexDescriptor, indexIdentifier, blockSize))
{
postingPointer = writer.write(expectedPostingList);
writer.complete();
}
IndexInput input = indexDescriptor.openPerIndexInput(IndexComponent.POSTING_LISTS, indexContext);
IndexInput input = indexDescriptor.openPerIndexInput(IndexComponent.POSTING_LISTS, indexIdentifier);
CountingPostingListEventListener listener = new CountingPostingListEventListener();
try (PostingsReader reader = new PostingsReader(input, postingPointer, listener))
{
@ -223,13 +224,13 @@ public class PostingsTest extends SAIRandomizedTester
final ArrayPostingList expected = new ArrayPostingList(postings);
long fp;
try (PostingsWriter writer = new PostingsWriter(indexDescriptor, indexContext, blockSize))
try (PostingsWriter writer = new PostingsWriter(indexDescriptor, indexIdentifier, blockSize))
{
fp = writer.write(expected);
writer.complete();
}
try (IndexInput input = indexDescriptor.openPerIndexInput(IndexComponent.POSTING_LISTS, indexContext))
try (IndexInput input = indexDescriptor.openPerIndexInput(IndexComponent.POSTING_LISTS, indexIdentifier))
{
SAICodecUtils.validate(input);
input.seek(fp);
@ -264,13 +265,13 @@ public class PostingsTest extends SAIRandomizedTester
final ArrayPostingList expected = new ArrayPostingList(postings);
long fp;
try (PostingsWriter writer = new PostingsWriter(indexDescriptor, indexContext, blockSize))
try (PostingsWriter writer = new PostingsWriter(indexDescriptor, indexIdentifier, blockSize))
{
fp = writer.write(expected);
writer.complete();
}
try (IndexInput input = indexDescriptor.openPerIndexInput(IndexComponent.POSTING_LISTS, indexContext))
try (IndexInput input = indexDescriptor.openPerIndexInput(IndexComponent.POSTING_LISTS, indexIdentifier))
{
SAICodecUtils.validate(input);
input.seek(fp);
@ -291,7 +292,7 @@ public class PostingsTest extends SAIRandomizedTester
@SuppressWarnings("all")
public void testNullPostingList() throws IOException
{
try (PostingsWriter writer = new PostingsWriter(indexDescriptor, indexContext))
try (PostingsWriter writer = new PostingsWriter(indexDescriptor, indexIdentifier))
{
expectedException.expect(IllegalArgumentException.class);
writer.write(null);
@ -302,7 +303,7 @@ public class PostingsTest extends SAIRandomizedTester
@Test
public void testEmptyPostingList() throws IOException
{
try (PostingsWriter writer = new PostingsWriter(indexDescriptor, indexContext))
try (PostingsWriter writer = new PostingsWriter(indexDescriptor, indexIdentifier))
{
expectedException.expect(IllegalArgumentException.class);
writer.write(new ArrayPostingList());
@ -312,7 +313,7 @@ public class PostingsTest extends SAIRandomizedTester
@Test
public void testNonAscendingPostingList() throws IOException
{
try (PostingsWriter writer = new PostingsWriter(indexDescriptor, indexContext))
try (PostingsWriter writer = new PostingsWriter(indexDescriptor, indexIdentifier))
{
expectedException.expect(IllegalArgumentException.class);
writer.write(new ArrayPostingList(1, 0));
@ -350,7 +351,7 @@ public class PostingsTest extends SAIRandomizedTester
private PostingsReader openReader(long fp, QueryEventListener.PostingListEventListener listener) throws IOException
{
IndexInput input = indexDescriptor.openPerIndexInput(IndexComponent.POSTING_LISTS, indexContext);
IndexInput input = indexDescriptor.openPerIndexInput(IndexComponent.POSTING_LISTS, indexIdentifier);
input.seek(fp);
return new PostingsReader(input, fp, listener);
}

21
test/unit/org/apache/cassandra/index/sai/disk/v1/trie/TrieTermsDictionaryTest.java
View File

@ -28,11 +28,9 @@ import org.apache.commons.lang3.mutable.MutableLong;
import org.junit.Before;
import org.junit.Test;
import org.apache.cassandra.db.marshal.UTF8Type;
import org.apache.cassandra.index.sai.IndexContext;
import org.apache.cassandra.index.sai.SAITester;
import org.apache.cassandra.index.sai.disk.format.IndexComponent;
import org.apache.cassandra.index.sai.disk.format.IndexDescriptor;
import org.apache.cassandra.index.sai.utils.IndexIdentifier;
import org.apache.cassandra.index.sai.utils.SAIRandomizedTester;
import org.apache.cassandra.io.util.FileHandle;
import org.apache.cassandra.utils.ByteBufferUtil;
@ -48,14 +46,13 @@ import static org.junit.Assert.assertTrue;
public class TrieTermsDictionaryTest extends SAIRandomizedTester
{
private IndexDescriptor indexDescriptor;
private IndexContext indexContext;
private IndexIdentifier indexIdentifier;
@Before
public void setup() throws Throwable
{
indexDescriptor = newIndexDescriptor();
String index = newIndex();
indexContext = SAITester.createIndexContext(index, UTF8Type.instance);
indexIdentifier = createIndexIdentifier("test", "test", newIndex());
}
@Test
@ -67,7 +64,7 @@ public class TrieTermsDictionaryTest extends SAIRandomizedTester
private void doTestExactMatch() throws Exception
{
long fp;
try (TrieTermsDictionaryWriter writer = new TrieTermsDictionaryWriter(indexDescriptor, indexContext))
try (TrieTermsDictionaryWriter writer = new TrieTermsDictionaryWriter(indexDescriptor, indexIdentifier))
{
writer.add(asByteComparable("ab"), 0);
writer.add(asByteComparable("abb"), 1);
@ -77,7 +74,7 @@ public class TrieTermsDictionaryTest extends SAIRandomizedTester
fp = writer.complete(new MutableLong());
}
try (FileHandle input = indexDescriptor.createPerIndexFileHandle(IndexComponent.TERMS_DATA, indexContext, null);
try (FileHandle input = indexDescriptor.createPerIndexFileHandle(IndexComponent.TERMS_DATA, indexIdentifier);
TrieTermsDictionaryReader reader = new TrieTermsDictionaryReader(input.instantiateRebufferer(null), fp))
{
assertEquals(TrieTermsDictionaryReader.NOT_FOUND, reader.exactMatch(asByteComparable("a")));
@ -95,7 +92,7 @@ public class TrieTermsDictionaryTest extends SAIRandomizedTester
final List<ByteComparable> byteComparables = generateSortedByteComparables();
long fp;
try (TrieTermsDictionaryWriter writer = new TrieTermsDictionaryWriter(indexDescriptor, indexContext))
try (TrieTermsDictionaryWriter writer = new TrieTermsDictionaryWriter(indexDescriptor, indexIdentifier))
{
for (int i = 0; i < byteComparables.size(); ++i)
{
@ -104,7 +101,7 @@ public class TrieTermsDictionaryTest extends SAIRandomizedTester
fp = writer.complete(new MutableLong());
}
try (FileHandle input = indexDescriptor.createPerIndexFileHandle(IndexComponent.TERMS_DATA, indexContext, null);
try (FileHandle input = indexDescriptor.createPerIndexFileHandle(IndexComponent.TERMS_DATA, indexIdentifier);
TrieTermsIterator iterator = new TrieTermsIterator(input.instantiateRebufferer(null), fp))
{
final Iterator<ByteComparable> expected = byteComparables.iterator();
@ -127,7 +124,7 @@ public class TrieTermsDictionaryTest extends SAIRandomizedTester
final List<ByteComparable> byteComparables = generateSortedByteComparables();
long fp;
try (TrieTermsDictionaryWriter writer = new TrieTermsDictionaryWriter(indexDescriptor, indexContext))
try (TrieTermsDictionaryWriter writer = new TrieTermsDictionaryWriter(indexDescriptor, indexIdentifier))
{
for (int i = 0; i < byteComparables.size(); ++i)
{
@ -136,7 +133,7 @@ public class TrieTermsDictionaryTest extends SAIRandomizedTester
fp = writer.complete(new MutableLong());
}
try (FileHandle input = indexDescriptor.createPerIndexFileHandle(IndexComponent.TERMS_DATA, indexContext, null);
try (FileHandle input = indexDescriptor.createPerIndexFileHandle(IndexComponent.TERMS_DATA, indexIdentifier);
TrieTermsDictionaryReader reader = new TrieTermsDictionaryReader(input.instantiateRebufferer(null), fp))
{
final ByteComparable expectedMaxTerm = byteComparables.get(byteComparables.size() - 1);

34
test/unit/org/apache/cassandra/index/sai/functional/CompactionTest.java
View File

@ -39,9 +39,10 @@ import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
import org.apache.cassandra.db.marshal.Int32Type;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.index.sai.IndexContext;
import org.apache.cassandra.index.sai.SAITester;
import org.apache.cassandra.index.sai.utils.IndexIdentifier;
import org.apache.cassandra.index.sai.disk.v1.SSTableIndexWriter;
import org.apache.cassandra.index.sai.utils.IndexTermType;
import org.apache.cassandra.inject.ActionBuilder;
import org.apache.cassandra.inject.Expression;
import org.apache.cassandra.inject.Injection;
@ -72,7 +73,8 @@ public class CompactionTest extends SAITester
public void testAntiCompaction() throws Throwable
{
createTable(CREATE_TABLE_TEMPLATE);
IndexContext numericIndexContext = createIndexContext(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1")), Int32Type.instance);
IndexIdentifier numericIndexIdentifier = createIndexIdentifier(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1")));
IndexTermType numericIndexTermType = createIndexTermType(Int32Type.instance);
verifyNoIndexFiles();
// create 100 rows in 1 sstable
@ -83,8 +85,8 @@ public class CompactionTest extends SAITester
// verify 1 sstable index
assertNumRows(num, "SELECT * FROM %%s WHERE v1 >= 0");
verifyIndexFiles(numericIndexContext, null, 1, 0);
verifySSTableIndexes(numericIndexContext.getIndexName(), 1);
verifyIndexFiles(numericIndexTermType, numericIndexIdentifier, 1);
verifySSTableIndexes(numericIndexIdentifier, 1);
// split sstable into repaired and unrepaired
ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(currentTable());
@ -110,8 +112,8 @@ public class CompactionTest extends SAITester
// verify 2 sstable indexes
assertNumRows(num, "SELECT * FROM %%s WHERE v1 >= 0");
waitForAssert(() -> verifyIndexFiles(numericIndexContext, null, 2, 0));
verifySSTableIndexes(numericIndexContext.getIndexName(), 2);
waitForAssert(() -> verifyIndexFiles(numericIndexTermType, numericIndexIdentifier, 2));
verifySSTableIndexes(numericIndexIdentifier, 2);
// index components are included after anti-compaction
verifyIndexComponentsIncludedInSSTable();
@ -121,8 +123,8 @@ public class CompactionTest extends SAITester
public void testConcurrentQueryWithCompaction()
{
createTable(CREATE_TABLE_TEMPLATE);
String v1IndexName = createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1"));
String v2IndexName = createIndex(String.format(CREATE_INDEX_TEMPLATE, "v2"));
IndexIdentifier numericIndexIdentifier = createIndexIdentifier(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1")));
IndexIdentifier literalIndexIdentifier = createIndexIdentifier(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v2")));
waitForTableIndexesQueryable();
int num = 10;
@ -149,16 +151,16 @@ public class CompactionTest extends SAITester
compactionTest.start();
verifySSTableIndexes(v1IndexName, num);
verifySSTableIndexes(v2IndexName, num);
verifySSTableIndexes(numericIndexIdentifier, num);
verifySSTableIndexes(literalIndexIdentifier, num);
}
@Test
public void testAbortCompactionWithEarlyOpenSSTables() throws Throwable
{
createTable(CREATE_TABLE_TEMPLATE);
String v1IndexName = createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1"));
String v2IndexName = createIndex(String.format(CREATE_INDEX_TEMPLATE, "v2"));
IndexIdentifier numericIndexIdentifier = createIndexIdentifier(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1")));
IndexIdentifier literalIndexIdentifier = createIndexIdentifier(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v2")));
int sstables = 2;
int num = 10;
@ -207,8 +209,8 @@ public class CompactionTest extends SAITester
// verify indexes are working
assertNumRows(num, "SELECT id1 FROM %%s WHERE v1=0");
assertNumRows(num, "SELECT id1 FROM %%s WHERE v2='0'");
verifySSTableIndexes(v1IndexName, sstables);
verifySSTableIndexes(v2IndexName, sstables);
verifySSTableIndexes(numericIndexIdentifier, sstables);
verifySSTableIndexes(literalIndexIdentifier, sstables);
}
@Test
@ -277,8 +279,8 @@ public class CompactionTest extends SAITester
assertNumRows(num, "SELECT id1 FROM %%s WHERE v1>=0");
assertNumRows(num, "SELECT id1 FROM %%s WHERE v2='0'");
verifySSTableIndexes(IndexMetadata.generateDefaultIndexName(currentTable(), V1_COLUMN_IDENTIFIER), sstables);
verifySSTableIndexes(IndexMetadata.generateDefaultIndexName(currentTable(), V2_COLUMN_IDENTIFIER), sstables);
verifySSTableIndexes(createIndexIdentifier(IndexMetadata.generateDefaultIndexName(currentTable(), V1_COLUMN_IDENTIFIER)), sstables);
verifySSTableIndexes(createIndexIdentifier(IndexMetadata.generateDefaultIndexName(currentTable(), V2_COLUMN_IDENTIFIER)), sstables);
}
@Test

27
test/unit/org/apache/cassandra/index/sai/functional/FailureTest.java
View File

@ -24,9 +24,10 @@ import org.junit.Test;
import org.apache.cassandra.db.marshal.Int32Type;
import org.apache.cassandra.index.IndexNotAvailableException;
import org.apache.cassandra.index.sai.IndexContext;
import org.apache.cassandra.index.sai.SAITester;
import org.apache.cassandra.index.sai.SSTableContext;
import org.apache.cassandra.index.sai.utils.IndexIdentifier;
import org.apache.cassandra.index.sai.utils.IndexTermType;
import org.apache.cassandra.inject.Injection;
import org.apache.cassandra.inject.Injections;
import org.assertj.core.api.Assertions;
@ -39,7 +40,8 @@ public class FailureTest extends SAITester
public void shouldMakeIndexNonQueryableOnSSTableContextFailureDuringFlush() throws Throwable
{
createTable(CREATE_TABLE_TEMPLATE);
IndexContext numericIndexContext = createIndexContext(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1")), Int32Type.instance);
IndexIdentifier indexIdentifier = createIndexIdentifier(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1")));
IndexTermType indexTermType = createIndexTermType(Int32Type.instance);
execute("INSERT INTO %s (id1, v1) VALUES ('1', 1)");
execute("INSERT INTO %s (id1, v1) VALUES ('2', 2)");
@ -47,8 +49,8 @@ public class FailureTest extends SAITester
assertEquals(1, execute("SELECT id1 FROM %s WHERE v1 > 1").size());
verifyIndexFiles(numericIndexContext, null, 1, 1, 0, 1, 0);
verifySSTableIndexes(numericIndexContext.getIndexName(), 1, 1);
verifyIndexFiles(indexTermType, indexIdentifier, 1, 1, 1);
verifySSTableIndexes(indexIdentifier, 1, 1);
execute("INSERT INTO %s (id1, v1) VALUES ('3', 3)");
@ -66,8 +68,8 @@ public class FailureTest extends SAITester
// Now verify that a restart actually repairs the index...
simulateNodeRestart();
verifyIndexFiles(numericIndexContext, null, 2, 0);
verifySSTableIndexes(numericIndexContext.getIndexName(), 2, 2);
verifyIndexFiles(indexTermType, indexIdentifier, 2, 2, 2);
verifySSTableIndexes(indexIdentifier, 2, 2);
assertEquals(2, execute("SELECT id1 FROM %s WHERE v1 > 1").size());
}
@ -76,7 +78,8 @@ public class FailureTest extends SAITester
public void shouldMakeIndexNonQueryableOnSSTableContextFailureDuringCompaction() throws Throwable
{
createTable(CREATE_TABLE_TEMPLATE);
IndexContext numericIndexContext = createIndexContext(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1")), Int32Type.instance);
IndexIdentifier indexIdentifier = createIndexIdentifier(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1")));
IndexTermType indexTermType = createIndexTermType(Int32Type.instance);
execute("INSERT INTO %s (id1, v1) VALUES ('1', 1)");
flush();
@ -86,8 +89,8 @@ public class FailureTest extends SAITester
assertEquals(1, execute("SELECT id1 FROM %s WHERE v1 > 1").size());
verifyIndexFiles(numericIndexContext, null, 2, 2, 0, 2, 0);
verifySSTableIndexes(numericIndexContext.getIndexName(), 2, 2);
verifyIndexFiles(indexTermType, indexIdentifier, 2, 2, 2);
verifySSTableIndexes(indexIdentifier, 2, 2);
Injection ssTableContextCreationFailure = newFailureOnEntry("context_failure_on_compaction", SSTableContext.class, "<init>", RuntimeException.class);
Injections.inject(ssTableContextCreationFailure);
@ -114,13 +117,13 @@ public class FailureTest extends SAITester
Injection ssTableContextCreationFailure = newFailureOnEntry("context_failure_on_creation", SSTableContext.class, "<init>", RuntimeException.class);
Injections.inject(ssTableContextCreationFailure);
String v2IndexName = createIndexAsync(String.format(CREATE_INDEX_TEMPLATE, "v2"));
IndexIdentifier indexIdentifier = createIndexIdentifier(createIndexAsync(String.format(CREATE_INDEX_TEMPLATE, "v2")));
// Verify that the initial index build fails...
verifyInitialIndexFailed(v2IndexName);
verifyInitialIndexFailed(indexIdentifier.indexName);
verifyNoIndexFiles();
verifySSTableIndexes(v2IndexName, 0);
verifySSTableIndexes(indexIdentifier, 0);
// ...and then verify that, while the node is still operational, the index is not.
Assertions.assertThatThrownBy(() -> execute("SELECT * FROM %s WHERE v2 = '1'"))

18
test/unit/org/apache/cassandra/index/sai/functional/FlushingTest.java
View File

@ -24,16 +24,17 @@ import org.junit.Test;
import com.datastax.driver.core.ResultSet;
import org.apache.cassandra.db.marshal.Int32Type;
import org.apache.cassandra.index.sai.IndexContext;
import org.apache.cassandra.index.sai.SAITester;
import org.apache.cassandra.index.sai.utils.IndexIdentifier;
import org.apache.cassandra.index.sai.disk.v1.bbtree.NumericIndexWriter;
import org.apache.cassandra.index.sai.utils.IndexTermType;
import static org.junit.Assert.assertEquals;
public class FlushingTest extends SAITester
{
@Test
public void testFlushingLargeStaleMemtableIndex() throws Throwable
public void testFlushingLargeStaleMemtableIndex()
{
createTable(CREATE_TABLE_TEMPLATE);
createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1"));
@ -51,10 +52,11 @@ public class FlushingTest extends SAITester
}
@Test
public void testFlushingOverwriteDelete() throws Throwable
public void testFlushingOverwriteDelete()
{
createTable(CREATE_TABLE_TEMPLATE);
IndexContext numericIndexContext = createIndexContext(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1")), Int32Type.instance);
IndexIdentifier indexIdentifier = createIndexIdentifier(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1")));
IndexTermType indexTermType = createIndexTermType(Int32Type.instance);
int sstables = 3;
for (int j = 0; j < sstables; j++)
@ -66,14 +68,14 @@ public class FlushingTest extends SAITester
ResultSet rows = executeNet("SELECT id1 FROM %s WHERE v1 >= 0");
assertEquals(0, rows.all().size());
verifyIndexFiles(numericIndexContext, null, sstables, 0, 0, sstables, 0);
verifySSTableIndexes(numericIndexContext.getIndexName(), sstables, 0);
verifyIndexFiles(indexTermType, indexIdentifier, sstables, 0, sstables);
verifySSTableIndexes(indexIdentifier, sstables, 0);
compact();
waitForAssert(() -> verifyIndexFiles(numericIndexContext, null, 1, 0, 0, 1, 0));
waitForAssert(() -> verifyIndexFiles(indexTermType, indexIdentifier, 1, 0, 1));
rows = executeNet("SELECT id1 FROM %s WHERE v1 >= 0");
assertEquals(0, rows.all().size());
verifySSTableIndexes(numericIndexContext.getIndexName(), 1, 0);
verifySSTableIndexes(indexIdentifier, 1, 0);
}
}

11
test/unit/org/apache/cassandra/index/sai/functional/GroupComponentsTest.java
View File

@ -28,11 +28,11 @@ import org.junit.Test;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.marshal.UTF8Type;
import org.apache.cassandra.index.sai.IndexContext;
import org.apache.cassandra.index.sai.SAITester;
import org.apache.cassandra.index.sai.StorageAttachedIndex;
import org.apache.cassandra.index.sai.StorageAttachedIndexGroup;
import org.apache.cassandra.index.sai.disk.format.Version;
import org.apache.cassandra.index.sai.utils.IndexTermType;
import org.apache.cassandra.io.sstable.Component;
import org.apache.cassandra.io.sstable.format.SSTableReader;
@ -61,7 +61,7 @@ public class GroupComponentsTest extends SAITester
// index files are released but not removed
cfs.invalidate(true, false);
Assert.assertTrue(index.getIndexContext().getView().getIndexes().isEmpty());
Assert.assertTrue(index.view().getIndexes().isEmpty());
for (Component component : components)
Assert.assertTrue(sstable.descriptor.fileFor(component).exists());
}
@ -91,7 +91,10 @@ public class GroupComponentsTest extends SAITester
public void getLiveComponentsForPopulatedIndex()
{
createTable("CREATE TABLE %s (pk int primary key, value text)");
IndexContext indexContext = createIndexContext(createIndex("CREATE INDEX ON %s(value) USING 'sai'"), UTF8Type.instance);
createIndex("CREATE INDEX ON %s(value) USING 'sai'");
IndexTermType indexTermType = createIndexTermType(UTF8Type.instance);
execute("INSERT INTO %s (pk, value) VALUES (1, '1')");
flush();
@ -106,7 +109,7 @@ public class GroupComponentsTest extends SAITester
Set<Component> components = StorageAttachedIndexGroup.getLiveComponents(sstables.iterator().next(), getIndexesFromGroup(group));
assertEquals(Version.LATEST.onDiskFormat().perSSTableIndexComponents(false).size() +
Version.LATEST.onDiskFormat().perColumnIndexComponents(indexContext).size(),
Version.LATEST.onDiskFormat().perColumnIndexComponents(indexTermType).size(),
components.size());
}

22
test/unit/org/apache/cassandra/index/sai/functional/NodeRestartTest.java
View File

@ -25,9 +25,10 @@ import org.junit.Test;
import org.apache.cassandra.db.marshal.Int32Type;
import org.apache.cassandra.db.marshal.UTF8Type;
import org.apache.cassandra.index.sai.IndexContext;
import org.apache.cassandra.index.sai.SAITester;
import org.apache.cassandra.index.sai.StorageAttachedIndex;
import org.apache.cassandra.index.sai.utils.IndexIdentifier;
import org.apache.cassandra.index.sai.utils.IndexTermType;
import org.apache.cassandra.inject.Injection;
import org.apache.cassandra.inject.Injections;
import org.apache.cassandra.inject.InvokePointBuilder;
@ -112,17 +113,20 @@ public class NodeRestartTest extends SAITester
execute("INSERT INTO %s (id1, v1, v2) VALUES ('0', 0, '0');");
flush();
IndexContext numericIndexContext = createIndexContext(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1")), Int32Type.instance);
IndexContext literalIndexContext = createIndexContext(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v2")), UTF8Type.instance);
waitForTableIndexesQueryable();
verifyIndexFiles(numericIndexContext,literalIndexContext, 1, 1);
IndexIdentifier numericIndexIdentifier = createIndexIdentifier(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1")));
IndexIdentifier literalIndexIdentifier = createIndexIdentifier(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v2")));
IndexTermType numericIndexTermType = createIndexTermType(Int32Type.instance);
IndexTermType literalIndexTermType = createIndexTermType(UTF8Type.instance);
verifyIndexFiles(numericIndexTermType, numericIndexIdentifier, 1);
verifyIndexFiles(literalIndexTermType, literalIndexIdentifier, 1);
assertNumRows(1, "SELECT * FROM %%s WHERE v1 >= 0");
assertNumRows(1, "SELECT * FROM %%s WHERE v2 = '0'");
assertValidationCount(0, 0);
simulateNodeRestart();
verifyIndexFiles(numericIndexContext, literalIndexContext, 1, 1);
verifyIndexFiles(numericIndexTermType, numericIndexIdentifier, 1);
verifyIndexFiles(literalIndexTermType, literalIndexIdentifier, 1);
assertNumRows(1, "SELECT * FROM %%s WHERE v1 >= 0");
assertNumRows(1, "SELECT * FROM %%s WHERE v2 = '0'");
@ -174,9 +178,11 @@ public class NodeRestartTest extends SAITester
execute("INSERT INTO %s (id1, v1, v2) VALUES ('0', 0, '0')");
flush();
IndexContext numericIndexContext = createIndexContext(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1")), Int32Type.instance);
IndexIdentifier numericIndexIdentifier = createIndexIdentifier(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1")));
IndexTermType numericIndexTermType = createIndexTermType(Int32Type.instance);
waitForTableIndexesQueryable();
verifyIndexFiles(numericIndexContext, null, 1, 0);
verifyIndexFiles(numericIndexTermType, numericIndexIdentifier, 1);
assertNumRows(1, "SELECT * FROM %%s WHERE v1 >= 0");
assertValidationCount(0, 0);
}

25
test/unit/org/apache/cassandra/index/sai/functional/SnapshotTest.java
View File

@ -22,8 +22,9 @@ import org.junit.Before;
import org.junit.Test;
import org.apache.cassandra.db.marshal.Int32Type;
import org.apache.cassandra.index.sai.IndexContext;
import org.apache.cassandra.index.sai.SAITester;
import org.apache.cassandra.index.sai.utils.IndexIdentifier;
import org.apache.cassandra.index.sai.utils.IndexTermType;
import org.apache.cassandra.inject.Injections;
import static org.junit.Assert.assertEquals;
@ -51,10 +52,11 @@ public class SnapshotTest extends SAITester
// Insert some initial data and create the index over it
execute("INSERT INTO %s (id1, v1) VALUES ('0', 0);");
IndexContext numericIndexContext = createIndexContext(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1")), Int32Type.instance);
IndexIdentifier indexIdentifier = createIndexIdentifier(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1")));
IndexTermType indexTermType = createIndexTermType(Int32Type.instance);
waitForTableIndexesQueryable();
flush();
verifyIndexFiles(numericIndexContext, null, 1, 1, 0, 1, 0);
verifyIndexFiles(indexTermType, indexIdentifier, 1, 1, 1);
// Note: This test will fail here if it is run on its own because the per-index validation
// is run if the node is starting up but validatation isn't done once the node is started
assertValidationCount(0, 0);
@ -63,7 +65,7 @@ public class SnapshotTest extends SAITester
// Add some data into a second sstable
execute("INSERT INTO %s (id1, v1) VALUES ('1', 0);");
flush();
verifyIndexFiles(numericIndexContext, null, 2, 2, 0, 2, 0);
verifyIndexFiles(indexTermType, indexIdentifier, 2, 2, 2);
assertValidationCount(0, 0);
// Take a snapshot recording the index files last modified date
@ -79,7 +81,7 @@ public class SnapshotTest extends SAITester
// Add some data into a third sstable, out of the scope of our snapshot
execute("INSERT INTO %s (id1, v1) VALUES ('2', 0);");
flush();
verifyIndexFiles(numericIndexContext, null, 3, 3, 0, 3, 0);
verifyIndexFiles(indexTermType, indexIdentifier, 3, 3, 3);
assertNumRows(3, "SELECT * FROM %%s WHERE v1 >= 0");
assertValidationCount(0, 0);
@ -91,7 +93,7 @@ public class SnapshotTest extends SAITester
// Restore the snapshot, only the two first sstables should be restored
restoreSnapshot(snapshot);
verifyIndexFiles(numericIndexContext, null, 2, 2, 0, 2, 0);
verifyIndexFiles(indexTermType, indexIdentifier, 2, 2, 2);
assertEquals(snapshotLastModified, indexFilesLastModified());
assertNumRows(2, "SELECT * FROM %%s WHERE v1 >= 0");
assertValidationCount(2, 2); // newly loaded
@ -100,8 +102,8 @@ public class SnapshotTest extends SAITester
verifyIndexComponentsIncludedInSSTable();
// Rebuild the index to verify that the index files are overridden
rebuildIndexes(numericIndexContext.getIndexName());
verifyIndexFiles(numericIndexContext, null, 2, 0);
rebuildIndexes(indexIdentifier.indexName);
verifyIndexFiles(indexTermType, indexIdentifier, 2);
assertNotEquals(snapshotLastModified, indexFilesLastModified());
assertNumRows(2, "SELECT * FROM %%s WHERE v1 >= 0");
assertValidationCount(2, 2); // compaction should not validate
@ -128,9 +130,10 @@ public class SnapshotTest extends SAITester
verifyIndexComponentsNotIncludedInSSTable();
// create index
IndexContext numericIndexContext = createIndexContext(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1")), Int32Type.instance);
IndexIdentifier indexIdentifier = createIndexIdentifier(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1")));
IndexTermType indexTermType = createIndexTermType(Int32Type.instance);
waitForTableIndexesQueryable();
verifyIndexFiles(numericIndexContext, null, 2, 0);
verifyIndexFiles(indexTermType, indexIdentifier, 2);
assertValidationCount(0, 0);
// index components are included after initial build
@ -154,7 +157,7 @@ public class SnapshotTest extends SAITester
// Restore the snapshot
restoreSnapshot(snapshot);
verifyIndexFiles(numericIndexContext, null, 2, 0);
verifyIndexFiles(indexTermType, indexIdentifier, 2);
assertEquals(snapshotLastModified, indexFilesLastModified());
assertNumRows(2, "SELECT * FROM %%s WHERE v1 >= 0");
assertValidationCount(2, 2); // newly loaded

29
test/unit/org/apache/cassandra/index/sai/memory/TrieMemoryIndexTest.java
View File

@ -34,6 +34,7 @@ import org.junit.Test;
import org.apache.cassandra.cql3.Operator;
import org.apache.cassandra.cql3.statements.schema.IndexTarget;
import org.apache.cassandra.db.Clustering;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.db.PartitionPosition;
import org.apache.cassandra.db.marshal.AbstractType;
@ -45,17 +46,14 @@ import org.apache.cassandra.dht.ExcludingBounds;
import org.apache.cassandra.dht.IncludingExcludingBounds;
import org.apache.cassandra.dht.Murmur3Partitioner;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.index.TargetParser;
import org.apache.cassandra.index.sai.IndexContext;
import org.apache.cassandra.index.sai.StorageAttachedIndex;
import org.apache.cassandra.index.sai.plan.Expression;
import org.apache.cassandra.index.sai.iterators.KeyRangeIterator;
import org.apache.cassandra.index.sai.plan.Expression;
import org.apache.cassandra.index.sai.utils.PrimaryKeys;
import org.apache.cassandra.index.sai.utils.SAIRandomizedTester;
import org.apache.cassandra.index.sai.utils.TypeUtil;
import org.apache.cassandra.schema.CachingParams;
import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.schema.IndexMetadata;
import org.apache.cassandra.schema.MockSchema;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.Pair;
@ -74,7 +72,7 @@ public class TrieMemoryIndexTest extends SAIRandomizedTester
private static final DecoratedKey key = Murmur3Partitioner.instance.decorateKey(ByteBufferUtil.bytes("key"));
private IndexContext indexContext;
private StorageAttachedIndex index;
@Test
public void heapGrowsAsDataIsAddedTest()
@ -169,7 +167,7 @@ public class TrieMemoryIndexTest extends SAIRandomizedTester
private Expression generateRandomExpression()
{
Expression expression = new Expression(indexContext);
Expression expression = Expression.create(index);
int equality = getRandom().nextIntBetween(0, 100);
int lower = getRandom().nextIntBetween(0, 75);
@ -216,7 +214,7 @@ public class TrieMemoryIndexTest extends SAIRandomizedTester
assertEquals(1, pair.right.size());
final int rowId = i;
final ByteComparable expectedByteComparable = TypeUtil.isLiteral(type)
final ByteComparable expectedByteComparable = index.index.termType().isLiteral()
? ByteComparable.fixedLength(decompose.apply(rowId))
: version -> type.asComparableBytes(decompose.apply(rowId), version);
final ByteComparable actualByteComparable = pair.left;
@ -241,15 +239,10 @@ public class TrieMemoryIndexTest extends SAIRandomizedTester
options.put("target", REG_COL);
IndexMetadata indexMetadata = IndexMetadata.fromSchemaMetadata("col_index", IndexMetadata.Kind.CUSTOM, options);
Pair<ColumnMetadata, IndexTarget.Type> target = TargetParser.parse(table, indexMetadata);
indexContext = new IndexContext(table.keyspace,
table.name,
table.partitionKeyType,
table.partitioner,
table.comparator,
target.left,
target.right,
indexMetadata);
return new TrieMemoryIndex(indexContext);
ColumnFamilyStore cfs = MockSchema.newCFS(table);
index = new StorageAttachedIndex(cfs, indexMetadata);
return new TrieMemoryIndex(index);
}
}

19
test/unit/org/apache/cassandra/index/sai/memory/VectorMemoryIndexTest.java
View File

@ -56,9 +56,9 @@ import org.apache.cassandra.dht.ExcludingBounds;
import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.dht.IncludingExcludingBounds;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.index.sai.IndexContext;
import org.apache.cassandra.index.sai.QueryContext;
import org.apache.cassandra.index.sai.SAITester;
import org.apache.cassandra.index.sai.StorageAttachedIndex;
import org.apache.cassandra.index.sai.iterators.KeyRangeIterator;
import org.apache.cassandra.index.sai.plan.Expression;
import org.apache.cassandra.index.sai.utils.PrimaryKey;
@ -66,7 +66,6 @@ import org.apache.cassandra.index.sai.utils.RangeUtil;
import org.apache.cassandra.inject.Injections;
import org.apache.cassandra.inject.InvokePointBuilder;
import org.apache.cassandra.locator.TokenMetadata;
import org.apache.cassandra.schema.MockSchema;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.utils.FBUtilities;
@ -85,7 +84,7 @@ public class VectorMemoryIndexTest extends SAITester
.build();
private ColumnFamilyStore cfs;
private IndexContext indexContext;
private StorageAttachedIndex index;
private VectorMemoryIndex memtableIndex;
private IPartitioner partitioner;
private Map<DecoratedKey, Integer> keyMap;
@ -104,14 +103,10 @@ public class VectorMemoryIndexTest extends SAITester
TokenMetadata metadata = StorageService.instance.getTokenMetadata();
metadata.updateNormalTokens(BootStrapper.getRandomTokens(metadata, 10), FBUtilities.getBroadcastAddressAndPort());
TableMetadata tableMetadata = TableMetadata.builder("ks", "tb")
.addPartitionKeyColumn("pk", Int32Type.instance)
.addRegularColumn("val", Int32Type.instance)
.build();
cfs = MockSchema.newCFS(tableMetadata);
partitioner = cfs.getPartitioner();
dimensionCount = getRandom().nextIntBetween(2, 2048);
indexContext = SAITester.createIndexContext("index", VectorType.getInstance(FloatType.instance, dimensionCount));
index = SAITester.createMockIndex("index", VectorType.getInstance(FloatType.instance, dimensionCount));
cfs = index.baseCfs();
partitioner = cfs.getPartitioner();
indexSearchCounter.reset();
keyMap = new TreeMap<>();
rowMap = new HashMap<>();
@ -122,7 +117,7 @@ public class VectorMemoryIndexTest extends SAITester
@Test
public void randomQueryTest() throws Exception
{
memtableIndex = new VectorMemoryIndex(indexContext);
memtableIndex = new VectorMemoryIndex(index);
for (int row = 0; row < getRandom().nextIntBetween(1000, 5000); row++)
{
@ -186,7 +181,7 @@ public class VectorMemoryIndexTest extends SAITester
private Expression generateRandomExpression()
{
Expression expression = new Expression(indexContext);
Expression expression = Expression.create(index);
expression.add(Operator.ANN, randomVector());
return expression;
}

Some files were not shown because too many files have changed in this diff Show More