From 0e42b77c9735d1124fe0a5766447f29c891cdb5b Mon Sep 17 00:00:00 2001 From: Mike Adamson Date: Fri, 10 Nov 2023 14:49:41 +0000 Subject: [PATCH] Improve code model around IndexContext MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Replace IndexContext with IndexTermType and IndexDefinition - Move index specific managers, factories and metrics to StorageAttachedIndex - Refactor Expression to explicitly define indexed and unindexed expressions patch by Mike Adamson; reviewed by Andres de la Peña, Caleb Rackliffe for CASSANDRA-18166 --- CHANGES.txt | 1 + .../cassandra/index/sai/IndexContext.java | 588 ------------ .../cassandra/index/sai/SSTableContext.java | 4 +- .../index/sai/StorageAttachedIndex.java | 865 ++++++++++------- .../sai/StorageAttachedIndexBuilder.java | 4 +- .../StorageAttachedIndexBuildingSupport.java | 72 ++ .../index/sai/StorageAttachedIndexGroup.java | 22 +- .../index/sai/analyzer/AbstractAnalyzer.java | 13 +- .../index/sai/analyzer/NoOpAnalyzer.java | 66 -- .../sai/analyzer/NonTokenizingAnalyzer.java | 24 +- .../sai/disk/IndexSearchResultIterator.java | 8 +- .../index/sai/disk/SSTableIndex.java | 41 +- .../sai/disk/format/IndexDescriptor.java | 108 ++- .../index/sai/disk/format/OnDiskFormat.java | 30 +- .../index/sai/disk/format/Version.java | 20 +- .../disk/v1/ColumnCompletionMarkerUtil.java | 14 +- .../index/sai/disk/v1/IndexWriterConfig.java | 8 +- .../sai/disk/v1/MemtableIndexWriter.java | 62 +- .../index/sai/disk/v1/MetadataSource.java | 6 +- .../sai/disk/v1/PerColumnIndexFiles.java | 15 +- .../index/sai/disk/v1/SSTableIndexWriter.java | 98 +- .../index/sai/disk/v1/V1OnDiskFormat.java | 44 +- .../index/sai/disk/v1/V1SSTableIndex.java | 25 +- .../v1/bbtree/BlockBalancedTreeIterator.java | 7 +- .../BlockBalancedTreePostingsWriter.java | 9 +- .../v1/bbtree/BlockBalancedTreeQueries.java | 23 +- .../v1/bbtree/BlockBalancedTreeReader.java | 14 +- .../disk/v1/bbtree/NumericIndexWriter.java | 30 +- .../v1/postings/PostingListRangeIterator.java | 16 +- .../sai/disk/v1/postings/PostingsWriter.java | 10 +- .../disk/v1/segment/IndexSegmentSearcher.java | 22 +- .../segment/LiteralIndexSegmentSearcher.java | 25 +- .../LiteralIndexSegmentTermsReader.java | 10 +- .../segment/NumericIndexSegmentSearcher.java | 22 +- .../index/sai/disk/v1/segment/Segment.java | 8 +- .../sai/disk/v1/segment/SegmentBuilder.java | 63 +- .../segment/VectorIndexSegmentSearcher.java | 25 +- .../sai/disk/v1/trie/LiteralIndexWriter.java | 16 +- .../v1/trie/TrieTermsDictionaryWriter.java | 8 +- .../disk/v1/vector/CheckpointingIterator.java | 101 -- .../index/sai/disk/v1/vector/DiskAnn.java | 6 +- .../index/sai/disk/v1/vector/OnHeapGraph.java | 10 +- .../index/sai/memory/MemoryIndex.java | 11 +- .../index/sai/memory/MemtableIndex.java | 31 +- .../sai/memory/MemtableIndexManager.java | 24 +- .../index/sai/memory/TrieMemoryIndex.java | 133 +-- .../index/sai/memory/VectorMemoryIndex.java | 40 +- .../index/sai/metrics/AbstractMetrics.java | 6 + .../index/sai/metrics/ColumnQueryMetrics.java | 14 +- .../index/sai/metrics/IndexMetrics.java | 13 +- .../cassandra/index/sai/plan/Expression.java | 270 ++++-- .../cassandra/index/sai/plan/FilterTree.java | 12 +- .../cassandra/index/sai/plan/Operation.java | 160 +++- .../index/sai/plan/QueryController.java | 43 +- .../index/sai/plan/QueryViewBuilder.java | 8 +- .../plan/StorageAttachedIndexQueryPlan.java | 31 +- .../plan/StorageAttachedIndexSearcher.java | 13 +- .../index/sai/plan/VectorTopKProcessor.java | 26 +- .../index/sai/utils/IndexIdentifier.java | 81 ++ .../index/sai/utils/IndexTermType.java | 868 ++++++++++++++++++ .../cassandra/index/sai/utils/TypeUtil.java | 522 ----------- .../index/sai/view/IndexViewManager.java | 81 +- .../index/sai/view/RangeTermTree.java | 51 +- .../apache/cassandra/index/sai/view/View.java | 13 +- .../sai/virtual/ColumnIndexesSystemView.java | 17 +- .../sai/virtual/SSTableIndexesSystemView.java | 15 +- .../index/sai/virtual/SegmentsSystemView.java | 15 +- .../test/sai/IndexStreamingFailureTest.java | 17 +- .../test/microbench/sai/KeyLookupBench.java | 3 +- .../apache/cassandra/index/sai/SAITester.java | 156 ++-- .../analyzer/NonTokenizingAnalyzerTest.java | 3 +- .../sai/cql/StorageAttachedIndexDDLTest.java | 245 ++--- .../sai/cql/UnindexedExpressionsTest.java | 106 +++ .../index/sai/cql/types/DataSet.java | 8 +- .../sai/cql/types/NumericTypeSortingTest.java | 18 +- .../index/sai/disk/NodeStartupTest.java | 28 +- .../sai/disk/format/IndexDescriptorTest.java | 9 +- .../index/sai/disk/format/VersionTest.java | 2 +- .../v1/BalancedTreeIndexSearcherTest.java | 40 +- .../disk/v1/InvertedIndexSearcherTest.java | 42 +- .../index/sai/disk/v1/MetadataTest.java | 23 +- .../index/sai/disk/v1/SegmentFlushTest.java | 32 +- .../index/sai/disk/v1/SegmentTest.java | 2 +- .../index/sai/disk/v1/TermsReaderTest.java | 30 +- .../bbtree/BlockBalancedTreeIndexBuilder.java | 48 +- .../BlockBalancedTreePostingsWriterTest.java | 27 +- .../bbtree/BlockBalancedTreeQueriesTest.java | 5 +- .../BlockBalancedTreeRamBufferTest.java | 6 +- .../bbtree/BlockBalancedTreeReaderTest.java | 16 +- .../v1/bbtree/NumericIndexWriterTest.java | 30 +- .../sai/disk/v1/postings/PostingsTest.java | 45 +- .../disk/v1/trie/TrieTermsDictionaryTest.java | 21 +- .../index/sai/functional/CompactionTest.java | 34 +- .../index/sai/functional/FailureTest.java | 27 +- .../index/sai/functional/FlushingTest.java | 18 +- .../sai/functional/GroupComponentsTest.java | 11 +- .../index/sai/functional/NodeRestartTest.java | 22 +- .../index/sai/functional/SnapshotTest.java | 25 +- .../index/sai/memory/TrieMemoryIndexTest.java | 29 +- .../sai/memory/VectorMemoryIndexTest.java | 19 +- .../sai/metrics/IndexGroupMetricsTest.java | 7 +- .../index/sai/plan/ExpressionTest.java | 12 +- .../index/sai/plan/OperationTest.java | 109 +-- .../sai/utils/AbstractPrimaryKeyTester.java | 2 +- ...peUtilTest.java => IndexTermTypeTest.java} | 140 +-- .../sai/virtual/IndexesSystemViewTest.java | 4 +- .../sai/virtual/SSTablesSystemViewTest.java | 2 +- .../sai/virtual/SegmentsSystemViewTest.java | 22 +- 108 files changed, 3440 insertions(+), 3066 deletions(-) delete mode 100644 src/java/org/apache/cassandra/index/sai/IndexContext.java create mode 100644 src/java/org/apache/cassandra/index/sai/StorageAttachedIndexBuildingSupport.java delete mode 100644 src/java/org/apache/cassandra/index/sai/analyzer/NoOpAnalyzer.java delete mode 100644 src/java/org/apache/cassandra/index/sai/disk/v1/vector/CheckpointingIterator.java create mode 100644 src/java/org/apache/cassandra/index/sai/utils/IndexIdentifier.java create mode 100644 src/java/org/apache/cassandra/index/sai/utils/IndexTermType.java delete mode 100644 src/java/org/apache/cassandra/index/sai/utils/TypeUtil.java create mode 100644 test/unit/org/apache/cassandra/index/sai/cql/UnindexedExpressionsTest.java rename test/unit/org/apache/cassandra/index/sai/utils/{TypeUtilTest.java => IndexTermTypeTest.java} (55%) diff --git a/CHANGES.txt b/CHANGES.txt index 32f4c5efbd..dc0bfeb9d5 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ 5.0-beta1 + * Improve SAI IndexContext handling of indexed and non-indexed columns in queries (CASSANDRA-18166) * Fixed bug where UnifiedCompactionTask constructor was calling the wrong base constructor of CompactionTask (CASSANDRA-18757) * Fix SAI unindexed contexts not considering CONTAINS KEY (CASSANDRA-19040) * Ensure that empty SAI column indexes do not fail on validation after full-SSTable streaming (CASSANDRA-19017) diff --git a/src/java/org/apache/cassandra/index/sai/IndexContext.java b/src/java/org/apache/cassandra/index/sai/IndexContext.java deleted file mode 100644 index b88d59b6e6..0000000000 --- a/src/java/org/apache/cassandra/index/sai/IndexContext.java +++ /dev/null @@ -1,588 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.cassandra.index.sai; - -import java.nio.ByteBuffer; -import java.util.Collection; -import java.util.Collections; -import java.util.HashSet; -import java.util.Iterator; -import java.util.Objects; -import java.util.Set; -import java.util.concurrent.TimeUnit; -import javax.annotation.Nullable; - -import com.google.common.base.MoreObjects; -import com.google.common.collect.ImmutableSet; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import org.apache.cassandra.cql3.Operator; -import org.apache.cassandra.cql3.statements.schema.IndexTarget; -import org.apache.cassandra.db.ClusteringComparator; -import org.apache.cassandra.db.DecoratedKey; -import org.apache.cassandra.db.marshal.AbstractType; -import org.apache.cassandra.db.marshal.AsciiType; -import org.apache.cassandra.db.marshal.BooleanType; -import org.apache.cassandra.db.marshal.CompositeType; -import org.apache.cassandra.db.marshal.FloatType; -import org.apache.cassandra.db.marshal.UTF8Type; -import org.apache.cassandra.db.marshal.UUIDType; -import org.apache.cassandra.db.marshal.VectorType; -import org.apache.cassandra.db.rows.Cell; -import org.apache.cassandra.db.rows.Row; -import org.apache.cassandra.dht.IPartitioner; -import org.apache.cassandra.index.sai.analyzer.AbstractAnalyzer; -import org.apache.cassandra.index.sai.disk.SSTableIndex; -import org.apache.cassandra.index.sai.disk.format.Version; -import org.apache.cassandra.index.sai.disk.v1.IndexWriterConfig; -import org.apache.cassandra.index.sai.memory.MemtableIndexManager; -import org.apache.cassandra.index.sai.metrics.ColumnQueryMetrics; -import org.apache.cassandra.index.sai.metrics.IndexMetrics; -import org.apache.cassandra.index.sai.plan.Expression; -import org.apache.cassandra.index.sai.utils.PrimaryKey; -import org.apache.cassandra.index.sai.utils.TypeUtil; -import org.apache.cassandra.index.sai.view.IndexViewManager; -import org.apache.cassandra.index.sai.view.View; -import org.apache.cassandra.io.sstable.format.SSTableReader; -import org.apache.cassandra.schema.ColumnMetadata; -import org.apache.cassandra.schema.IndexMetadata; -import org.apache.cassandra.service.ClientWarn; -import org.apache.cassandra.utils.FBUtilities; -import org.apache.cassandra.utils.NoSpamLogger; -import org.apache.cassandra.utils.Pair; - -import static org.apache.cassandra.config.CassandraRelevantProperties.SAI_MAX_FROZEN_TERM_SIZE; -import static org.apache.cassandra.config.CassandraRelevantProperties.SAI_MAX_STRING_TERM_SIZE; -import static org.apache.cassandra.config.CassandraRelevantProperties.SAI_MAX_VECTOR_TERM_SIZE; - -/** - * Manages metadata for each column index. - */ -public class IndexContext -{ - private static final Logger logger = LoggerFactory.getLogger(IndexContext.class); - private static final NoSpamLogger noSpamLogger = NoSpamLogger.getLogger(logger, 1, TimeUnit.MINUTES); - - public static final long MAX_STRING_TERM_SIZE = SAI_MAX_STRING_TERM_SIZE.getSizeInBytes(); - public static final long MAX_FROZEN_TERM_SIZE = SAI_MAX_FROZEN_TERM_SIZE.getSizeInBytes(); - public static final long MAX_VECTOR_TERM_SIZE = SAI_MAX_VECTOR_TERM_SIZE.getSizeInBytes(); - public static final String TERM_OVERSIZE_MESSAGE = "Can't add term of column %s to index for key: %s, term size %s " + - "max allowed size %s, use analyzed = true (if not yet set) for that column."; - - private static final Set> EQ_ONLY_TYPES = ImmutableSet.of(UTF8Type.instance, - AsciiType.instance, - BooleanType.instance, - UUIDType.instance); - - private final AbstractType partitionKeyType; - private final ClusteringComparator clusteringComparator; - - private final String keyspace; - private final String table; - private final ColumnMetadata columnMetadata; - private final IndexTarget.Type indexType; - private final AbstractType validator; - - // Config can be null if the column context is "fake" (i.e. created for a filtering expression). - @Nullable - private final IndexMetadata indexMetadata; - - private final MemtableIndexManager memtableIndexManager; - - private final IndexViewManager viewManager; - private final IndexMetrics indexMetrics; - private final ColumnQueryMetrics columnQueryMetrics; - private final IndexWriterConfig indexWriterConfig; - private final AbstractAnalyzer.AnalyzerFactory analyzerFactory; - private final PrimaryKey.Factory primaryKeyFactory; - - private final long maxTermSize; - - public IndexContext(String keyspace, - String table, - AbstractType partitionKeyType, - IPartitioner partitioner, - ClusteringComparator clusteringComparator, - ColumnMetadata columnMetadata, - IndexTarget.Type indexType, - @Nullable IndexMetadata indexMetadata) - { - this.keyspace = Objects.requireNonNull(keyspace); - this.table = Objects.requireNonNull(table); - this.partitionKeyType = Objects.requireNonNull(partitionKeyType); - this.clusteringComparator = Objects.requireNonNull(clusteringComparator); - this.columnMetadata = Objects.requireNonNull(columnMetadata); - this.indexType = Objects.requireNonNull(indexType); - this.validator = TypeUtil.cellValueType(columnMetadata, indexType); - this.primaryKeyFactory = new PrimaryKey.Factory(partitioner, clusteringComparator); - - this.indexMetadata = indexMetadata; - this.memtableIndexManager = indexMetadata == null ? null : new MemtableIndexManager(this); - - this.indexWriterConfig = indexMetadata == null ? IndexWriterConfig.emptyConfig() - : IndexWriterConfig.fromOptions(indexMetadata.name, validator, indexMetadata.options); - this.indexMetrics = indexMetadata == null ? null : new IndexMetrics(this); - this.viewManager = new IndexViewManager(this); - this.columnQueryMetrics = isLiteral() ? new ColumnQueryMetrics.TrieIndexMetrics(this) - : new ColumnQueryMetrics.BalancedTreeIndexMetrics(this); - - this.analyzerFactory = indexMetadata == null ? AbstractAnalyzer.fromOptions(getValidator(), Collections.emptyMap()) - : AbstractAnalyzer.fromOptions(getValidator(), indexMetadata.options); - - maxTermSize = isVector() ? MAX_VECTOR_TERM_SIZE : (isFrozen() ? MAX_FROZEN_TERM_SIZE : MAX_STRING_TERM_SIZE); - } - - public boolean hasClustering() - { - return clusteringComparator.size() > 0; - } - - public AbstractType keyValidator() - { - return partitionKeyType; - } - - public PrimaryKey.Factory keyFactory() - { - return primaryKeyFactory; - } - - public String getKeyspace() - { - return keyspace; - } - - public IndexWriterConfig getIndexWriterConfig() - { - return indexWriterConfig; - } - - public IndexMetrics getIndexMetrics() - { - return indexMetrics; - } - - public ColumnQueryMetrics getColumnQueryMetrics() - { - return columnQueryMetrics; - } - - public String getTable() - { - return table; - } - - public IndexMetadata getIndexMetadata() - { - return indexMetadata; - } - - /** - * @return A set of SSTables which have attached to them invalid index components. - */ - public Collection onSSTableChanged(Collection oldSSTables, Collection newSSTables, IndexValidation validation) - { - return viewManager.update(oldSSTables, newSSTables, validation); - } - - public ColumnMetadata getDefinition() - { - return columnMetadata; - } - - public AbstractType getValidator() - { - return validator; - } - - public boolean isNonFrozenCollection() - { - return TypeUtil.isNonFrozenCollection(columnMetadata.type); - } - - public boolean isFrozen() - { - return TypeUtil.isFrozen(columnMetadata.type); - } - - public String getColumnName() - { - return columnMetadata.name.toString(); - } - - @Nullable - public String getIndexName() - { - return indexMetadata == null ? null : indexMetadata.name; - } - - /** - * Returns an {@link AbstractAnalyzer.AnalyzerFactory} for use by write and query paths to transform - * literal values. - */ - public AbstractAnalyzer.AnalyzerFactory getAnalyzerFactory() - { - return analyzerFactory; - } - - public View getView() - { - return viewManager.getView(); - } - - public MemtableIndexManager getMemtableIndexManager() - { - assert memtableIndexManager != null : "Attempt to use memtable index manager on non-indexed context"; - - return memtableIndexManager; - } - - /** - * @return total number of per-index open files - */ - public int openPerIndexFiles() - { - return viewManager.getView().size() * Version.LATEST.onDiskFormat().openFilesPerColumnIndex(this); - } - - public void drop(Collection sstablesToRebuild) - { - viewManager.drop(sstablesToRebuild); - } - - public boolean isNotIndexed() - { - return indexMetadata == null; - } - - /** - * Called when index is dropped. Clear all live in-memory indexes and close - * analyzer factories. Mark all {@link SSTableIndex} as released and per-column index files - * will be removed when in-flight queries are completed. - */ - public void invalidate() - { - viewManager.invalidate(); - analyzerFactory.close(); - if (memtableIndexManager != null) - memtableIndexManager.invalidate(); - if (indexMetrics != null) - indexMetrics.release(); - if (columnQueryMetrics != null) - columnQueryMetrics.release(); - } - - public boolean supports(Operator op) - { - if (op == Operator.LIKE || - op == Operator.LIKE_CONTAINS || - op == Operator.LIKE_PREFIX || - op == Operator.LIKE_MATCHES || - op == Operator.LIKE_SUFFIX) return false; - - // ANN is only supported against vectors, and vector indexes only support ANN - if (isVector()) - return op == Operator.ANN; - if (op == Operator.ANN) - return false; - - Expression.IndexOperator operator = Expression.IndexOperator.valueOf(op); - - if (isNonFrozenCollection()) - { - if (indexType == IndexTarget.Type.KEYS) return operator == Expression.IndexOperator.CONTAINS_KEY; - if (indexType == IndexTarget.Type.VALUES) return operator == Expression.IndexOperator.CONTAINS_VALUE; - return indexType == IndexTarget.Type.KEYS_AND_VALUES && operator == Expression.IndexOperator.EQ; - } - - if (indexType == IndexTarget.Type.FULL) - return operator == Expression.IndexOperator.EQ; - - AbstractType validator = getValidator(); - - if (operator != Expression.IndexOperator.EQ && EQ_ONLY_TYPES.contains(validator)) return false; - - // RANGE only applicable to non-literal indexes - return (operator != null) && !(TypeUtil.isLiteral(validator) && operator == Expression.IndexOperator.RANGE); - } - - public ByteBuffer getValueOf(DecoratedKey key, Row row, long nowInSecs) - { - if (row == null) - return null; - - switch (columnMetadata.kind) - { - case PARTITION_KEY: - return partitionKeyType instanceof CompositeType - ? CompositeType.extractComponent(key.getKey(), columnMetadata.position()) - : key.getKey(); - case CLUSTERING: - // skip indexing of static clustering when regular column is indexed - return row.isStatic() ? null : row.clustering().bufferAt(columnMetadata.position()); - - // treat static cell retrieval the same was as regular - // only if row kind is STATIC otherwise return null - case STATIC: - if (!row.isStatic()) - return null; - case REGULAR: - Cell cell = row.getCell(columnMetadata); - return cell == null || !cell.isLive(nowInSecs) ? null : cell.buffer(); - - default: - return null; - } - } - - public Iterator getValuesOf(Row row, long nowInSecs) - { - if (row == null) - return null; - - switch (columnMetadata.kind) - { - // treat static cell retrieval the same was as regular - // only if row kind is STATIC otherwise return null - case STATIC: - if (!row.isStatic()) - return null; - case REGULAR: - return TypeUtil.collectionIterator(validator, - row.getComplexColumnData(columnMetadata), - columnMetadata, - indexType, - nowInSecs); - - default: - return null; - } - } - - @Override - public String toString() - { - return MoreObjects.toStringHelper(this) - .add("columnName", getColumnName()) - .add("indexName", getIndexName()) - .toString(); - } - - public boolean isLiteral() - { - return TypeUtil.isLiteral(getValidator()); - } - - public boolean isVector() - { - return getValidator().isVector() && ((VectorType)getValidator()).elementType instanceof FloatType; - } - - @Override - public boolean equals(Object obj) - { - if (obj == this) - return true; - - if (!(obj instanceof IndexContext)) - return false; - - IndexContext other = (IndexContext) obj; - - return Objects.equals(columnMetadata, other.columnMetadata) && - (indexType == other.indexType) && - Objects.equals(indexMetadata, other.indexMetadata) && - Objects.equals(partitionKeyType, other.partitionKeyType) && - Objects.equals(clusteringComparator, other.clusteringComparator); - } - - @Override - public int hashCode() - { - return Objects.hash(columnMetadata, indexType, indexMetadata, partitionKeyType, clusteringComparator); - } - - /** - * A helper method for constructing consistent log messages for specific column indexes. - *

- * Example: For the index "idx" in keyspace "ks" on table "tb", calling this method with the raw message - * "Flushing new index segment..." will produce... - *

- * "[ks.tb.idx] Flushing new index segment..." - * - * @param message The raw content of a logging message, without information identifying it with an index. - * - * @return A log message with the proper keyspace, table and index name prepended to it. - */ - public String logMessage(String message) - { - // Index names are unique only within a keyspace. - return String.format("[%s.%s.%s] %s", keyspace, table, indexMetadata == null ? "?" : indexMetadata.name, message); - } - - /** - * @return the indexes that are built on the given SSTables on the left and corrupted indexes' - * corresponding contexts on the right - */ - public Pair, Collection> getBuiltIndexes(Collection sstableContexts, IndexValidation validation) - { - Set valid = new HashSet<>(sstableContexts.size()); - Set invalid = new HashSet<>(); - - for (SSTableContext sstableContext : sstableContexts) - { - if (sstableContext.sstable.isMarkedCompacted()) - continue; - - if (!sstableContext.indexDescriptor.isPerColumnIndexBuildComplete(this)) - { - logger.debug(logMessage("An on-disk index build for SSTable {} has not completed."), sstableContext.descriptor()); - continue; - } - - if (sstableContext.indexDescriptor.isIndexEmpty(this)) - { - logger.debug(logMessage("No on-disk index was built for SSTable {} because the SSTable " + - "had no indexable rows for the index."), sstableContext.descriptor()); - continue; - } - - try - { - if (validation != IndexValidation.NONE) - { - if (!sstableContext.indexDescriptor.validatePerIndexComponents(this, validation)) - { - invalid.add(sstableContext); - continue; - } - } - - SSTableIndex index = sstableContext.newSSTableIndex(this); - logger.debug(logMessage("Successfully created index for SSTable {}."), sstableContext.descriptor()); - - // Try to add new index to the set, if set already has such index, we'll simply release and move on. - // This covers situation when SSTable collection has the same SSTable multiple - // times because we don't know what kind of collection it actually is. - if (!valid.add(index)) - { - index.release(); - } - } - catch (Throwable e) - { - logger.warn(logMessage("Failed to update per-column components for SSTable {}"), sstableContext.descriptor(), e); - invalid.add(sstableContext); - } - } - - return Pair.create(valid, invalid); - } - - /** - * @return the number of indexed rows in this index (aka. a pair of term and rowId) - */ - public long getCellCount() - { - return getView().getIndexes() - .stream() - .mapToLong(SSTableIndex::getRowCount) - .sum(); - } - - /** - * @return the total size (in bytes) of per-column index components - */ - public long diskUsage() - { - return getView().getIndexes() - .stream() - .mapToLong(SSTableIndex::sizeOfPerColumnComponents) - .sum(); - } - - /** - * @return the total memory usage (in bytes) of per-column index on-disk data structure - */ - public long indexFileCacheSize() - { - return getView().getIndexes() - .stream() - .mapToLong(SSTableIndex::indexFileCacheSize) - .sum(); - } - - /** - * Validate maximum term size for given row - */ - public void validateMaxTermSizeForRow(DecoratedKey key, Row row, boolean sendClientWarning) - { - AbstractAnalyzer analyzer = getAnalyzerFactory().create(); - if (isNonFrozenCollection()) - { - Iterator bufferIterator = getValuesOf(row, FBUtilities.nowInSeconds()); - while (bufferIterator != null && bufferIterator.hasNext()) - validateMaxTermSizeForCell(analyzer, key, bufferIterator.next(), sendClientWarning); - } - else - { - ByteBuffer value = getValueOf(key, row, FBUtilities.nowInSeconds()); - validateMaxTermSizeForCell(analyzer, key, value, sendClientWarning); - } - } - - private void validateMaxTermSizeForCell(AbstractAnalyzer analyzer, DecoratedKey key, @Nullable ByteBuffer cellBuffer, boolean sendClientWarning) - { - if (cellBuffer == null || cellBuffer.remaining() == 0) - return; - - // analyzer should not return terms that are larger than the origin value. - if (cellBuffer.remaining() <= maxTermSize) - return; - - analyzer.reset(cellBuffer.duplicate()); - while (analyzer.hasNext()) - validateMaxTermSize(key, analyzer.next(), sendClientWarning); - } - - /** - * Validate maximum term size for given term - * @return true if given term is valid; otherwise false. - */ - public boolean validateMaxTermSize(DecoratedKey key, ByteBuffer term, boolean sendClientWarning) - { - if (term.remaining() > maxTermSize) - { - String message = logMessage(String.format(TERM_OVERSIZE_MESSAGE, - getColumnName(), - keyValidator().getString(key.getKey()), - FBUtilities.prettyPrintMemory(term.remaining()), - FBUtilities.prettyPrintMemory(maxTermSize))); - - if (sendClientWarning) - ClientWarn.instance.warn(message); - - noSpamLogger.warn(message); - return false; - } - - return true; - } -} diff --git a/src/java/org/apache/cassandra/index/sai/SSTableContext.java b/src/java/org/apache/cassandra/index/sai/SSTableContext.java index d9aa49e276..96c53a228a 100644 --- a/src/java/org/apache/cassandra/index/sai/SSTableContext.java +++ b/src/java/org/apache/cassandra/index/sai/SSTableContext.java @@ -104,9 +104,9 @@ public class SSTableContext extends SharedCloseableImpl /** * Returns a new {@link SSTableIndex} for a per-column index */ - public SSTableIndex newSSTableIndex(IndexContext indexContext) + public SSTableIndex newSSTableIndex(StorageAttachedIndex index) { - return indexDescriptor.newSSTableIndex(this, indexContext); + return indexDescriptor.newSSTableIndex(this, index); } /** diff --git a/src/java/org/apache/cassandra/index/sai/StorageAttachedIndex.java b/src/java/org/apache/cassandra/index/sai/StorageAttachedIndex.java index 6ada975010..4304a3edfe 100644 --- a/src/java/org/apache/cassandra/index/sai/StorageAttachedIndex.java +++ b/src/java/org/apache/cassandra/index/sai/StorageAttachedIndex.java @@ -24,27 +24,28 @@ import java.util.Collection; import java.util.Collections; import java.util.Comparator; import java.util.HashMap; -import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; -import java.util.NavigableMap; import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.SortedMap; import java.util.TreeMap; import java.util.concurrent.Callable; -import java.util.concurrent.CompletableFuture; // checkstyle: permit this import +import java.util.concurrent.CompletableFuture; //checkstyle: permit this import import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; import java.util.function.BooleanSupplier; import java.util.stream.Collectors; +import javax.annotation.Nullable; + import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableSet; -import com.google.common.util.concurrent.Futures; // checkstyle: permit this import -import com.google.common.util.concurrent.ListenableFuture; // checkstyle: permit this import +import com.google.common.util.concurrent.Futures; //checkstyle: permit this import +import com.google.common.util.concurrent.ListenableFuture; //checkstyle: permit this import import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -68,7 +69,6 @@ import org.apache.cassandra.db.filter.RowFilter; import org.apache.cassandra.db.lifecycle.LifecycleNewTracker; import org.apache.cassandra.db.marshal.AbstractType; import org.apache.cassandra.db.marshal.FloatType; -import org.apache.cassandra.db.marshal.VectorType; import org.apache.cassandra.db.memtable.Memtable; import org.apache.cassandra.db.partitions.PartitionUpdate; import org.apache.cassandra.db.rows.Row; @@ -80,7 +80,6 @@ import org.apache.cassandra.dht.RandomPartitioner; import org.apache.cassandra.exceptions.InvalidRequestException; import org.apache.cassandra.index.Index; import org.apache.cassandra.index.IndexRegistry; -import org.apache.cassandra.index.SecondaryIndexBuilder; import org.apache.cassandra.index.TargetParser; import org.apache.cassandra.index.sai.analyzer.AbstractAnalyzer; import org.apache.cassandra.index.sai.analyzer.NonTokenizingOptions; @@ -88,7 +87,13 @@ import org.apache.cassandra.index.sai.disk.SSTableIndex; import org.apache.cassandra.index.sai.disk.format.IndexDescriptor; import org.apache.cassandra.index.sai.disk.format.Version; import org.apache.cassandra.index.sai.disk.v1.IndexWriterConfig; -import org.apache.cassandra.index.sai.utils.TypeUtil; +import org.apache.cassandra.index.sai.memory.MemtableIndexManager; +import org.apache.cassandra.index.sai.metrics.ColumnQueryMetrics; +import org.apache.cassandra.index.sai.metrics.IndexMetrics; +import org.apache.cassandra.index.sai.utils.IndexIdentifier; +import org.apache.cassandra.index.sai.utils.IndexTermType; +import org.apache.cassandra.index.sai.utils.PrimaryKey; +import org.apache.cassandra.index.sai.view.IndexViewManager; import org.apache.cassandra.index.sai.view.View; import org.apache.cassandra.index.transactions.IndexTransaction; import org.apache.cassandra.io.sstable.Component; @@ -101,9 +106,14 @@ import org.apache.cassandra.schema.IndexMetadata; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.service.ClientWarn; import org.apache.cassandra.service.StorageService; +import org.apache.cassandra.utils.FBUtilities; +import org.apache.cassandra.utils.NoSpamLogger; import org.apache.cassandra.utils.Pair; import org.apache.cassandra.utils.concurrent.OpOrder; +import static org.apache.cassandra.config.CassandraRelevantProperties.SAI_MAX_FROZEN_TERM_SIZE; +import static org.apache.cassandra.config.CassandraRelevantProperties.SAI_MAX_STRING_TERM_SIZE; +import static org.apache.cassandra.config.CassandraRelevantProperties.SAI_MAX_VECTOR_TERM_SIZE; import static org.apache.cassandra.index.sai.disk.v1.IndexWriterConfig.MAX_TOP_K; public class StorageAttachedIndex implements Index @@ -131,44 +141,13 @@ public class StorageAttachedIndex implements Index private static final Logger logger = LoggerFactory.getLogger(StorageAttachedIndex.class); - private static class StorageAttachedIndexBuildingSupport implements IndexBuildingSupport - { - @Override - public SecondaryIndexBuilder getIndexBuildTask(ColumnFamilyStore cfs, - Set indexes, - Collection sstablesToRebuild, - boolean isFullRebuild) - { - NavigableMap> sstables = new TreeMap<>(Comparator.comparing(s -> s.descriptor.id, SSTableIdFactory.COMPARATOR)); - StorageAttachedIndexGroup group = StorageAttachedIndexGroup.getIndexGroup(cfs); + private static final NoSpamLogger noSpamLogger = NoSpamLogger.getLogger(logger, 1, TimeUnit.MINUTES); - assert group != null : "Index group does not exist for table " + cfs.keyspace + '.' + cfs.name; - - indexes.stream() - .filter((i) -> i instanceof StorageAttachedIndex) - .forEach((i) -> - { - StorageAttachedIndex sai = (StorageAttachedIndex) i; - IndexContext indexContext = ((StorageAttachedIndex) i).getIndexContext(); - - // If this is not a full manual index rebuild we can skip SSTables that already have an - // attached index. Otherwise, we override any pre-existent index. - Collection ss = sstablesToRebuild; - if (!isFullRebuild) - { - ss = sstablesToRebuild.stream() - .filter(s -> !IndexDescriptor.create(s).isPerColumnIndexBuildComplete(indexContext)) - .collect(Collectors.toList()); - } - - group.dropIndexSSTables(ss, sai); - - ss.forEach(sstable -> sstables.computeIfAbsent(sstable, ignore -> new HashSet<>()).add(sai)); - }); - - return new StorageAttachedIndexBuilder(group, sstables, isFullRebuild, false); - } - } + public static final long MAX_STRING_TERM_SIZE = SAI_MAX_STRING_TERM_SIZE.getSizeInBytes(); + public static final long MAX_FROZEN_TERM_SIZE = SAI_MAX_FROZEN_TERM_SIZE.getSizeInBytes(); + public static final long MAX_VECTOR_TERM_SIZE = SAI_MAX_VECTOR_TERM_SIZE.getSizeInBytes(); + public static final String TERM_OVERSIZE_MESSAGE = "Can't add term of column %s to index for key: %s, term size %s " + + "max allowed size %s, use analyzed = true (if not yet set) for that column."; // Used to build indexes on newly added SSTables: private static final StorageAttachedIndexBuildingSupport INDEX_BUILDER_SUPPORT = new StorageAttachedIndexBuildingSupport(); @@ -194,7 +173,17 @@ public class StorageAttachedIndex implements Index ImmutableSet.of(OrderPreservingPartitioner.class, LocalPartitioner.class, ByteOrderedPartitioner.class, RandomPartitioner.class); private final ColumnFamilyStore baseCfs; - private final IndexContext indexContext; + private final IndexMetadata indexMetadata; + private final IndexTermType indexTermType; + private final IndexIdentifier indexIdentifier; + private final IndexViewManager viewManager; + private final ColumnQueryMetrics columnQueryMetrics; + private final IndexWriterConfig indexWriterConfig; + @Nullable private final AbstractAnalyzer.AnalyzerFactory analyzerFactory; + private final PrimaryKey.Factory primaryKeyFactory; + private final MemtableIndexManager memtableIndexManager; + private final IndexMetrics indexMetrics; + private final long maxTermSize; // Tracks whether we've started the index build on initialization. private volatile boolean initBuildStarted = false; @@ -202,19 +191,24 @@ public class StorageAttachedIndex implements Index // Tracks whether the index has been invalidated due to removal, a table drop, etc. private volatile boolean valid = true; - public StorageAttachedIndex(ColumnFamilyStore baseCfs, IndexMetadata config) + public StorageAttachedIndex(ColumnFamilyStore baseCfs, IndexMetadata indexMetadata) { this.baseCfs = baseCfs; + this.indexMetadata = indexMetadata; TableMetadata tableMetadata = baseCfs.metadata(); - Pair target = TargetParser.parse(tableMetadata, config); - this.indexContext = new IndexContext(tableMetadata.keyspace, - tableMetadata.name, - tableMetadata.partitionKeyType, - tableMetadata.partitioner, - tableMetadata.comparator, - target.left, - target.right, - config); + Pair target = TargetParser.parse(tableMetadata, indexMetadata); + indexTermType = IndexTermType.create(target.left, tableMetadata.partitionKeyColumns(), target.right); + indexIdentifier = new IndexIdentifier(baseCfs.getKeyspaceName(), baseCfs.getTableName(), indexMetadata.name); + primaryKeyFactory = new PrimaryKey.Factory(tableMetadata.partitioner, tableMetadata.comparator); + indexWriterConfig = IndexWriterConfig.fromOptions(indexMetadata.name, indexTermType, indexMetadata.options); + viewManager = new IndexViewManager(this); + columnQueryMetrics = indexTermType.isLiteral() ? new ColumnQueryMetrics.TrieIndexMetrics(indexIdentifier) + : new ColumnQueryMetrics.BalancedTreeIndexMetrics(indexIdentifier); + analyzerFactory = AbstractAnalyzer.fromOptions(indexTermType, indexMetadata.options); + memtableIndexManager = new MemtableIndexManager(this); + indexMetrics = new IndexMetrics(this, memtableIndexManager); + maxTermSize = indexTermType.isVector() ? MAX_VECTOR_TERM_SIZE + : (indexTermType.isFrozen() ? MAX_FROZEN_TERM_SIZE : MAX_STRING_TERM_SIZE); } /** @@ -280,31 +274,30 @@ public class StorageAttachedIndex implements Index throw new InvalidRequestException(ANALYSIS_ON_KEY_COLUMNS_MESSAGE + new CqlBuilder().append(analysisOptions)); } - AbstractType type = TypeUtil.cellValueType(target.left, target.right); - AbstractAnalyzer.fromOptions(type, analysisOptions); - IndexWriterConfig config = IndexWriterConfig.fromOptions(null, type, options); + IndexTermType indexTermType = IndexTermType.create(target.left, metadata.partitionKeyColumns(), target.right); + AbstractAnalyzer.fromOptions(indexTermType, analysisOptions); + IndexWriterConfig config = IndexWriterConfig.fromOptions(null, indexTermType, options); // If we are indexing map entries we need to validate the subtypes - if (TypeUtil.isComposite(type)) + if (indexTermType.isComposite()) { - for (AbstractType subType : type.subTypes()) + for (IndexTermType subType : indexTermType.subTypes()) { - if (!SUPPORTED_TYPES.contains(subType.asCQL3Type()) && !TypeUtil.isFrozen(subType)) + if (!SUPPORTED_TYPES.contains(subType.asCQL3Type()) && !subType.isFrozen()) throw new InvalidRequestException("Unsupported type: " + subType.asCQL3Type()); } } - else if (!SUPPORTED_TYPES.contains(type.asCQL3Type()) && !TypeUtil.isFrozen(type)) + else if (!SUPPORTED_TYPES.contains(indexTermType.asCQL3Type()) && !indexTermType.isFrozen()) { - throw new InvalidRequestException("Unsupported type: " + type.asCQL3Type()); + throw new InvalidRequestException("Unsupported type: " + indexTermType.asCQL3Type()); } - else if (type.isVector()) + // If this is a vector type we need to validate it for the current vector index constraints + else if (indexTermType.isVector()) { - VectorType vectorType = (VectorType) type; - - if (!(vectorType.elementType instanceof FloatType)) + if (!(indexTermType.vectorElementType() instanceof FloatType)) throw new InvalidRequestException(VECTOR_NON_FLOAT_ERROR); - if (vectorType.dimension == 1 && config.getSimilarityFunction() == VectorSimilarityFunction.COSINE) + if (indexTermType.vectorDimension() == 1 && config.getSimilarityFunction() == VectorSimilarityFunction.COSINE) throw new InvalidRequestException(VECTOR_1_DIMENSION_COSINE_ERROR); if (DatabaseDescriptor.getRawConfig().data_file_directories.length > 1) @@ -332,7 +325,7 @@ public class StorageAttachedIndex implements Index @Override public IndexMetadata getIndexMetadata() { - return indexContext.getIndexMetadata(); + return indexMetadata; } @Override @@ -345,56 +338,214 @@ public class StorageAttachedIndex implements Index return () -> startInitialBuild(baseCfs, validation).get(); } - private Future startInitialBuild(ColumnFamilyStore baseCfs, IndexValidation validation) + @Override + public Callable getMetadataReloadTask(IndexMetadata indexMetadata) { - if (baseCfs.indexManager.isIndexQueryable(this)) + return null; + } + + @Override + public Callable getBlockingFlushTask() + { + return null; // storage-attached indexes are flushed alongside memtable + } + + @Override + public Callable getInvalidateTask() + { + return () -> { - logger.debug(indexContext.logMessage("Skipping validation and building in initialization task, as pre-join has already made the storage-attached index queryable...")); - initBuildStarted = true; - return CompletableFuture.completedFuture(null); + // mark index as invalid, in-progress SSTableIndexWriters will abort + valid = false; + + // in case of dropping table, SSTable indexes should already been removed by SSTableListChangedNotification. + Set toRemove = getComponents(); + for (SSTableIndex sstableIndex : view().getIndexes()) + sstableIndex.getSSTable().unregisterComponents(toRemove, baseCfs.getTracker()); + + viewManager.invalidate(); + if (analyzerFactory != null) + analyzerFactory.close(); + columnQueryMetrics.release(); + memtableIndexManager.invalidate(); + indexMetrics.release(); + return null; + }; + } + + @Override + public Callable getPreJoinTask(boolean hadBootstrap) + { + /* + * During bootstrap, streamed SSTable are already built for existing indexes via {@link StorageAttachedIndexBuildingSupport} + * from {@link org.apache.cassandra.streaming.StreamReceiveTask.OnCompletionRunnable}. + * + * For indexes created during bootstrapping, we don't have to block bootstrap for them. + */ + + return this::startPreJoinTask; + } + + @Override + public Callable getTruncateTask(long truncatedAt) + { + /* + * index files will be removed as part of base sstable lifecycle in + * {@link LogTransaction#delete(java.io.File)} asynchronously. + */ + return null; + } + + @Override + public boolean shouldBuildBlocking() + { + return true; + } + + @Override + public boolean isSSTableAttached() + { + return true; + } + + @Override + public Optional getBackingTable() + { + return Optional.empty(); + } + + @Override + public boolean dependsOn(ColumnMetadata column) + { + return indexTermType.dependsOn(column); + } + + @Override + public boolean supportsExpression(ColumnMetadata column, Operator operator) + { + return dependsOn(column) && indexTermType.supports(operator); + } + + @Override + public boolean filtersMultipleContains() + { + return false; + } + + @Override + public AbstractType customExpressionValueType() + { + return null; + } + + @Override + public RowFilter getPostIndexQueryFilter(RowFilter filter) + { + // it should be executed from the SAI query plan, this is only used by the singleton index query plan + throw new UnsupportedOperationException(); + } + + @Override + public Comparator getPostQueryOrdering(Restriction restriction, QueryOptions options) + { + // For now, only support ANN + assert restriction instanceof SingleColumnRestriction.AnnRestriction; + + Preconditions.checkState(indexTermType.isVector()); + + SingleColumnRestriction.AnnRestriction annRestriction = (SingleColumnRestriction.AnnRestriction) restriction; + VectorSimilarityFunction function = indexWriterConfig.getSimilarityFunction(); + + float[] target = indexTermType.decomposeVector(annRestriction.value(options).duplicate()); + + return (leftBuf, rightBuf) -> { + float[] left = indexTermType.decomposeVector(leftBuf.duplicate()); + double scoreLeft = function.compare(left, target); + + float[] right = indexTermType.decomposeVector(rightBuf.duplicate()); + double scoreRight = function.compare(right, target); + return Double.compare(scoreRight, scoreLeft); // descending order + }; + } + + @Override + public void validate(ReadCommand command) throws InvalidRequestException + { + if (!indexTermType.isVector()) + return; + + // to avoid overflow of the vector graph internal data structure and avoid OOM when filtering top-k + if (command.limits().count() > MAX_TOP_K) + throw new InvalidRequestException(String.format(ANN_LIMIT_ERROR, MAX_TOP_K, command.limits().count())); + } + + @Override + public long getEstimatedResultRows() + { + throw new UnsupportedOperationException("Use StorageAttachedIndexQueryPlan#getEstimatedResultRows() instead."); + } + + @Override + public boolean isQueryable(Status status) + { + // consider unknown status as queryable, because gossip may not be up-to-date for newly joining nodes. + return status == Status.BUILD_SUCCEEDED || status == Status.UNKNOWN; + } + + @Override + public void validate(PartitionUpdate update) throws InvalidRequestException + { + DecoratedKey key = update.partitionKey(); + for (Row row : update) + validateMaxTermSizeForRow(key, row, true); + } + + @Override + public Searcher searcherFor(ReadCommand command) throws InvalidRequestException + { + // searchers should be created from the query plan, this is only used by the singleton index query plan + throw new UnsupportedOperationException(); + } + + @Override + public SSTableFlushObserver getFlushObserver(Descriptor descriptor, LifecycleNewTracker tracker) + { + // flush observers should be created from the index group, this is only used by the singleton index group + throw new UnsupportedOperationException("Storage-attached index flush observers should never be created directly."); + } + + @Override + public Set getComponents() + { + return Version.LATEST.onDiskFormat() + .perColumnIndexComponents(indexTermType) + .stream() + .map(c -> Version.LATEST.makePerIndexComponent(c, indexIdentifier)) + .collect(Collectors.toSet()); + } + + @Override + public Indexer indexerFor(DecoratedKey key, + RegularAndStaticColumns columns, + long nowInSec, + WriteContext writeContext, + IndexTransaction.Type transactionType, + Memtable memtable) + { + if (transactionType == IndexTransaction.Type.UPDATE) + { + return new UpdateIndexer(key, memtable, writeContext); } - // stop in-progress compaction tasks to prevent compacted sstable not being indexed. - logger.debug(indexContext.logMessage("Stopping active compactions to make sure all sstables are indexed after initial build.")); - CompactionManager.instance.interruptCompactionFor(Collections.singleton(baseCfs.metadata()), - ssTableReader -> true, - true); + // we are only interested in the data from Memtable + // everything else is going to be handled by SSTableWriter observers + return null; + } - // Force another flush to make sure on disk index is generated for memtable data before marking it queryable. - // In the case of offline scrub, there are no live memtables. - if (!baseCfs.getTracker().getView().liveMemtables.isEmpty()) - { - baseCfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.INDEX_BUILD_STARTED); - } - - // It is now safe to flush indexes directly from flushing Memtables. - initBuildStarted = true; - - StorageAttachedIndexGroup indexGroup = StorageAttachedIndexGroup.getIndexGroup(baseCfs); - - assert indexGroup != null : "Index group does not exist for table " + baseCfs.keyspace + '.' + baseCfs.name; - - List nonIndexed = findNonIndexedSSTables(baseCfs, indexGroup, validation); - - if (nonIndexed.isEmpty()) - { - return CompletableFuture.completedFuture(null); - } - - // split sorted sstables into groups with similar size and build each group in separate compaction thread - List> groups = groupBySize(nonIndexed, DatabaseDescriptor.getConcurrentIndexBuilders()); - List> futures = new ArrayList<>(); - - for (List group : groups) - { - SortedMap> current = new TreeMap<>(Comparator.comparing(s -> s.descriptor.id, SSTableIdFactory.COMPARATOR)); - group.forEach(sstable -> current.put(sstable, Collections.singleton(this))); - - futures.add(CompactionManager.instance.submitIndexBuild(new StorageAttachedIndexBuilder(indexGroup, current, false, true))); - } - - logger.info(indexContext.logMessage("Submitting {} parallel initial index builds over {} total sstables..."), futures.size(), nonIndexed.size()); - return Futures.allAsList(futures); + @Override + public IndexBuildingSupport getBuildTaskSupport() + { + return INDEX_BUILDER_SUPPORT; } /** @@ -433,47 +584,78 @@ public class StorageAttachedIndex implements Index return groups; } - @Override - public Callable getMetadataReloadTask(IndexMetadata indexMetadata) + /** + * @return A set of SSTables which have attached to them invalid index components. + */ + public Collection onSSTableChanged(Collection oldSSTables, Collection newSSTables, IndexValidation validation) { - return null; + return viewManager.update(oldSSTables, newSSTables, validation); } - @Override - public Callable getBlockingFlushTask() + public void drop(Collection sstablesToRebuild) { - return null; // storage-attached indexes are flushed alongside memtable + viewManager.drop(sstablesToRebuild); } - @Override - public Callable getInvalidateTask() + public MemtableIndexManager memtableIndexManager() { - return () -> - { - // mark index as invalid, in-progress SSTableIndexWriters will abort - valid = false; - - // in case of dropping table, SSTable indexes should already been removed by SSTableListChangedNotification. - Set toRemove = getComponents(); - for (SSTableIndex sstableIndex : indexContext.getView().getIndexes()) - sstableIndex.getSSTable().unregisterComponents(toRemove, baseCfs.getTracker()); - - indexContext.invalidate(); - return null; - }; + return memtableIndexManager; } - @Override - public Callable getPreJoinTask(boolean hadBootstrap) + public View view() { - /* - * During bootstrap, streamed SSTable are already built for existing indexes via {@link StorageAttachedIndexBuildingSupport} - * from {@link org.apache.cassandra.streaming.StreamReceiveTask.OnCompletionRunnable}. - * - * For indexes created during bootstrapping, we don't have to block bootstrap for them. - */ + return viewManager.view(); + } - return this::startPreJoinTask; + public IndexTermType termType() + { + return indexTermType; + } + + public IndexIdentifier identifier() + { + return indexIdentifier; + } + + public PrimaryKey.Factory keyFactory() + { + return primaryKeyFactory; + } + + @VisibleForTesting + public ColumnFamilyStore baseCfs() + { + return baseCfs; + } + + public IndexWriterConfig indexWriterConfig() + { + return indexWriterConfig; + } + + public boolean hasAnalyzer() + { + return analyzerFactory != null; + } + + /** + * Returns an {@link AbstractAnalyzer} for use by write and query paths to transform + * literal values. + */ + public AbstractAnalyzer analyzer() + { + assert analyzerFactory != null : "Index does not support string analysis"; + return analyzerFactory.create(); + } + + public IndexMetrics indexMetrics() + { + return indexMetrics; + } + + public ColumnQueryMetrics columnQueryMetrics() + { + return columnQueryMetrics; } public boolean isInitBuildStarted() @@ -486,13 +668,212 @@ public class StorageAttachedIndex implements Index return () -> valid; } + public boolean hasClustering() + { + return baseCfs.getComparator().size() > 0; + } + + /** + * @return the number of indexed rows in this index (aka. a pair of term and rowId) + */ + public long cellCount() + { + return view().getIndexes() + .stream() + .mapToLong(SSTableIndex::getRowCount) + .sum(); + } + + /** + * @return total number of per-index open files + */ + public int openPerColumnIndexFiles() + { + return viewManager.view().size() * Version.LATEST.onDiskFormat().openFilesPerColumnIndex(); + } + + /** + * @return the total size (in bytes) of per-column index components + */ + public long diskUsage() + { + return view().getIndexes() + .stream() + .mapToLong(SSTableIndex::sizeOfPerColumnComponents) + .sum(); + } + + /** + * @return the total memory usage (in bytes) of per-column index on-disk data structure + */ + public long indexFileCacheSize() + { + return view().getIndexes() + .stream() + .mapToLong(SSTableIndex::indexFileCacheSize) + .sum(); + } + + /** + * Removes this index from the {@code SecondaryIndexManager}'s set of queryable indexes. + */ + public void makeIndexNonQueryable() + { + baseCfs.indexManager.makeIndexNonQueryable(this, Status.BUILD_FAILED); + logger.warn(indexIdentifier.logMessage("Storage-attached index is no longer queryable. Please restart this node to repair it.")); + } + + /** + * Validate maximum term size for given row + */ + public void validateMaxTermSizeForRow(DecoratedKey key, Row row, boolean sendClientWarning) + { + AbstractAnalyzer analyzer = hasAnalyzer() ? analyzer() : null; + if (indexTermType.isNonFrozenCollection()) + { + Iterator bufferIterator = indexTermType.valuesOf(row, FBUtilities.nowInSeconds()); + while (bufferIterator != null && bufferIterator.hasNext()) + validateMaxTermSizeForCell(analyzer, key, bufferIterator.next(), sendClientWarning); + } + else + { + ByteBuffer value = indexTermType.valueOf(key, row, FBUtilities.nowInSeconds()); + validateMaxTermSizeForCell(analyzer, key, value, sendClientWarning); + } + } + + private void validateMaxTermSizeForCell(AbstractAnalyzer analyzer, DecoratedKey key, @Nullable ByteBuffer cellBuffer, boolean sendClientWarning) + { + if (cellBuffer == null || cellBuffer.remaining() == 0) + return; + + // analyzer should not return terms that are larger than the origin value. + if (cellBuffer.remaining() <= maxTermSize) + return; + + if (analyzer != null) + { + analyzer.reset(cellBuffer.duplicate()); + while (analyzer.hasNext()) + validateMaxTermSize(key, analyzer.next(), sendClientWarning); + } + else + { + validateMaxTermSize(key, cellBuffer.duplicate(), sendClientWarning); + } + } + + /** + * Validate maximum term size for given term + * @return true if given term is valid; otherwise false. + */ + public boolean validateMaxTermSize(DecoratedKey key, ByteBuffer term, boolean sendClientWarning) + { + if (term.remaining() > maxTermSize) + { + String message = indexIdentifier.logMessage(String.format(TERM_OVERSIZE_MESSAGE, + indexTermType.columnName(), + key, + FBUtilities.prettyPrintMemory(term.remaining()), + FBUtilities.prettyPrintMemory(maxTermSize))); + + if (sendClientWarning) + ClientWarn.instance.warn(message); + + noSpamLogger.warn(message); + return false; + } + + return true; + } + + @Override + public String toString() + { + return indexIdentifier.toString(); + } + + @Override + public boolean equals(Object obj) + { + if (obj == this) + return true; + + if (!(obj instanceof StorageAttachedIndex)) + return false; + + StorageAttachedIndex other = (StorageAttachedIndex) obj; + + return Objects.equals(indexTermType, other.indexTermType) && + Objects.equals(indexMetadata, other.indexMetadata) && + Objects.equals(baseCfs.getComparator(), other.baseCfs.getComparator()); + } + + @Override + public int hashCode() + { + return Objects.hash(indexTermType, indexMetadata, baseCfs.getComparator()); + } + + private Future startInitialBuild(ColumnFamilyStore baseCfs, IndexValidation validation) + { + if (baseCfs.indexManager.isIndexQueryable(this)) + { + logger.debug(indexIdentifier.logMessage("Skipping validation and building in initialization task, as pre-join has already made the storage-attached index queryable...")); + initBuildStarted = true; + return CompletableFuture.completedFuture(null); + } + + // stop in-progress compaction tasks to prevent compacted sstable not being indexed. + logger.debug(indexIdentifier.logMessage("Stopping active compactions to make sure all sstables are indexed after initial build.")); + CompactionManager.instance.interruptCompactionFor(Collections.singleton(baseCfs.metadata()), + ssTableReader -> true, + true); + + // Force another flush to make sure on disk index is generated for memtable data before marking it queryable. + // In the case of offline scrub, there are no live memtables. + if (!baseCfs.getTracker().getView().liveMemtables.isEmpty()) + { + baseCfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.INDEX_BUILD_STARTED); + } + + // It is now safe to flush indexes directly from flushing Memtables. + initBuildStarted = true; + + StorageAttachedIndexGroup indexGroup = StorageAttachedIndexGroup.getIndexGroup(baseCfs); + + assert indexGroup != null : "Index group does not exist for table " + baseCfs.keyspace + '.' + baseCfs.name; + + List nonIndexed = findNonIndexedSSTables(baseCfs, indexGroup, validation); + + if (nonIndexed.isEmpty()) + { + return CompletableFuture.completedFuture(null); + } + + // split sorted sstables into groups with similar size and build each group in separate compaction thread + List> groups = groupBySize(nonIndexed, DatabaseDescriptor.getConcurrentIndexBuilders()); + List> futures = new ArrayList<>(); + + for (List group : groups) + { + SortedMap> current = new TreeMap<>(Comparator.comparing(s -> s.descriptor.id, SSTableIdFactory.COMPARATOR)); + group.forEach(sstable -> current.put(sstable, Collections.singleton(this))); + + futures.add(CompactionManager.instance.submitIndexBuild(new StorageAttachedIndexBuilder(indexGroup, current, false, true))); + } + + logger.info(indexIdentifier.logMessage("Submitting {} parallel initial index builds over {} total sstables..."), futures.size(), nonIndexed.size()); + return Futures.allAsList(futures); + } + private Future startPreJoinTask() { try { if (baseCfs.indexManager.isIndexQueryable(this)) { - logger.debug(indexContext.logMessage("Skipping validation in pre-join task, as the initialization task has already made the index queryable...")); + logger.debug(indexIdentifier.logMessage("Skipping validation in pre-join task, as the initialization task has already made the index queryable...")); baseCfs.indexManager.makeIndexQueryable(this, Status.BUILD_SUCCEEDED); return null; } @@ -511,126 +892,12 @@ public class StorageAttachedIndex implements Index } catch (Throwable t) { - logger.error(indexContext.logMessage("Failed in pre-join task!"), t); + logger.error(indexIdentifier.logMessage("Failed in pre-join task!"), t); } return null; } - @Override - public Callable getTruncateTask(long truncatedAt) - { - /* - * index files will be removed as part of base sstable lifecycle in - * {@link LogTransaction#delete(java.io.File)} asynchronously. - */ - return null; - } - - @Override - public boolean shouldBuildBlocking() - { - return true; - } - - @Override - public boolean isSSTableAttached() - { - return true; - } - - @Override - public Optional getBackingTable() - { - return Optional.empty(); - } - - @Override - public boolean dependsOn(ColumnMetadata column) - { - return indexContext.getDefinition().compareTo(column) == 0; - } - - @Override - public boolean supportsExpression(ColumnMetadata column, Operator operator) - { - return dependsOn(column) && indexContext.supports(operator); - } - - @Override - public boolean filtersMultipleContains() - { - return false; - } - - @Override - public AbstractType customExpressionValueType() - { - return null; - } - - @Override - public RowFilter getPostIndexQueryFilter(RowFilter filter) - { - // it should be executed from the SAI query plan, this is only used by the singleton index query plan - throw new UnsupportedOperationException(); - } - - @Override - public Comparator getPostQueryOrdering(Restriction restriction, QueryOptions options) - { - // For now, only support ANN - assert restriction instanceof SingleColumnRestriction.AnnRestriction; - - Preconditions.checkState(indexContext.isVector()); - - SingleColumnRestriction.AnnRestriction annRestriction = (SingleColumnRestriction.AnnRestriction) restriction; - VectorSimilarityFunction function = indexContext.getIndexWriterConfig().getSimilarityFunction(); - - float[] target = TypeUtil.decomposeVector(indexContext, annRestriction.value(options).duplicate()); - - return (leftBuf, rightBuf) -> { - float[] left = TypeUtil.decomposeVector(indexContext, leftBuf.duplicate()); - double scoreLeft = function.compare(left, target); - - float[] right = TypeUtil.decomposeVector(indexContext, rightBuf.duplicate()); - double scoreRight = function.compare(right, target); - return Double.compare(scoreRight, scoreLeft); // descending order - }; - } - - @Override - public void validate(ReadCommand command) throws InvalidRequestException - { - if (!getIndexContext().isVector()) - return; - - // to avoid overflow of the vector graph internal data structure and avoid OOM when filtering top-k - if (command.limits().count() > MAX_TOP_K) - throw new InvalidRequestException(String.format(ANN_LIMIT_ERROR, MAX_TOP_K, command.limits().count())); - } - - @Override - public long getEstimatedResultRows() - { - throw new UnsupportedOperationException("Use StorageAttachedIndexQueryPlan#getEstimatedResultRows() instead."); - } - - @Override - public boolean isQueryable(Status status) - { - // consider unknown status as queryable, because gossip may not be up-to-date for newly joining nodes. - return status == Status.BUILD_SUCCEEDED || status == Status.UNKNOWN; - } - - @Override - public void validate(PartitionUpdate update) throws InvalidRequestException - { - DecoratedKey key = update.partitionKey(); - for (Row row : update) - indexContext.validateMaxTermSizeForRow(key, row, true); - } - /** * This method is called by the startup tasks to find SSTables that don't have indexes. The method is * synchronized so that the view is unchanged between validation and the selection of non-indexed SSTables. @@ -647,7 +914,7 @@ public class StorageAttachedIndex implements Index // ...then identify and rebuild the SSTable indexes that are missing. List nonIndexed = new ArrayList<>(); - View view = indexContext.getView(); + View view = viewManager.view(); for (SSTableReader sstable : sstables) { @@ -656,7 +923,7 @@ public class StorageAttachedIndex implements Index // 2. The SSTable is not marked compacted // 3. The column index does not have a completion marker if (!view.containsSSTable(sstable) && !sstable.isMarkedCompacted() && - !IndexDescriptor.create(sstable).isPerColumnIndexBuildComplete(indexContext)) + !IndexDescriptor.create(sstable).isPerColumnIndexBuildComplete(indexIdentifier)) { nonIndexed.add(sstable); } @@ -665,74 +932,6 @@ public class StorageAttachedIndex implements Index return nonIndexed; } - @Override - public Searcher searcherFor(ReadCommand command) throws InvalidRequestException - { - // searchers should be created from the query plan, this is only used by the singleton index query plan - throw new UnsupportedOperationException(); - } - - @Override - public SSTableFlushObserver getFlushObserver(Descriptor descriptor, LifecycleNewTracker tracker) - { - // flush observers should be created from the index group, this is only used by the singleton index group - throw new UnsupportedOperationException("Storage-attached index flush observers should never be created directly."); - } - - @Override - public Set getComponents() - { - return Version.LATEST.onDiskFormat() - .perColumnIndexComponents(indexContext) - .stream() - .map(c -> Version.LATEST.makePerIndexComponent(c, indexContext)) - .collect(Collectors.toSet()); - } - - @Override - public Indexer indexerFor(DecoratedKey key, - RegularAndStaticColumns columns, - long nowInSec, - WriteContext writeContext, - IndexTransaction.Type transactionType, - Memtable memtable) - { - if (transactionType == IndexTransaction.Type.UPDATE) - { - return new UpdateIndexer(key, memtable, writeContext); - } - - // we are only interested in the data from Memtable - // everything else is going to be handled by SSTableWriter observers - return null; - } - - @Override - public IndexBuildingSupport getBuildTaskSupport() - { - return INDEX_BUILDER_SUPPORT; - } - - public IndexContext getIndexContext() - { - return indexContext; - } - - @Override - public String toString() - { - return String.format("%s.%s.%s", baseCfs.keyspace.getName(), baseCfs.name, getIndexMetadata() == null ? "?" : getIndexMetadata()); - } - - /** - * Removes this index from the {@code SecondaryIndexManager}'s set of queryable indexes. - */ - public void makeIndexNonQueryable() - { - baseCfs.indexManager.makeIndexNonQueryable(this, Status.BUILD_FAILED); - logger.warn(indexContext.logMessage("Storage-attached index is no longer queryable. Please restart this node to repair it.")); - } - private class UpdateIndexer implements Index.Indexer { private final DecoratedKey key; @@ -749,14 +948,14 @@ public class StorageAttachedIndex implements Index @Override public void insertRow(Row row) { - adjustMemtableSize(indexContext.getMemtableIndexManager().index(key, row, memtable), + adjustMemtableSize(memtableIndexManager.index(key, row, memtable), CassandraWriteContext.fromContext(writeContext).getGroup()); } @Override public void updateRow(Row oldRow, Row newRow) { - adjustMemtableSize(indexContext.getMemtableIndexManager().update(key, oldRow, newRow, memtable), + adjustMemtableSize(memtableIndexManager.update(key, oldRow, newRow, memtable), CassandraWriteContext.fromContext(writeContext).getGroup()); } diff --git a/src/java/org/apache/cassandra/index/sai/StorageAttachedIndexBuilder.java b/src/java/org/apache/cassandra/index/sai/StorageAttachedIndexBuilder.java index c45b604ffc..5bf27b27b0 100644 --- a/src/java/org/apache/cassandra/index/sai/StorageAttachedIndexBuilder.java +++ b/src/java/org/apache/cassandra/index/sai/StorageAttachedIndexBuilder.java @@ -152,7 +152,7 @@ public class StorageAttachedIndexBuilder extends SecondaryIndexBuilder boolean perIndexComponentsOnly = perSSTableFileLock == null; // remove existing per column index files instead of overwriting IndexDescriptor indexDescriptor = IndexDescriptor.create(sstable); - indexes.forEach(index -> indexDescriptor.deleteColumnIndex(index.getIndexContext())); + indexes.forEach(index -> indexDescriptor.deleteColumnIndex(index.termType(), index.identifier())); indexWriter = StorageAttachedIndexWriter.createBuilderWriter(indexDescriptor, indexes, txn, perIndexComponentsOnly); @@ -342,7 +342,7 @@ public class StorageAttachedIndexBuilder extends SecondaryIndexBuilder if (!dropped.isEmpty()) { - String droppedIndexes = dropped.stream().map(sai -> sai.getIndexContext().getIndexName()).collect(Collectors.toList()).toString(); + String droppedIndexes = dropped.stream().map(sai -> sai.identifier().indexName).collect(Collectors.toList()).toString(); if (isFullRebuild) throw new RuntimeException(logMessage(String.format("%s are dropped, will stop index build.", droppedIndexes))); else diff --git a/src/java/org/apache/cassandra/index/sai/StorageAttachedIndexBuildingSupport.java b/src/java/org/apache/cassandra/index/sai/StorageAttachedIndexBuildingSupport.java new file mode 100644 index 0000000000..7a13b4b186 --- /dev/null +++ b/src/java/org/apache/cassandra/index/sai/StorageAttachedIndexBuildingSupport.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.index.sai; + +import java.util.Collection; +import java.util.Comparator; +import java.util.HashSet; +import java.util.NavigableMap; +import java.util.Set; +import java.util.TreeMap; +import java.util.stream.Collectors; + +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.index.Index; +import org.apache.cassandra.index.SecondaryIndexBuilder; +import org.apache.cassandra.index.sai.disk.format.IndexDescriptor; +import org.apache.cassandra.io.sstable.SSTableIdFactory; +import org.apache.cassandra.io.sstable.format.SSTableReader; + +class StorageAttachedIndexBuildingSupport implements Index.IndexBuildingSupport +{ + @Override + public SecondaryIndexBuilder getIndexBuildTask(ColumnFamilyStore cfs, + Set indexes, + Collection sstablesToRebuild, + boolean isFullRebuild) + { + NavigableMap> sstables = new TreeMap<>(Comparator.comparing(s -> s.descriptor.id, SSTableIdFactory.COMPARATOR)); + StorageAttachedIndexGroup group = StorageAttachedIndexGroup.getIndexGroup(cfs); + + assert group != null : "Index group does not exist for table " + cfs.keyspace + '.' + cfs.name; + + indexes.stream() + .filter((i) -> i instanceof StorageAttachedIndex) + .forEach((i) -> + { + StorageAttachedIndex sai = (StorageAttachedIndex) i; + + // If this is not a full manual index rebuild we can skip SSTables that already have an + // attached index. Otherwise, we override any pre-existent index. + Collection ss = sstablesToRebuild; + if (!isFullRebuild) + { + ss = sstablesToRebuild.stream() + .filter(s -> !IndexDescriptor.create(s).isPerColumnIndexBuildComplete(sai.identifier())) + .collect(Collectors.toList()); + } + + group.dropIndexSSTables(ss, sai); + + ss.forEach(sstable -> sstables.computeIfAbsent(sstable, ignore -> new HashSet<>()).add(sai)); + }); + + return new StorageAttachedIndexBuilder(group, sstables, isFullRebuild, false); + } +} diff --git a/src/java/org/apache/cassandra/index/sai/StorageAttachedIndexGroup.java b/src/java/org/apache/cassandra/index/sai/StorageAttachedIndexGroup.java index 5dac6edd47..fd04e44dac 100644 --- a/src/java/org/apache/cassandra/index/sai/StorageAttachedIndexGroup.java +++ b/src/java/org/apache/cassandra/index/sai/StorageAttachedIndexGroup.java @@ -244,7 +244,7 @@ public class StorageAttachedIndexGroup implements Index.Group, INotificationCons { IndexDescriptor indexDescriptor = IndexDescriptor.create(sstable); Set components = indexDescriptor.getLivePerSSTableComponents(); - indices.forEach(index -> components.addAll(indexDescriptor.getLivePerIndexComponents(index.getIndexContext()))); + indices.forEach(index -> components.addAll(indexDescriptor.getLivePerIndexComponents(index.termType(), index.identifier()))); return components; } @@ -269,11 +269,11 @@ public class StorageAttachedIndexGroup implements Index.Group, INotificationCons } else if (notification instanceof MemtableRenewedNotification) { - indexes.forEach(index -> index.getIndexContext().getMemtableIndexManager().renewMemtable(((MemtableRenewedNotification) notification).renewed)); + indexes.forEach(index -> index.memtableIndexManager().renewMemtable(((MemtableRenewedNotification) notification).renewed)); } else if (notification instanceof MemtableDiscardedNotification) { - indexes.forEach(index -> index.getIndexContext().getMemtableIndexManager().discardMemtable(((MemtableDiscardedNotification) notification).memtable)); + indexes.forEach(index -> index.memtableIndexManager().discardMemtable(((MemtableDiscardedNotification) notification).memtable)); } } @@ -287,7 +287,7 @@ public class StorageAttachedIndexGroup implements Index.Group, INotificationCons { try { - index.getIndexContext().drop(ss); + index.drop(ss); } catch (Throwable t) { @@ -317,7 +317,7 @@ public class StorageAttachedIndexGroup implements Index.Group, INotificationCons // Column indexes are invalid if their SSTable-level components are corrupted so delete // their associated index files and mark them non-queryable. indexes.forEach(index -> { - indexDescriptor.deleteColumnIndex(index.getIndexContext()); + indexDescriptor.deleteColumnIndex(index.termType(), index.identifier()); index.makeIndexNonQueryable(); }); }); @@ -328,13 +328,13 @@ public class StorageAttachedIndexGroup implements Index.Group, INotificationCons for (StorageAttachedIndex index : indexes) { - Collection invalid = index.getIndexContext().onSSTableChanged(removed, results.left, validation); + Collection invalid = index.onSSTableChanged(removed, results.left, validation); if (!invalid.isEmpty()) { // Delete the index files and mark the index non-queryable, as its view may be compromised, // and incomplete, for our callers: - invalid.forEach(context -> context.indexDescriptor.deleteColumnIndex(index.getIndexContext())); + invalid.forEach(context -> context.indexDescriptor.deleteColumnIndex(index.termType(), index.identifier())); index.makeIndexNonQueryable(); incomplete.add(index); } @@ -357,8 +357,8 @@ public class StorageAttachedIndexGroup implements Index.Group, INotificationCons for (StorageAttachedIndex index : indexes) { - if (indexDescriptor.isPerColumnIndexBuildComplete(index.getIndexContext())) - indexDescriptor.checksumPerIndexComponents(index.getIndexContext()); + if (indexDescriptor.isPerColumnIndexBuildComplete(index.identifier())) + indexDescriptor.checksumPerIndexComponents(index.termType(), index.identifier()); else if (throwOnIncomplete) throw new IllegalStateException(indexDescriptor.logMessage("Incomplete per-column index build")); else @@ -386,7 +386,7 @@ public class StorageAttachedIndexGroup implements Index.Group, INotificationCons */ public int openIndexFiles() { - return contextManager.openFiles() + indexes.stream().mapToInt(index -> index.getIndexContext().openPerIndexFiles()).sum(); + return contextManager.openFiles() + indexes.stream().mapToInt(StorageAttachedIndex::openPerColumnIndexFiles).sum(); } /** @@ -426,7 +426,7 @@ public class StorageAttachedIndexGroup implements Index.Group, INotificationCons */ public long totalDiskUsage() { - return diskUsage() + indexes.stream().flatMap(index -> index.getIndexContext().getView().getIndexes().stream()) + return diskUsage() + indexes.stream().flatMap(index -> index.view().getIndexes().stream()) .mapToLong(SSTableIndex::sizeOfPerColumnComponents).sum(); } diff --git a/src/java/org/apache/cassandra/index/sai/analyzer/AbstractAnalyzer.java b/src/java/org/apache/cassandra/index/sai/analyzer/AbstractAnalyzer.java index da0b2a4d19..f9878c2f3f 100644 --- a/src/java/org/apache/cassandra/index/sai/analyzer/AbstractAnalyzer.java +++ b/src/java/org/apache/cassandra/index/sai/analyzer/AbstractAnalyzer.java @@ -24,9 +24,8 @@ import java.util.Map; import java.util.NoSuchElementException; import java.util.stream.Collectors; -import org.apache.cassandra.db.marshal.AbstractType; -import org.apache.cassandra.db.marshal.StringType; import org.apache.cassandra.exceptions.InvalidRequestException; +import org.apache.cassandra.index.sai.utils.IndexTermType; public abstract class AbstractAnalyzer implements Iterator { @@ -83,23 +82,23 @@ public abstract class AbstractAnalyzer implements Iterator } } - public static AnalyzerFactory fromOptions(AbstractType type, Map options) + public static AnalyzerFactory fromOptions(IndexTermType indexTermType, Map options) { if (hasNonTokenizingOptions(options)) { - if (type instanceof StringType) + if (indexTermType.isString()) { // validate options NonTokenizingOptions.fromMap(options); - return () -> new NonTokenizingAnalyzer(type, options); + return () -> new NonTokenizingAnalyzer(indexTermType, options); } else { - throw new InvalidRequestException("CQL type " + type.asCQL3Type() + " cannot be analyzed."); + throw new InvalidRequestException("CQL type " + indexTermType.asCQL3Type() + " cannot be analyzed."); } } - return NoOpAnalyzer::new; + return null; } private static boolean hasNonTokenizingOptions(Map options) diff --git a/src/java/org/apache/cassandra/index/sai/analyzer/NoOpAnalyzer.java b/src/java/org/apache/cassandra/index/sai/analyzer/NoOpAnalyzer.java deleted file mode 100644 index fbf76bcc25..0000000000 --- a/src/java/org/apache/cassandra/index/sai/analyzer/NoOpAnalyzer.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.cassandra.index.sai.analyzer; - -import java.nio.ByteBuffer; - -import com.google.common.base.MoreObjects; - -/** - * Default NoOp tokenizer. The iterator will iterate only once returning the unmodified input - */ -public class NoOpAnalyzer extends AbstractAnalyzer -{ - private ByteBuffer input; - private boolean hasNext = false; - - public NoOpAnalyzer() {} - - @Override - public boolean hasNext() - { - if (hasNext) - { - this.next = input; - this.hasNext = false; - return true; - } - this.next = null; - return false; - } - - @Override - protected void resetInternal(ByteBuffer input) - { - this.input = input; - this.hasNext = true; - } - - @Override - public boolean transformValue() - { - return false; - } - - @Override - public String toString() - { - return MoreObjects.toStringHelper(this).toString(); - } -} diff --git a/src/java/org/apache/cassandra/index/sai/analyzer/NonTokenizingAnalyzer.java b/src/java/org/apache/cassandra/index/sai/analyzer/NonTokenizingAnalyzer.java index f58158a823..30eedc08f3 100644 --- a/src/java/org/apache/cassandra/index/sai/analyzer/NonTokenizingAnalyzer.java +++ b/src/java/org/apache/cassandra/index/sai/analyzer/NonTokenizingAnalyzer.java @@ -25,11 +25,10 @@ import com.google.common.base.MoreObjects; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.cassandra.db.marshal.AbstractType; -import org.apache.cassandra.db.marshal.StringType; import org.apache.cassandra.index.sai.analyzer.filter.BasicFilters; import org.apache.cassandra.index.sai.analyzer.filter.FilterPipeline; import org.apache.cassandra.index.sai.analyzer.filter.FilterPipelineExecutor; +import org.apache.cassandra.index.sai.utils.IndexTermType; import org.apache.cassandra.serializers.MarshalException; import org.apache.cassandra.utils.ByteBufferUtil; @@ -41,21 +40,21 @@ public class NonTokenizingAnalyzer extends AbstractAnalyzer { private static final Logger logger = LoggerFactory.getLogger(NonTokenizingAnalyzer.class); - private final AbstractType type; + private final IndexTermType indexTermType; private final NonTokenizingOptions options; private final FilterPipeline filterPipeline; private ByteBuffer input; private boolean hasNext = false; - NonTokenizingAnalyzer(AbstractType type, Map options) + NonTokenizingAnalyzer(IndexTermType indexTermType, Map options) { - this(type, NonTokenizingOptions.fromMap(options)); + this(indexTermType, NonTokenizingOptions.fromMap(options)); } - NonTokenizingAnalyzer(AbstractType type, NonTokenizingOptions tokenizerOptions) + NonTokenizingAnalyzer(IndexTermType indexTermType, NonTokenizingOptions tokenizerOptions) { - this.type = type; + this.indexTermType = indexTermType; this.options = tokenizerOptions; this.filterPipeline = getFilterPipeline(); } @@ -64,18 +63,19 @@ public class NonTokenizingAnalyzer extends AbstractAnalyzer public boolean hasNext() { // check that we know how to handle the input, otherwise bail - if (!(type instanceof StringType)) return false; + if (!indexTermType.isString()) + return false; if (hasNext) { try { - String input = type.getString(this.input); + String input = indexTermType.asString(this.input); if (input == null) { throw new MarshalException(String.format("'null' deserialized value for %s with %s", - ByteBufferUtil.bytesToHex(this.input), type)); + ByteBufferUtil.bytesToHex(this.input), indexTermType)); } String result = FilterPipelineExecutor.execute(filterPipeline, input); @@ -88,13 +88,13 @@ public class NonTokenizingAnalyzer extends AbstractAnalyzer } nextLiteral = result; - next = type.fromString(result); + next = indexTermType.fromString(result); return true; } catch (MarshalException e) { - logger.error("Failed to deserialize value with " + type, e); + logger.error("Failed to deserialize value with " + indexTermType, e); return false; } finally diff --git a/src/java/org/apache/cassandra/index/sai/disk/IndexSearchResultIterator.java b/src/java/org/apache/cassandra/index/sai/disk/IndexSearchResultIterator.java index 3e8bc4aa0d..1433598cc8 100644 --- a/src/java/org/apache/cassandra/index/sai/disk/IndexSearchResultIterator.java +++ b/src/java/org/apache/cassandra/index/sai/disk/IndexSearchResultIterator.java @@ -65,7 +65,7 @@ public class IndexSearchResultIterator extends KeyRangeIterator { List subIterators = new ArrayList<>(1 + sstableIndexes.size()); - KeyRangeIterator memtableIterator = expression.context.getMemtableIndexManager().searchMemtableIndexes(queryContext, expression, keyRange); + KeyRangeIterator memtableIterator = expression.getIndex().memtableIndexManager().searchMemtableIndexes(queryContext, expression, keyRange); if (memtableIterator != null) subIterators.add(memtableIterator); @@ -77,7 +77,7 @@ public class IndexSearchResultIterator extends KeyRangeIterator queryContext.sstablesHit++; if (sstableIndex.isReleased()) - throw new IllegalStateException(sstableIndex.getIndexContext().logMessage("Index was released from the view during the query")); + throw new IllegalStateException(sstableIndex.getIndexIdentifier().logMessage("Index was released from the view during the query")); List indexIterators = sstableIndex.search(expression, keyRange, queryContext); @@ -87,7 +87,7 @@ public class IndexSearchResultIterator extends KeyRangeIterator catch (Throwable e) { if (!(e instanceof QueryCancelledException)) - logger.debug(sstableIndex.getIndexContext().logMessage(String.format("Failed search an index %s, aborting query.", sstableIndex.getSSTable())), e); + logger.debug(sstableIndex.getIndexIdentifier().logMessage(String.format("Failed search an index %s, aborting query.", sstableIndex.getSSTable())), e); throw Throwables.cleaned(e); } @@ -152,7 +152,7 @@ public class IndexSearchResultIterator extends KeyRangeIterator } catch (Throwable e) { - logger.error(index.getIndexContext().logMessage(String.format("Failed to release index on SSTable %s", index.getSSTable())), e); + logger.error(index.getIndexIdentifier().logMessage(String.format("Failed to release index on SSTable %s", index.getSSTable())), e); } } } diff --git a/src/java/org/apache/cassandra/index/sai/disk/SSTableIndex.java b/src/java/org/apache/cassandra/index/sai/disk/SSTableIndex.java index 40a2cbacd2..aca92e6dd2 100644 --- a/src/java/org/apache/cassandra/index/sai/disk/SSTableIndex.java +++ b/src/java/org/apache/cassandra/index/sai/disk/SSTableIndex.java @@ -33,13 +33,15 @@ import org.slf4j.LoggerFactory; import org.apache.cassandra.db.PartitionPosition; import org.apache.cassandra.db.virtual.SimpleDataSet; import org.apache.cassandra.dht.AbstractBounds; -import org.apache.cassandra.index.sai.IndexContext; import org.apache.cassandra.index.sai.QueryContext; import org.apache.cassandra.index.sai.SSTableContext; +import org.apache.cassandra.index.sai.StorageAttachedIndex; +import org.apache.cassandra.index.sai.utils.IndexIdentifier; import org.apache.cassandra.index.sai.disk.format.Version; import org.apache.cassandra.index.sai.disk.v1.segment.SegmentOrdering; import org.apache.cassandra.index.sai.iterators.KeyRangeIterator; import org.apache.cassandra.index.sai.plan.Expression; +import org.apache.cassandra.index.sai.utils.IndexTermType; import org.apache.cassandra.io.sstable.SSTableIdFactory; import org.apache.cassandra.io.sstable.format.SSTableReader; @@ -61,17 +63,17 @@ public abstract class SSTableIndex implements SegmentOrdering .thenComparing(s -> s.getSSTable().descriptor.id, SSTableIdFactory.COMPARATOR); protected final SSTableContext sstableContext; - protected final IndexContext indexContext; + protected final IndexTermType indexTermType; + protected final IndexIdentifier indexIdentifier; private final AtomicInteger references = new AtomicInteger(1); private final AtomicBoolean obsolete = new AtomicBoolean(false); - public SSTableIndex(SSTableContext sstableContext, IndexContext indexContext) + public SSTableIndex(SSTableContext sstableContext, StorageAttachedIndex index) { - assert indexContext.getValidator() != null; - this.sstableContext = sstableContext.sharedCopy(); // this line must not be before any code that may throw - this.indexContext = indexContext; + this.indexTermType = index.termType(); + this.indexIdentifier = index.identifier(); } /** @@ -152,12 +154,17 @@ public abstract class SSTableIndex implements SegmentOrdering */ public long sizeOfPerColumnComponents() { - return sstableContext.indexDescriptor.sizeOnDiskOfPerIndexComponents(indexContext); + return sstableContext.indexDescriptor.sizeOnDiskOfPerIndexComponents(indexTermType, indexIdentifier); } - public IndexContext getIndexContext() + public IndexTermType getIndexTermType() { - return indexContext; + return indexTermType; + } + + public IndexIdentifier getIndexIdentifier() + { + return indexIdentifier; } public SSTableContext getSSTableContext() @@ -202,7 +209,7 @@ public abstract class SSTableIndex implements SegmentOrdering } catch (Throwable e) { - logger.error(getIndexContext().logMessage("Failed to release index on SSTable {}"), getSSTable().descriptor, e); + logger.error(indexIdentifier.logMessage("Failed to release index on SSTable {}"), getSSTable().descriptor, e); } } @@ -221,7 +228,7 @@ public abstract class SSTableIndex implements SegmentOrdering */ if (obsolete.get()) { - sstableContext.indexDescriptor.deleteColumnIndex(indexContext); + sstableContext.indexDescriptor.deleteColumnIndex(indexTermType, indexIdentifier); } } } @@ -238,23 +245,25 @@ public abstract class SSTableIndex implements SegmentOrdering if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; SSTableIndex other = (SSTableIndex)o; - return Objects.equal(sstableContext, other.sstableContext) && Objects.equal(indexContext, other.indexContext); + return Objects.equal(sstableContext, other.sstableContext) && + Objects.equal(indexTermType, other.indexTermType) && + Objects.equal(indexIdentifier, other.indexIdentifier); } @Override public int hashCode() { - return Objects.hashCode(sstableContext, indexContext); + return Objects.hashCode(sstableContext, indexTermType, indexIdentifier); } @Override public String toString() { return MoreObjects.toStringHelper(this) - .add("column", indexContext.getColumnName()) + .add("column", indexTermType.columnName()) .add("sstable", sstableContext.sstable.descriptor) - .add("minTerm", indexContext.getValidator().getString(minTerm())) - .add("maxTerm", indexContext.getValidator().getString(maxTerm())) + .add("minTerm", indexTermType.asString(minTerm())) + .add("maxTerm", indexTermType.asString(maxTerm())) .add("totalRows", sstableContext.sstable.getTotalRows()) .toString(); } diff --git a/src/java/org/apache/cassandra/index/sai/disk/format/IndexDescriptor.java b/src/java/org/apache/cassandra/index/sai/disk/format/IndexDescriptor.java index eda5cb7ed3..8280cf3664 100644 --- a/src/java/org/apache/cassandra/index/sai/disk/format/IndexDescriptor.java +++ b/src/java/org/apache/cassandra/index/sai/disk/format/IndexDescriptor.java @@ -32,10 +32,9 @@ import org.slf4j.LoggerFactory; import org.apache.cassandra.db.ClusteringComparator; import org.apache.cassandra.db.lifecycle.LifecycleNewTracker; import org.apache.cassandra.dht.IPartitioner; -import org.apache.cassandra.index.sai.IndexContext; +import org.apache.cassandra.index.sai.StorageAttachedIndex; import org.apache.cassandra.index.sai.IndexValidation; import org.apache.cassandra.index.sai.SSTableContext; -import org.apache.cassandra.index.sai.StorageAttachedIndex; import org.apache.cassandra.index.sai.disk.PerColumnIndexWriter; import org.apache.cassandra.index.sai.disk.PerSSTableIndexWriter; import org.apache.cassandra.index.sai.disk.PrimaryKeyMap; @@ -43,6 +42,8 @@ import org.apache.cassandra.index.sai.disk.RowMapping; import org.apache.cassandra.index.sai.disk.SSTableIndex; import org.apache.cassandra.index.sai.disk.io.IndexFileUtils; import org.apache.cassandra.index.sai.disk.io.IndexOutputWriter; +import org.apache.cassandra.index.sai.utils.IndexIdentifier; +import org.apache.cassandra.index.sai.utils.IndexTermType; import org.apache.cassandra.index.sai.utils.PrimaryKey; import org.apache.cassandra.io.sstable.Component; import org.apache.cassandra.io.sstable.Descriptor; @@ -123,9 +124,9 @@ public class IndexDescriptor return version.onDiskFormat().newPrimaryKeyMapFactory(this, sstable); } - public SSTableIndex newSSTableIndex(SSTableContext sstableContext, IndexContext indexContext) + public SSTableIndex newSSTableIndex(SSTableContext sstableContext, StorageAttachedIndex index) { - return version.onDiskFormat().newSSTableIndex(sstableContext, indexContext); + return version.onDiskFormat().newSSTableIndex(sstableContext, index); } public PerSSTableIndexWriter newPerSSTableIndexWriter() throws IOException @@ -145,9 +146,9 @@ public class IndexDescriptor return version.onDiskFormat().isPerSSTableIndexBuildComplete(this); } - public boolean isPerColumnIndexBuildComplete(IndexContext indexContext) + public boolean isPerColumnIndexBuildComplete(IndexIdentifier indexIdentifier) { - return version.onDiskFormat().isPerColumnIndexBuildComplete(this, indexContext); + return version.onDiskFormat().isPerColumnIndexBuildComplete(this, indexIdentifier); } public boolean hasComponent(IndexComponent indexComponent) @@ -155,9 +156,9 @@ public class IndexDescriptor return fileFor(indexComponent).exists(); } - public boolean hasComponent(IndexComponent indexComponent, IndexContext indexContext) + public boolean hasComponent(IndexComponent indexComponent, IndexIdentifier indexIdentifier) { - return fileFor(indexComponent, indexContext).exists(); + return fileFor(indexComponent, indexIdentifier).exists(); } public File fileFor(IndexComponent indexComponent) @@ -165,19 +166,19 @@ public class IndexDescriptor return createFile(indexComponent, null); } - public File fileFor(IndexComponent indexComponent, IndexContext indexContext) + public File fileFor(IndexComponent indexComponent, IndexIdentifier indexIdentifier) { - return createFile(indexComponent, indexContext); + return createFile(indexComponent, indexIdentifier); } - public boolean isIndexEmpty(IndexContext indexContext) + public boolean isIndexEmpty(IndexTermType indexTermType, IndexIdentifier indexIdentifier) { // The index is empty if the index build completed successfully in that both // a GROUP_COMPLETION_MARKER companent and a COLUMN_COMPLETION_MARKER exist for // the index and the number of per-index components is 1 indicating that only the // COLUMN_COMPLETION_MARKER exists for the index, as this is the only file that // will be written if the index is empty - return isPerColumnIndexBuildComplete(indexContext) && numberOfPerIndexComponents(indexContext) == 1; + return isPerColumnIndexBuildComplete(indexIdentifier) && numberOfPerIndexComponents(indexTermType, indexIdentifier) == 1; } public void createComponentOnDisk(IndexComponent component) throws IOException @@ -185,9 +186,9 @@ public class IndexDescriptor Files.touch(fileFor(component).toJavaIOFile()); } - public void createComponentOnDisk(IndexComponent component, IndexContext indexContext) throws IOException + public void createComponentOnDisk(IndexComponent component, IndexIdentifier indexIdentifier) throws IOException { - Files.touch(fileFor(component, indexContext).toJavaIOFile()); + Files.touch(fileFor(component, indexIdentifier).toJavaIOFile()); } public IndexInput openPerSSTableInput(IndexComponent indexComponent) @@ -201,9 +202,9 @@ public class IndexDescriptor return IndexFileUtils.instance.openBlockingInput(file); } - public IndexInput openPerIndexInput(IndexComponent indexComponent, IndexContext indexContext) + public IndexInput openPerIndexInput(IndexComponent indexComponent, IndexIdentifier indexIdentifier) { - final File file = fileFor(indexComponent, indexContext); + final File file = fileFor(indexComponent, indexIdentifier); if (logger.isTraceEnabled()) logger.trace(logMessage("Opening blocking index input for file {} ({})"), file, @@ -236,19 +237,17 @@ public class IndexDescriptor return writer; } - public IndexOutputWriter openPerIndexOutput(IndexComponent indexComponent, IndexContext indexContext) throws IOException + public IndexOutputWriter openPerIndexOutput(IndexComponent indexComponent, IndexIdentifier indexIdentifier) throws IOException { - return openPerIndexOutput(indexComponent, indexContext, false); + return openPerIndexOutput(indexComponent, indexIdentifier, false); } - public IndexOutputWriter openPerIndexOutput(IndexComponent component, IndexContext indexContext, boolean append) throws IOException + public IndexOutputWriter openPerIndexOutput(IndexComponent component, IndexIdentifier indexIdentifier, boolean append) throws IOException { - final File file = fileFor(component, indexContext); + final File file = fileFor(component, indexIdentifier); if (logger.isTraceEnabled()) - logger.trace(indexContext.logMessage("Creating sstable attached index output for component {} on file {}..."), - component, - file); + logger.trace(logMessage("Creating sstable attached index output for component {} on file {}..."), component, file); IndexOutputWriter writer = IndexFileUtils.instance.openOutput(file); @@ -267,10 +266,7 @@ public class IndexDescriptor final File file = fileFor(indexComponent); if (logger.isTraceEnabled()) - { - logger.trace(logMessage("Opening {} file handle for {} ({})"), - file, FBUtilities.prettyPrintMemory(file.length())); - } + logger.trace(logMessage("Opening file handle for {} ({})"), file, FBUtilities.prettyPrintMemory(file.length())); return new FileHandle.Builder(file).mmapped(true).complete(); } @@ -280,17 +276,19 @@ public class IndexDescriptor } } - public FileHandle createPerIndexFileHandle(IndexComponent indexComponent, IndexContext indexContext, Throwables.DiscreteAction cleanup) + public FileHandle createPerIndexFileHandle(IndexComponent indexComponent, IndexIdentifier indexIdentifier) + { + return createPerIndexFileHandle(indexComponent, indexIdentifier, null); + } + + public FileHandle createPerIndexFileHandle(IndexComponent indexComponent, IndexIdentifier indexIdentifier, Throwables.DiscreteAction cleanup) { try { - final File file = fileFor(indexComponent, indexContext); + final File file = fileFor(indexComponent, indexIdentifier); if (logger.isTraceEnabled()) - { - logger.trace(indexContext.logMessage("Opening file handle for {} ({})"), - file, FBUtilities.prettyPrintMemory(file.length())); - } + logger.trace(logMessage("Opening file handle for {} ({})"), file, FBUtilities.prettyPrintMemory(file.length())); return new FileHandle.Builder(file).mmapped(true).complete(); } @@ -326,13 +324,13 @@ public class IndexDescriptor .collect(Collectors.toSet()); } - public Set getLivePerIndexComponents(IndexContext indexContext) + public Set getLivePerIndexComponents(IndexTermType indexTermType, IndexIdentifier indexIdentifier) { return version.onDiskFormat() - .perColumnIndexComponents(indexContext) + .perColumnIndexComponents(indexTermType) .stream() - .filter(c -> fileFor(c, indexContext).exists()) - .map(c -> version.makePerIndexComponent(c, indexContext)) + .filter(c -> fileFor(c, indexIdentifier).exists()) + .map(c -> version.makePerIndexComponent(c, indexIdentifier)) .collect(Collectors.toSet()); } @@ -347,36 +345,36 @@ public class IndexDescriptor .sum(); } - public long sizeOnDiskOfPerIndexComponents(IndexContext indexContext) + public long sizeOnDiskOfPerIndexComponents(IndexTermType indexTermType, IndexIdentifier indexIdentifier) { return version.onDiskFormat() - .perColumnIndexComponents(indexContext) + .perColumnIndexComponents(indexTermType) .stream() - .map(c -> fileFor(c, indexContext)) + .map(c -> fileFor(c, indexIdentifier)) .filter(File::exists) .mapToLong(File::length) .sum(); } @VisibleForTesting - public long sizeOnDiskOfPerIndexComponent(IndexComponent indexComponent, IndexContext indexContext) + public long sizeOnDiskOfPerIndexComponent(IndexComponent indexComponent, IndexIdentifier indexIdentifier) { - File componentFile = fileFor(indexComponent, indexContext); + File componentFile = fileFor(indexComponent, indexIdentifier); return componentFile.exists() ? componentFile.length() : 0; } @SuppressWarnings("BooleanMethodIsAlwaysInverted") - public boolean validatePerIndexComponents(IndexContext indexContext, IndexValidation validation) + public boolean validatePerIndexComponents(IndexTermType indexTermType, IndexIdentifier indexIdentifier, IndexValidation validation) { if (validation == IndexValidation.NONE) return true; - logger.info(indexContext.logMessage("Validating per-column index components using mode " + validation)); + logger.info(logMessage("Validating per-column index components for {} using mode {}"), indexIdentifier, validation); boolean checksum = validation == IndexValidation.CHECKSUM; try { - version.onDiskFormat().validatePerColumnIndexComponents(this, indexContext, checksum); + version.onDiskFormat().validatePerColumnIndexComponents(this, indexTermType, indexIdentifier, checksum); return true; } catch (UncheckedIOException e) @@ -405,9 +403,9 @@ public class IndexDescriptor } } - public void checksumPerIndexComponents(IndexContext indexContext) + public void checksumPerIndexComponents(IndexTermType indexTermType, IndexIdentifier indexIdentifier) { - version.onDiskFormat().validatePerColumnIndexComponents(this, indexContext, true); + version.onDiskFormat().validatePerColumnIndexComponents(this, indexTermType, indexIdentifier, true); } @SuppressWarnings("BooleanMethodIsAlwaysInverted") @@ -426,12 +424,12 @@ public class IndexDescriptor .forEach(this::deleteComponent); } - public void deleteColumnIndex(IndexContext indexContext) + public void deleteColumnIndex(IndexTermType indexTermType, IndexIdentifier indexIdentifier) { version.onDiskFormat() - .perColumnIndexComponents(indexContext) + .perColumnIndexComponents(indexTermType) .stream() - .map(c -> fileFor(c, indexContext)) + .map(c -> fileFor(c, indexIdentifier)) .filter(File::exists) .forEach(this::deleteComponent); } @@ -467,18 +465,18 @@ public class IndexDescriptor message); } - private File createFile(IndexComponent component, IndexContext indexContext) + private File createFile(IndexComponent component, IndexIdentifier indexIdentifier) { - Component customComponent = version.makePerIndexComponent(component, indexContext); + Component customComponent = version.makePerIndexComponent(component, indexIdentifier); return sstableDescriptor.fileFor(customComponent); } - private long numberOfPerIndexComponents(IndexContext indexContext) + private long numberOfPerIndexComponents(IndexTermType indexTermType, IndexIdentifier indexIdentifier) { return version.onDiskFormat() - .perColumnIndexComponents(indexContext) + .perColumnIndexComponents(indexTermType) .stream() - .map(c -> fileFor(c, indexContext)) + .map(c -> fileFor(c, indexIdentifier)) .filter(File::exists) .count(); } diff --git a/src/java/org/apache/cassandra/index/sai/disk/format/OnDiskFormat.java b/src/java/org/apache/cassandra/index/sai/disk/format/OnDiskFormat.java index 0d06f11e29..30ba3b6295 100644 --- a/src/java/org/apache/cassandra/index/sai/disk/format/OnDiskFormat.java +++ b/src/java/org/apache/cassandra/index/sai/disk/format/OnDiskFormat.java @@ -23,7 +23,6 @@ import java.io.UncheckedIOException; import java.util.Set; import org.apache.cassandra.db.lifecycle.LifecycleNewTracker; -import org.apache.cassandra.index.sai.IndexContext; import org.apache.cassandra.index.sai.SSTableContext; import org.apache.cassandra.index.sai.StorageAttachedIndex; import org.apache.cassandra.index.sai.disk.PerColumnIndexWriter; @@ -31,6 +30,8 @@ import org.apache.cassandra.index.sai.disk.PerSSTableIndexWriter; import org.apache.cassandra.index.sai.disk.PrimaryKeyMap; import org.apache.cassandra.index.sai.disk.RowMapping; import org.apache.cassandra.index.sai.disk.SSTableIndex; +import org.apache.cassandra.index.sai.utils.IndexIdentifier; +import org.apache.cassandra.index.sai.utils.IndexTermType; import org.apache.cassandra.io.sstable.format.SSTableReader; /** @@ -42,12 +43,10 @@ import org.apache.cassandra.io.sstable.format.SSTableReader; *

    *
  • Methods taking no parameters. These methods return static information about the * format. This can include static information about the per-sstable components
    • - *
  • Methods taking just an {@link IndexContext}. These methods return static information - * specific to the index. This can be information relating to the type of index being used
    • *
  • Methods taking an {@link IndexDescriptor}. These methods interact with the on-disk components, or * return objects that will interact with the on-disk components, or return information about the on-disk - * components. If they take an {@link IndexContext} as well they will be interacting with per-column index files; - * otherwise they will be interacting with per-sstable index files
    • + * components. If they take an {@link IndexTermType} and/or a {@link IndexIdentifier} as well they will be + * interacting with per-column index files; otherwise they will be interacting with per-sstable index files *
  • Methods taking an {@link IndexComponent}. These methods only interact with a single index component or * set of index components
    • * @@ -67,10 +66,10 @@ public interface OnDiskFormat * Create a new {@link SSTableIndex} for an on-disk index. * * @param sstableContext The {@link SSTableContext} holding the per-SSTable information for the index - * @param indexContext The {@link IndexContext} holding the per-index information for the index + * @param index The {@link StorageAttachedIndex} * @return the new {@link SSTableIndex} for the on-disk index */ - SSTableIndex newSSTableIndex(SSTableContext sstableContext, IndexContext indexContext); + SSTableIndex newSSTableIndex(SSTableContext sstableContext, StorageAttachedIndex index); /** * Create a new {@link PerSSTableIndexWriter} to write the per-SSTable on-disk components of an index. @@ -107,9 +106,9 @@ public interface OnDiskFormat * Returns true if the per-column index components have been built and are valid. * * @param indexDescriptor The {@link IndexDescriptor} for the SSTable SAI index - * @param indexContext The {@link IndexContext} for the index + * @param indexIdentifier The {@link IndexIdentifier} for the index */ - boolean isPerColumnIndexBuildComplete(IndexDescriptor indexDescriptor, IndexContext indexContext); + boolean isPerColumnIndexBuildComplete(IndexDescriptor indexDescriptor, IndexIdentifier indexIdentifier); /** * Validate all the per-SSTable on-disk components and throw if a component is not valid @@ -125,12 +124,13 @@ public interface OnDiskFormat * Validate all the per-column on-disk components and throw if a component is not valid * * @param indexDescriptor The {@link IndexDescriptor} for the SSTable SAI index - * @param indexContext The {@link IndexContext} holding the per-index information for the index + * @param indexTermType The {@link IndexTermType} of the index + * @param indexIdentifier The {@link IndexIdentifier} for the index * @param checksum {@code true} if the checksum should be tested as part of the validation * * @throws UncheckedIOException if there is a problem validating any on-disk component */ - void validatePerColumnIndexComponents(IndexDescriptor indexDescriptor, IndexContext indexContext, boolean checksum); + void validatePerColumnIndexComponents(IndexDescriptor indexDescriptor, IndexTermType indexTermType, IndexIdentifier indexIdentifier, boolean checksum); /** * Returns the set of {@link IndexComponent} for the per-SSTable part of an index. @@ -146,9 +146,9 @@ public interface OnDiskFormat * This is a complete set of components that could exist on-disk. It does not imply that the * components currently exist on-disk. * - * @param indexContext The {@link IndexContext} for the index + * @param indexTermType the {@link IndexTermType} of the index */ - Set perColumnIndexComponents(IndexContext indexContext); + Set perColumnIndexComponents(IndexTermType indexTermType); /** * Return the number of open per-SSTable files that can be open during a query. @@ -163,8 +163,6 @@ public interface OnDiskFormat * Return the number of open per-column index files that can be open during a query. * This is a static indication of the files that can be help open by an index * for queries. It is not a dynamic calculation. - * - * @param indexContext The {@link IndexContext} for the index */ - int openFilesPerColumnIndex(IndexContext indexContext); + int openFilesPerColumnIndex(); } diff --git a/src/java/org/apache/cassandra/index/sai/disk/format/Version.java b/src/java/org/apache/cassandra/index/sai/disk/format/Version.java index f5f3c3de3f..a536e2e91f 100644 --- a/src/java/org/apache/cassandra/index/sai/disk/format/Version.java +++ b/src/java/org/apache/cassandra/index/sai/disk/format/Version.java @@ -22,11 +22,13 @@ import java.util.SortedSet; import java.util.TreeSet; import java.util.stream.Collectors; +import javax.annotation.Nullable; + import com.google.common.base.Objects; import org.apache.cassandra.config.CassandraRelevantProperties; -import org.apache.cassandra.index.sai.IndexContext; import org.apache.cassandra.index.sai.disk.v1.V1OnDiskFormat; +import org.apache.cassandra.index.sai.utils.IndexIdentifier; import org.apache.cassandra.io.sstable.Component; import org.apache.cassandra.io.sstable.Descriptor; @@ -115,9 +117,9 @@ public class Version implements Comparable return indexComponent.type.createComponent(fileNameFormatter.format(indexComponent, null)); } - public Component makePerIndexComponent(IndexComponent indexComponent, IndexContext indexContext) + public Component makePerIndexComponent(IndexComponent indexComponent, IndexIdentifier indexIdentifier) { - return indexComponent.type.createComponent(fileNameFormatter.format(indexComponent, indexContext)); + return indexComponent.type.createComponent(fileNameFormatter.format(indexComponent, indexIdentifier)); } public FileNameFormatter fileNameFormatter() @@ -127,24 +129,26 @@ public class Version implements Comparable public interface FileNameFormatter { - String format(IndexComponent indexComponent, IndexContext indexContext); + String format(IndexComponent indexComponent, IndexIdentifier indexIdentifier); } /** * SAI default filename formatter. This is the current SAI on-disk filename format - * + *

    * Format: {@code -SAI+(+)+.db} * Note: The index name is excluded for per-SSTable index files that are shared * across all the per-column indexes for the SSTable. */ - private static String defaultFileNameFormat(IndexComponent indexComponent, IndexContext indexContext, String version) + private static String defaultFileNameFormat(IndexComponent indexComponent, + @Nullable IndexIdentifier indexIdentifier, + String version) { StringBuilder stringBuilder = new StringBuilder(); stringBuilder.append(SAI_DESCRIPTOR); stringBuilder.append(SAI_SEPARATOR).append(version); - if (indexContext != null) - stringBuilder.append(SAI_SEPARATOR).append(indexContext.getIndexName()); + if (indexIdentifier != null) + stringBuilder.append(SAI_SEPARATOR).append(indexIdentifier.indexName); stringBuilder.append(SAI_SEPARATOR).append(indexComponent.name); stringBuilder.append(Descriptor.EXTENSION); diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/ColumnCompletionMarkerUtil.java b/src/java/org/apache/cassandra/index/sai/disk/v1/ColumnCompletionMarkerUtil.java index 760083ad9d..6d3a3dd927 100644 --- a/src/java/org/apache/cassandra/index/sai/disk/v1/ColumnCompletionMarkerUtil.java +++ b/src/java/org/apache/cassandra/index/sai/disk/v1/ColumnCompletionMarkerUtil.java @@ -20,10 +20,10 @@ package org.apache.cassandra.index.sai.disk.v1; import java.io.IOException; -import org.apache.cassandra.index.sai.IndexContext; import org.apache.cassandra.index.sai.disk.format.IndexComponent; import org.apache.cassandra.index.sai.disk.format.IndexDescriptor; import org.apache.cassandra.index.sai.disk.io.IndexOutputWriter; +import org.apache.cassandra.index.sai.utils.IndexIdentifier; import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.IndexOutput; @@ -43,12 +43,12 @@ public class ColumnCompletionMarkerUtil * Creates a column index completion marker for the specified column index, storing in it whether the index is empty. * * @param descriptor the index descriptor - * @param context the column index context + * @param indexIdentifier the column index identifier * @param isEmpty whether the index is empty */ - public static void create(IndexDescriptor descriptor, IndexContext context, boolean isEmpty) throws IOException + public static void create(IndexDescriptor descriptor, IndexIdentifier indexIdentifier, boolean isEmpty) throws IOException { - try (IndexOutputWriter output = descriptor.openPerIndexOutput(IndexComponent.COLUMN_COMPLETION_MARKER, context)) + try (IndexOutputWriter output = descriptor.openPerIndexOutput(IndexComponent.COLUMN_COMPLETION_MARKER, indexIdentifier)) { SAICodecUtils.writeHeader(output); output.writeByte(isEmpty ? EMPTY : NOT_EMPTY); @@ -60,12 +60,12 @@ public class ColumnCompletionMarkerUtil * Reads the column index completion marker and returns whether if the index is empty. * * @param descriptor the index descriptor - * @param context the column index context + * @param indexIdentifier the column index identifier * @return {@code true} if the index is empty, {@code false} otherwise. */ - public static boolean isEmptyIndex(IndexDescriptor descriptor, IndexContext context) throws IOException + public static boolean isEmptyIndex(IndexDescriptor descriptor, IndexIdentifier indexIdentifier) throws IOException { - try (IndexInput input = descriptor.openPerIndexInput(IndexComponent.COLUMN_COMPLETION_MARKER, context)) + try (IndexInput input = descriptor.openPerIndexInput(IndexComponent.COLUMN_COMPLETION_MARKER, indexIdentifier)) { SAICodecUtils.checkHeader(input); // consume header return input.readByte() == EMPTY; diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/IndexWriterConfig.java b/src/java/org/apache/cassandra/index/sai/disk/v1/IndexWriterConfig.java index f40b2a04fa..fe9be1c98d 100644 --- a/src/java/org/apache/cassandra/index/sai/disk/v1/IndexWriterConfig.java +++ b/src/java/org/apache/cassandra/index/sai/disk/v1/IndexWriterConfig.java @@ -23,9 +23,9 @@ import java.util.stream.Collectors; import io.github.jbellis.jvector.vector.VectorSimilarityFunction; import org.apache.cassandra.config.CassandraRelevantProperties; -import org.apache.cassandra.db.marshal.AbstractType; import org.apache.cassandra.exceptions.InvalidRequestException; import org.apache.cassandra.index.sai.disk.v1.vector.OptimizeFor; +import org.apache.cassandra.index.sai.utils.IndexTermType; import static org.apache.cassandra.config.CassandraRelevantProperties.SAI_VECTOR_SEARCH_MAX_TOP_K; @@ -100,7 +100,7 @@ public class IndexWriterConfig return optimizeFor; } - public static IndexWriterConfig fromOptions(String indexName, AbstractType type, Map options) + public static IndexWriterConfig fromOptions(String indexName, IndexTermType indexTermType, Map options) { int maximumNodeConnections = DEFAULT_MAXIMUM_NODE_CONNECTIONS; int queueSize = DEFAULT_CONSTRUCTION_BEAM_WIDTH; @@ -112,8 +112,8 @@ public class IndexWriterConfig options.get(SIMILARITY_FUNCTION) != null || options.get(OPTIMIZE_FOR) != null) { - if (!type.isVector()) - throw new InvalidRequestException(String.format("CQL type %s cannot have vector options", type.asCQL3Type())); + if (!indexTermType.isVector()) + throw new InvalidRequestException(String.format("CQL type %s cannot have vector options", indexTermType.asCQL3Type())); if (options.containsKey(MAXIMUM_NODE_CONNECTIONS)) { diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/MemtableIndexWriter.java b/src/java/org/apache/cassandra/index/sai/disk/v1/MemtableIndexWriter.java index cd1495c3f6..0f8e322626 100644 --- a/src/java/org/apache/cassandra/index/sai/disk/v1/MemtableIndexWriter.java +++ b/src/java/org/apache/cassandra/index/sai/disk/v1/MemtableIndexWriter.java @@ -27,9 +27,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.carrotsearch.hppc.LongArrayList; -import org.apache.cassandra.db.marshal.AbstractType; import org.apache.cassandra.db.rows.Row; -import org.apache.cassandra.index.sai.IndexContext; import org.apache.cassandra.index.sai.disk.PerColumnIndexWriter; import org.apache.cassandra.index.sai.disk.RowMapping; import org.apache.cassandra.index.sai.disk.format.IndexComponent; @@ -40,8 +38,10 @@ import org.apache.cassandra.index.sai.disk.v1.segment.SegmentMetadata; import org.apache.cassandra.index.sai.disk.v1.trie.LiteralIndexWriter; import org.apache.cassandra.index.sai.memory.MemtableIndex; import org.apache.cassandra.index.sai.memory.MemtableTermsIterator; +import org.apache.cassandra.index.sai.metrics.IndexMetrics; +import org.apache.cassandra.index.sai.utils.IndexIdentifier; +import org.apache.cassandra.index.sai.utils.IndexTermType; import org.apache.cassandra.index.sai.utils.PrimaryKey; -import org.apache.cassandra.index.sai.utils.TypeUtil; import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.Pair; import org.apache.cassandra.utils.bytecomparable.ByteComparable; @@ -55,19 +55,25 @@ public class MemtableIndexWriter implements PerColumnIndexWriter private static final Logger logger = LoggerFactory.getLogger(MemtableIndexWriter.class); private final IndexDescriptor indexDescriptor; - private final IndexContext indexContext; + private final IndexTermType indexTermType; + private final IndexIdentifier indexIdentifier; + private final IndexMetrics indexMetrics; private final MemtableIndex memtable; private final RowMapping rowMapping; public MemtableIndexWriter(MemtableIndex memtable, IndexDescriptor indexDescriptor, - IndexContext indexContext, + IndexTermType indexTermType, + IndexIdentifier indexIdentifier, + IndexMetrics indexMetrics, RowMapping rowMapping) { assert rowMapping != null && rowMapping != RowMapping.DUMMY : "Row mapping must exist during FLUSH."; this.indexDescriptor = indexDescriptor; - this.indexContext = indexContext; + this.indexTermType = indexTermType; + this.indexIdentifier = indexIdentifier; + this.indexMetrics = indexMetrics; this.memtable = memtable; this.rowMapping = rowMapping; } @@ -83,8 +89,8 @@ public class MemtableIndexWriter implements PerColumnIndexWriter @Override public void abort(Throwable cause) { - logger.warn(indexContext.logMessage("Aborting index memtable flush for {}..."), indexDescriptor.sstableDescriptor, cause); - indexDescriptor.deleteColumnIndex(indexContext); + logger.warn(indexIdentifier.logMessage("Aborting index memtable flush for {}..."), indexDescriptor.sstableDescriptor, cause); + indexDescriptor.deleteColumnIndex(indexTermType, indexIdentifier); } @Override @@ -98,15 +104,15 @@ public class MemtableIndexWriter implements PerColumnIndexWriter { if (!rowMapping.hasRows() || memtable == null || memtable.isEmpty()) { - logger.debug(indexContext.logMessage("No indexed rows to flush from SSTable {}."), indexDescriptor.sstableDescriptor); + logger.debug(indexIdentifier.logMessage("No indexed rows to flush from SSTable {}."), indexDescriptor.sstableDescriptor); // Write a completion marker even though we haven't written anything to the index, // so we won't try to build the index again for the SSTable - ColumnCompletionMarkerUtil.create(indexDescriptor, indexContext, true); + ColumnCompletionMarkerUtil.create(indexDescriptor, indexIdentifier, true); return; } - if (indexContext.isVector()) + if (indexTermType.isVector()) { flushVectorIndex(start, stopwatch); } @@ -116,7 +122,7 @@ public class MemtableIndexWriter implements PerColumnIndexWriter try (MemtableTermsIterator terms = new MemtableTermsIterator(memtable.getMinTerm(), memtable.getMaxTerm(), iterator)) { - long cellCount = flush(indexContext.getValidator(), terms, rowMapping.maxSSTableRowId); + long cellCount = flush(terms, rowMapping.maxSSTableRowId); completeIndexFlush(cellCount, start, stopwatch); } @@ -124,21 +130,21 @@ public class MemtableIndexWriter implements PerColumnIndexWriter } catch (Throwable t) { - logger.error(indexContext.logMessage("Error while flushing index {}"), t.getMessage(), t); - indexContext.getIndexMetrics().memtableIndexFlushErrors.inc(); + logger.error(indexIdentifier.logMessage("Error while flushing index {}"), t.getMessage(), t); + indexMetrics.memtableIndexFlushErrors.inc(); throw t; } } - private long flush(AbstractType termComparator, MemtableTermsIterator terms, long maxSSTableRowId) throws IOException + private long flush(MemtableTermsIterator terms, long maxSSTableRowId) throws IOException { long numRows; SegmentMetadata.ComponentMetadataMap indexMetas; - if (TypeUtil.isLiteral(termComparator)) + if (indexTermType.isLiteral()) { - try (LiteralIndexWriter writer = new LiteralIndexWriter(indexDescriptor, indexContext)) + try (LiteralIndexWriter writer = new LiteralIndexWriter(indexDescriptor, indexIdentifier)) { indexMetas = writer.writeCompleteSegment(terms); numRows = writer.getPostingsCount(); @@ -147,10 +153,10 @@ public class MemtableIndexWriter implements PerColumnIndexWriter else { NumericIndexWriter writer = new NumericIndexWriter(indexDescriptor, - indexContext, - TypeUtil.fixedSizeOf(termComparator), + indexIdentifier, + indexTermType.fixedSizeOf(), maxSSTableRowId); - indexMetas = writer.writeCompleteSegment(BlockBalancedTreeIterator.fromTermsIterator(terms, termComparator)); + indexMetas = writer.writeCompleteSegment(BlockBalancedTreeIterator.fromTermsIterator(terms, indexTermType)); numRows = writer.getValueCount(); } @@ -158,7 +164,7 @@ public class MemtableIndexWriter implements PerColumnIndexWriter // so that the index is correctly identified as being empty (only having a completion marker) if (numRows == 0) { - indexDescriptor.deleteColumnIndex(indexContext); + indexDescriptor.deleteColumnIndex(indexTermType, indexIdentifier); return 0; } @@ -173,7 +179,7 @@ public class MemtableIndexWriter implements PerColumnIndexWriter terms.getMaxTerm(), indexMetas); - try (MetadataWriter writer = new MetadataWriter(indexDescriptor.openPerIndexOutput(IndexComponent.META, indexContext))) + try (MetadataWriter writer = new MetadataWriter(indexDescriptor.openPerIndexOutput(IndexComponent.META, indexIdentifier))) { SegmentMetadata.write(writer, Collections.singletonList(metadata)); } @@ -183,7 +189,7 @@ public class MemtableIndexWriter implements PerColumnIndexWriter private void flushVectorIndex(long startTime, Stopwatch stopwatch) throws IOException { - SegmentMetadata.ComponentMetadataMap metadataMap = memtable.writeDirect(indexDescriptor, indexContext, rowMapping::get); + SegmentMetadata.ComponentMetadataMap metadataMap = memtable.writeDirect(indexDescriptor, indexIdentifier, rowMapping::get); completeIndexFlush(rowMapping.size(), startTime, stopwatch); @@ -197,7 +203,7 @@ public class MemtableIndexWriter implements PerColumnIndexWriter ByteBufferUtil.bytes(0), metadataMap); - try (MetadataWriter writer = new MetadataWriter(indexDescriptor.openPerIndexOutput(IndexComponent.META, indexContext))) + try (MetadataWriter writer = new MetadataWriter(indexDescriptor.openPerIndexOutput(IndexComponent.META, indexIdentifier))) { SegmentMetadata.write(writer, Collections.singletonList(metadata)); } @@ -206,18 +212,18 @@ public class MemtableIndexWriter implements PerColumnIndexWriter private void completeIndexFlush(long cellCount, long startTime, Stopwatch stopwatch) throws IOException { // create a completion marker indicating that the index is complete and not-empty - ColumnCompletionMarkerUtil.create(indexDescriptor, indexContext, false); + ColumnCompletionMarkerUtil.create(indexDescriptor, indexIdentifier, false); - indexContext.getIndexMetrics().memtableIndexFlushCount.inc(); + indexMetrics.memtableIndexFlushCount.inc(); long elapsedTime = stopwatch.elapsed(TimeUnit.MILLISECONDS); - logger.debug(indexContext.logMessage("Completed flushing {} memtable index cells to SSTable {}. Duration: {} ms. Total elapsed: {} ms"), + logger.debug(indexIdentifier.logMessage("Completed flushing {} memtable index cells to SSTable {}. Duration: {} ms. Total elapsed: {} ms"), cellCount, indexDescriptor.sstableDescriptor, elapsedTime - startTime, elapsedTime); - indexContext.getIndexMetrics().memtableFlushCellsPerSecond.update((long) (cellCount * 1000.0 / Math.max(1, elapsedTime - startTime))); + indexMetrics.memtableFlushCellsPerSecond.update((long) (cellCount * 1000.0 / Math.max(1, elapsedTime - startTime))); } } diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/MetadataSource.java b/src/java/org/apache/cassandra/index/sai/disk/v1/MetadataSource.java index 766e52d575..e1c5c4f450 100644 --- a/src/java/org/apache/cassandra/index/sai/disk/v1/MetadataSource.java +++ b/src/java/org/apache/cassandra/index/sai/disk/v1/MetadataSource.java @@ -22,9 +22,9 @@ import java.util.HashMap; import java.util.Map; import javax.annotation.concurrent.NotThreadSafe; -import org.apache.cassandra.index.sai.IndexContext; import org.apache.cassandra.index.sai.disk.format.IndexComponent; import org.apache.cassandra.index.sai.disk.format.IndexDescriptor; +import org.apache.cassandra.index.sai.utils.IndexIdentifier; import org.apache.cassandra.index.sai.disk.io.IndexFileUtils; import org.apache.lucene.store.ByteArrayDataInput; import org.apache.lucene.store.ChecksumIndexInput; @@ -47,9 +47,9 @@ public class MetadataSource return MetadataSource.load(indexDescriptor.openPerSSTableInput(IndexComponent.GROUP_META)); } - public static MetadataSource loadColumnMetadata(IndexDescriptor indexDescriptor, IndexContext indexContext) throws IOException + public static MetadataSource loadColumnMetadata(IndexDescriptor indexDescriptor, IndexIdentifier indexIdentifier) throws IOException { - return MetadataSource.load(indexDescriptor.openPerIndexInput(IndexComponent.META, indexContext)); + return MetadataSource.load(indexDescriptor.openPerIndexInput(IndexComponent.META, indexIdentifier)); } private static MetadataSource load(IndexInput indexInput) throws IOException diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/PerColumnIndexFiles.java b/src/java/org/apache/cassandra/index/sai/disk/v1/PerColumnIndexFiles.java index 02832d937f..2e03d13b36 100644 --- a/src/java/org/apache/cassandra/index/sai/disk/v1/PerColumnIndexFiles.java +++ b/src/java/org/apache/cassandra/index/sai/disk/v1/PerColumnIndexFiles.java @@ -22,9 +22,10 @@ import java.io.Closeable; import java.util.EnumMap; import java.util.Map; -import org.apache.cassandra.index.sai.IndexContext; import org.apache.cassandra.index.sai.disk.format.IndexComponent; import org.apache.cassandra.index.sai.disk.format.IndexDescriptor; +import org.apache.cassandra.index.sai.utils.IndexIdentifier; +import org.apache.cassandra.index.sai.utils.IndexTermType; import org.apache.cassandra.io.util.FileHandle; import org.apache.cassandra.io.util.FileUtils; @@ -38,17 +39,17 @@ public class PerColumnIndexFiles implements Closeable { private final Map files = new EnumMap<>(IndexComponent.class); private final IndexDescriptor indexDescriptor; - private final IndexContext indexContext; + private final IndexIdentifier indexIdentifier; - public PerColumnIndexFiles(IndexDescriptor indexDescriptor, IndexContext indexContext) + public PerColumnIndexFiles(IndexDescriptor indexDescriptor, IndexTermType indexTermType, IndexIdentifier indexIdentifier) { this.indexDescriptor = indexDescriptor; - this.indexContext = indexContext; - for (IndexComponent component : indexDescriptor.version.onDiskFormat().perColumnIndexComponents(indexContext)) + this.indexIdentifier = indexIdentifier; + for (IndexComponent component : indexDescriptor.version.onDiskFormat().perColumnIndexComponents(indexTermType)) { if (component == IndexComponent.META || component == IndexComponent.COLUMN_COMPLETION_MARKER) continue; - files.put(component, indexDescriptor.createPerIndexFileHandle(component, indexContext, this::close)); + files.put(component, indexDescriptor.createPerIndexFileHandle(component, indexIdentifier, this::close)); } } @@ -76,7 +77,7 @@ public class PerColumnIndexFiles implements Closeable { FileHandle file = files.get(indexComponent); if (file == null) - throw new IllegalArgumentException(String.format(indexContext.logMessage("Component %s not found for SSTable %s"), + throw new IllegalArgumentException(String.format(indexIdentifier.logMessage("Component %s not found for SSTable %s"), indexComponent, indexDescriptor.sstableDescriptor)); return file.sharedCopy(); diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/SSTableIndexWriter.java b/src/java/org/apache/cassandra/index/sai/disk/v1/SSTableIndexWriter.java index 3f85c30225..d62b6421f5 100644 --- a/src/java/org/apache/cassandra/index/sai/disk/v1/SSTableIndexWriter.java +++ b/src/java/org/apache/cassandra/index/sai/disk/v1/SSTableIndexWriter.java @@ -30,9 +30,8 @@ import com.google.common.base.Stopwatch; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.cassandra.db.marshal.AbstractType; import org.apache.cassandra.db.rows.Row; -import org.apache.cassandra.index.sai.IndexContext; +import org.apache.cassandra.index.sai.StorageAttachedIndex; import org.apache.cassandra.index.sai.analyzer.AbstractAnalyzer; import org.apache.cassandra.index.sai.disk.PerColumnIndexWriter; import org.apache.cassandra.index.sai.disk.format.IndexComponent; @@ -41,7 +40,6 @@ import org.apache.cassandra.index.sai.disk.v1.segment.SegmentBuilder; import org.apache.cassandra.index.sai.disk.v1.segment.SegmentMetadata; import org.apache.cassandra.index.sai.utils.NamedMemoryLimiter; import org.apache.cassandra.index.sai.utils.PrimaryKey; -import org.apache.cassandra.index.sai.utils.TypeUtil; import org.apache.cassandra.utils.Clock; import org.apache.cassandra.utils.FBUtilities; @@ -54,7 +52,7 @@ public class SSTableIndexWriter implements PerColumnIndexWriter private static final Logger logger = LoggerFactory.getLogger(SSTableIndexWriter.class); private final IndexDescriptor indexDescriptor; - private final IndexContext indexContext; + private final StorageAttachedIndex index; private final long nowInSec = FBUtilities.nowInSeconds(); private final AbstractAnalyzer analyzer; private final NamedMemoryLimiter limiter; @@ -64,11 +62,14 @@ public class SSTableIndexWriter implements PerColumnIndexWriter private boolean aborted = false; private SegmentBuilder currentBuilder; - public SSTableIndexWriter(IndexDescriptor indexDescriptor, IndexContext indexContext, NamedMemoryLimiter limiter, BooleanSupplier isIndexValid) + public SSTableIndexWriter(IndexDescriptor indexDescriptor, + StorageAttachedIndex index, + NamedMemoryLimiter limiter, + BooleanSupplier isIndexValid) { this.indexDescriptor = indexDescriptor; - this.indexContext = indexContext; - this.analyzer = indexContext.getAnalyzerFactory().create(); + this.index = index; + this.analyzer = index.hasAnalyzer() ? index.analyzer() : null; this.limiter = limiter; this.isIndexValid = isIndexValid; } @@ -79,23 +80,23 @@ public class SSTableIndexWriter implements PerColumnIndexWriter if (maybeAbort()) return; - if (indexContext.isNonFrozenCollection()) + if (index.termType().isNonFrozenCollection()) { - Iterator valueIterator = indexContext.getValuesOf(row, nowInSec); + Iterator valueIterator = index.termType().valuesOf(row, nowInSec); if (valueIterator != null) { while (valueIterator.hasNext()) { ByteBuffer value = valueIterator.next(); - addTerm(TypeUtil.asIndexBytes(value.duplicate(), indexContext.getValidator()), key, sstableRowId, indexContext.getValidator()); + addTerm(index.termType().asIndexBytes(value.duplicate()), key, sstableRowId); } } } else { - ByteBuffer value = indexContext.getValueOf(key.partitionKey(), row, nowInSec); + ByteBuffer value = index.termType().valueOf(key.partitionKey(), row, nowInSec); if (value != null) - addTerm(TypeUtil.asIndexBytes(value.duplicate(), indexContext.getValidator()), key, sstableRowId, indexContext.getValidator()); + addTerm(index.termType().asIndexBytes(value.duplicate()), key, sstableRowId); } } @@ -109,7 +110,7 @@ public class SSTableIndexWriter implements PerColumnIndexWriter long elapsed; boolean emptySegment = currentBuilder == null || currentBuilder.isEmpty(); - logger.debug(indexContext.logMessage("Completing index flush with {}buffered data..."), emptySegment ? "no " : ""); + logger.debug(index.identifier().logMessage("Completing index flush with {}buffered data..."), emptySegment ? "no " : ""); try { @@ -118,7 +119,7 @@ public class SSTableIndexWriter implements PerColumnIndexWriter { flushSegment(); elapsed = stopwatch.elapsed(TimeUnit.MILLISECONDS); - logger.debug(indexContext.logMessage("Completed flush of final segment for SSTable {}. Duration: {} ms. Total elapsed: {} ms"), + logger.debug(index.identifier().logMessage("Completed flush of final segment for SSTable {}. Duration: {} ms. Total elapsed: {} ms"), indexDescriptor.sstableDescriptor, elapsed - start, elapsed); @@ -128,24 +129,21 @@ public class SSTableIndexWriter implements PerColumnIndexWriter if (currentBuilder != null) { long bytesAllocated = currentBuilder.totalBytesAllocated(); - long globalBytesUsed = currentBuilder.release(indexContext); - logger.debug(indexContext.logMessage("Flushing final segment for SSTable {} released {}. Global segment memory usage now at {}."), + long globalBytesUsed = currentBuilder.release(index.identifier()); + logger.debug(index.identifier().logMessage("Flushing final segment for SSTable {} released {}. Global segment memory usage now at {}."), indexDescriptor.sstableDescriptor, FBUtilities.prettyPrintMemory(bytesAllocated), FBUtilities.prettyPrintMemory(globalBytesUsed)); } writeSegmentsMetadata(); // write column index completion marker, indicating whether the index is empty - ColumnCompletionMarkerUtil.create(indexDescriptor, indexContext, segments.isEmpty()); + ColumnCompletionMarkerUtil.create(indexDescriptor, index.identifier(), segments.isEmpty()); } finally { - if (indexContext.getIndexMetrics() != null) - { - indexContext.getIndexMetrics().segmentsPerCompaction.update(segments.size()); - segments.clear(); - indexContext.getIndexMetrics().compactionCount.inc(); - } + index.indexMetrics().segmentsPerCompaction.update(segments.size()); + segments.clear(); + index.indexMetrics().compactionCount.inc(); } } @@ -154,7 +152,7 @@ public class SSTableIndexWriter implements PerColumnIndexWriter { aborted = true; - logger.warn(indexContext.logMessage("Aborting SSTable index flush for {}..."), indexDescriptor.sstableDescriptor, cause); + logger.warn(index.identifier().logMessage("Aborting SSTable index flush for {}..."), indexDescriptor.sstableDescriptor, cause); // It's possible for the current builder to be unassigned after we flush a final segment. if (currentBuilder != null) @@ -162,12 +160,12 @@ public class SSTableIndexWriter implements PerColumnIndexWriter // If an exception is thrown out of any writer operation prior to successful segment // flush, we will end up here, and we need to free up builder memory tracked by the limiter: long allocated = currentBuilder.totalBytesAllocated(); - long globalBytesUsed = currentBuilder.release(indexContext); - logger.debug(indexContext.logMessage("Aborting index writer for SSTable {} released {}. Global segment memory usage now at {}."), + long globalBytesUsed = currentBuilder.release(index.identifier()); + logger.debug(index.identifier().logMessage("Aborting index writer for SSTable {} released {}. Global segment memory usage now at {}."), indexDescriptor.sstableDescriptor, FBUtilities.prettyPrintMemory(allocated), FBUtilities.prettyPrintMemory(globalBytesUsed)); } - indexDescriptor.deleteColumnIndex(indexContext); + indexDescriptor.deleteColumnIndex(index.termType(), index.identifier()); } /** @@ -183,13 +181,13 @@ public class SSTableIndexWriter implements PerColumnIndexWriter if (isIndexValid.getAsBoolean()) return false; - abort(new RuntimeException(String.format("index %s is dropped", indexContext.getIndexName()))); + abort(new RuntimeException(String.format("index %s is dropped", index.identifier()))); return true; } - private void addTerm(ByteBuffer term, PrimaryKey key, long sstableRowId, AbstractType type) throws IOException + private void addTerm(ByteBuffer term, PrimaryKey key, long sstableRowId) throws IOException { - if (!indexContext.validateMaxTermSize(key.partitionKey(), term, false)) + if (!index.validateMaxTermSize(key.partitionKey(), term, false)) return; if (currentBuilder == null) @@ -204,7 +202,7 @@ public class SSTableIndexWriter implements PerColumnIndexWriter if (term.remaining() == 0) return; - if (!TypeUtil.isLiteral(type)) + if (analyzer == null || !index.termType().isLiteral()) { limiter.increment(currentBuilder.add(term, key, sstableRowId)); } @@ -233,8 +231,8 @@ public class SSTableIndexWriter implements PerColumnIndexWriter if (reachMemoryLimit) { - logger.debug(indexContext.logMessage("Global limit of {} and minimum flush size of {} exceeded. " + - "Current builder usage is {} for {} cells. Global Usage is {}. Flushing..."), + logger.debug(index.identifier().logMessage("Global limit of {} and minimum flush size of {} exceeded. " + + "Current builder usage is {} for {} cells. Global Usage is {}. Flushing..."), FBUtilities.prettyPrintMemory(limiter.limitBytes()), FBUtilities.prettyPrintMemory(currentBuilder.getMinimumFlushBytes()), FBUtilities.prettyPrintMemory(currentBuilder.totalBytesAllocated()), @@ -253,7 +251,7 @@ public class SSTableIndexWriter implements PerColumnIndexWriter { long bytesAllocated = currentBuilder.totalBytesAllocated(); - SegmentMetadata segmentMetadata = currentBuilder.flush(indexDescriptor, indexContext); + SegmentMetadata segmentMetadata = currentBuilder.flush(indexDescriptor, index.identifier()); long flushMillis = Math.max(1, TimeUnit.NANOSECONDS.toMillis(Clock.Global.nanoTime() - start)); @@ -262,14 +260,12 @@ public class SSTableIndexWriter implements PerColumnIndexWriter segments.add(segmentMetadata); double rowCount = segmentMetadata.numRows; - if (indexContext.getIndexMetrics() != null) - indexContext.getIndexMetrics().compactionSegmentCellsPerSecond.update((long)(rowCount / flushMillis * 1000.0)); + index.indexMetrics().compactionSegmentCellsPerSecond.update((long)(rowCount / flushMillis * 1000.0)); double segmentBytes = segmentMetadata.componentMetadatas.indexSize(); - if (indexContext.getIndexMetrics() != null) - indexContext.getIndexMetrics().compactionSegmentBytesPerSecond.update((long)(segmentBytes / flushMillis * 1000.0)); + index.indexMetrics().compactionSegmentBytesPerSecond.update((long)(segmentBytes / flushMillis * 1000.0)); - logger.debug(indexContext.logMessage("Flushed segment with {} cells for a total of {} in {} ms."), + logger.debug(index.identifier().logMessage("Flushed segment with {} cells for a total of {} in {} ms."), (long) rowCount, FBUtilities.prettyPrintMemory((long) segmentBytes), flushMillis); } @@ -277,17 +273,17 @@ public class SSTableIndexWriter implements PerColumnIndexWriter // flush. Note that any failure that occurs before this (even in term addition) will // actuate this column writer's abort logic from the parent SSTable-level writer, and // that abort logic will release the current builder's memory against the limiter. - long globalBytesUsed = currentBuilder.release(indexContext); + long globalBytesUsed = currentBuilder.release(index.identifier()); currentBuilder = null; - logger.debug(indexContext.logMessage("Flushing index segment for SSTable {} released {}. Global segment memory usage now at {}."), + logger.debug(index.identifier().logMessage("Flushing index segment for SSTable {} released {}. Global segment memory usage now at {}."), indexDescriptor.sstableDescriptor, FBUtilities.prettyPrintMemory(bytesAllocated), FBUtilities.prettyPrintMemory(globalBytesUsed)); } catch (Throwable t) { - logger.error(indexContext.logMessage("Failed to build index for SSTable {}."), indexDescriptor.sstableDescriptor, t); - indexDescriptor.deleteColumnIndex(indexContext); - indexContext.getIndexMetrics().segmentFlushErrors.inc(); + logger.error(index.identifier().logMessage("Failed to build index for SSTable {}."), indexDescriptor.sstableDescriptor, t); + indexDescriptor.deleteColumnIndex(index.termType(), index.identifier()); + index.indexMetrics().segmentFlushErrors.inc(); throw t; } } @@ -297,7 +293,7 @@ public class SSTableIndexWriter implements PerColumnIndexWriter if (segments.isEmpty()) return; - try (MetadataWriter writer = new MetadataWriter(indexDescriptor.openPerIndexOutput(IndexComponent.META, indexContext))) + try (MetadataWriter writer = new MetadataWriter(indexDescriptor.openPerIndexOutput(IndexComponent.META, index.identifier()))) { SegmentMetadata.write(writer, segments); } @@ -312,15 +308,15 @@ public class SSTableIndexWriter implements PerColumnIndexWriter { SegmentBuilder builder; - if (indexContext.isVector()) - builder = new SegmentBuilder.VectorSegmentBuilder(indexContext.getValidator(), limiter, indexContext.getIndexWriterConfig()); - else if (indexContext.isLiteral()) - builder = new SegmentBuilder.RAMStringSegmentBuilder(indexContext.getValidator(), limiter); + if (index.termType().isVector()) + builder = new SegmentBuilder.VectorSegmentBuilder(index.termType(), limiter, index.indexWriterConfig()); + else if (index.termType().isLiteral()) + builder = new SegmentBuilder.RAMStringSegmentBuilder(index.termType(), limiter); else - builder = new SegmentBuilder.BlockBalancedTreeSegmentBuilder(indexContext.getValidator(), limiter); + builder = new SegmentBuilder.BlockBalancedTreeSegmentBuilder(index.termType(), limiter); long globalBytesUsed = limiter.increment(builder.totalBytesAllocated()); - logger.debug(indexContext.logMessage("Created new segment builder while flushing SSTable {}. Global segment memory usage now at {}."), + logger.debug(index.identifier().logMessage("Created new segment builder while flushing SSTable {}. Global segment memory usage now at {}."), indexDescriptor.sstableDescriptor, FBUtilities.prettyPrintMemory(globalBytesUsed)); diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/V1OnDiskFormat.java b/src/java/org/apache/cassandra/index/sai/disk/v1/V1OnDiskFormat.java index 323d81131f..48b186fef9 100644 --- a/src/java/org/apache/cassandra/index/sai/disk/v1/V1OnDiskFormat.java +++ b/src/java/org/apache/cassandra/index/sai/disk/v1/V1OnDiskFormat.java @@ -24,8 +24,6 @@ import java.util.EnumSet; import java.util.Set; import com.google.common.annotations.VisibleForTesting; - -import org.apache.cassandra.utils.Throwables; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -33,7 +31,6 @@ import com.codahale.metrics.Gauge; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.compaction.OperationType; import org.apache.cassandra.db.lifecycle.LifecycleNewTracker; -import org.apache.cassandra.index.sai.IndexContext; import org.apache.cassandra.index.sai.SSTableContext; import org.apache.cassandra.index.sai.StorageAttachedIndex; import org.apache.cassandra.index.sai.disk.PerColumnIndexWriter; @@ -46,10 +43,13 @@ import org.apache.cassandra.index.sai.disk.format.IndexDescriptor; import org.apache.cassandra.index.sai.disk.format.OnDiskFormat; import org.apache.cassandra.index.sai.disk.v1.segment.SegmentBuilder; import org.apache.cassandra.index.sai.metrics.AbstractMetrics; +import org.apache.cassandra.index.sai.utils.IndexIdentifier; +import org.apache.cassandra.index.sai.utils.IndexTermType; import org.apache.cassandra.index.sai.utils.NamedMemoryLimiter; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.metrics.CassandraMetricsRegistry; import org.apache.cassandra.metrics.DefaultNameFactory; +import org.apache.cassandra.utils.Throwables; import org.apache.lucene.store.IndexInput; import static org.apache.cassandra.utils.FBUtilities.prettyPrintMemory; @@ -136,9 +136,9 @@ public class V1OnDiskFormat implements OnDiskFormat } @Override - public SSTableIndex newSSTableIndex(SSTableContext sstableContext, IndexContext indexContext) + public SSTableIndex newSSTableIndex(SSTableContext sstableContext, StorageAttachedIndex index) { - return new V1SSTableIndex(sstableContext, indexContext); + return new V1SSTableIndex(sstableContext, index); } @Override @@ -157,15 +157,17 @@ public class V1OnDiskFormat implements OnDiskFormat if (tracker.opType() != OperationType.FLUSH || !index.isInitBuildStarted()) { NamedMemoryLimiter limiter = SEGMENT_BUILD_MEMORY_LIMITER; - logger.info(index.getIndexContext().logMessage("Starting a compaction index build. Global segment memory usage: {}"), + logger.info(index.identifier().logMessage("Starting a compaction index build. Global segment memory usage: {}"), prettyPrintMemory(limiter.currentBytesUsed())); - return new SSTableIndexWriter(indexDescriptor, index.getIndexContext(), limiter, index.isIndexValid()); + return new SSTableIndexWriter(indexDescriptor, index, limiter, index.isIndexValid()); } - return new MemtableIndexWriter(index.getIndexContext().getMemtableIndexManager().getPendingMemtableIndex(tracker), + return new MemtableIndexWriter(index.memtableIndexManager().getPendingMemtableIndex(tracker), indexDescriptor, - index.getIndexContext(), + index.termType(), + index.identifier(), + index.indexMetrics(), rowMapping); } @@ -176,10 +178,10 @@ public class V1OnDiskFormat implements OnDiskFormat } @Override - public boolean isPerColumnIndexBuildComplete(IndexDescriptor indexDescriptor, IndexContext indexContext) + public boolean isPerColumnIndexBuildComplete(IndexDescriptor indexDescriptor, IndexIdentifier indexIdentifier) { return indexDescriptor.hasComponent(IndexComponent.GROUP_COMPLETION_MARKER) && - indexDescriptor.hasComponent(IndexComponent.COLUMN_COMPLETION_MARKER, indexContext); + indexDescriptor.hasComponent(IndexComponent.COLUMN_COMPLETION_MARKER, indexIdentifier); } @Override @@ -195,19 +197,19 @@ public class V1OnDiskFormat implements OnDiskFormat } @Override - public void validatePerColumnIndexComponents(IndexDescriptor indexDescriptor, IndexContext indexContext, boolean checksum) + public void validatePerColumnIndexComponents(IndexDescriptor indexDescriptor, IndexTermType indexTermType, IndexIdentifier indexIdentifier, boolean checksum) { // determine if the index is empty, which would be encoded in the column completion marker boolean isEmptyIndex = false; - if (indexDescriptor.hasComponent(IndexComponent.COLUMN_COMPLETION_MARKER, indexContext)) + if (indexDescriptor.hasComponent(IndexComponent.COLUMN_COMPLETION_MARKER, indexIdentifier)) { // first validate the file... - validateIndexComponent(indexDescriptor, indexContext, IndexComponent.COLUMN_COMPLETION_MARKER, checksum); + validateIndexComponent(indexDescriptor, indexIdentifier, IndexComponent.COLUMN_COMPLETION_MARKER, checksum); // ...then read to check if the index is empty try { - isEmptyIndex = ColumnCompletionMarkerUtil.isEmptyIndex(indexDescriptor, indexContext); + isEmptyIndex = ColumnCompletionMarkerUtil.isEmptyIndex(indexDescriptor, indexIdentifier); } catch (IOException e) { @@ -215,17 +217,17 @@ public class V1OnDiskFormat implements OnDiskFormat } } - for (IndexComponent indexComponent : perColumnIndexComponents(indexContext)) + for (IndexComponent indexComponent : perColumnIndexComponents(indexTermType)) { if (!isEmptyIndex && isNotBuildCompletionMarker(indexComponent)) { - validateIndexComponent(indexDescriptor, indexContext, indexComponent, checksum); + validateIndexComponent(indexDescriptor, indexIdentifier, indexComponent, checksum); } } } private static void validateIndexComponent(IndexDescriptor indexDescriptor, - IndexContext indexContext, + IndexIdentifier indexContext, IndexComponent indexComponent, boolean checksum) { @@ -264,9 +266,9 @@ public class V1OnDiskFormat implements OnDiskFormat } @Override - public Set perColumnIndexComponents(IndexContext indexContext) + public Set perColumnIndexComponents(IndexTermType indexTermType) { - return indexContext.isVector() ? VECTOR_COMPONENTS : indexContext.isLiteral() ? LITERAL_COMPONENTS : NUMERIC_COMPONENTS; + return indexTermType.isVector() ? VECTOR_COMPONENTS : indexTermType.isLiteral() ? LITERAL_COMPONENTS : NUMERIC_COMPONENTS; } @Override @@ -281,7 +283,7 @@ public class V1OnDiskFormat implements OnDiskFormat } @Override - public int openFilesPerColumnIndex(IndexContext indexContext) + public int openFilesPerColumnIndex() { // For the V1 format there are always 2 open files per index - index (balanced tree or terms) + auxiliary postings // for the balanced tree and postings for the literal terms diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/V1SSTableIndex.java b/src/java/org/apache/cassandra/index/sai/disk/v1/V1SSTableIndex.java index 98ee40d839..8813ab3154 100644 --- a/src/java/org/apache/cassandra/index/sai/disk/v1/V1SSTableIndex.java +++ b/src/java/org/apache/cassandra/index/sai/disk/v1/V1SSTableIndex.java @@ -30,9 +30,9 @@ import org.apache.cassandra.db.PartitionPosition; import org.apache.cassandra.db.virtual.SimpleDataSet; import org.apache.cassandra.dht.AbstractBounds; import org.apache.cassandra.dht.Token; -import org.apache.cassandra.index.sai.IndexContext; import org.apache.cassandra.index.sai.QueryContext; import org.apache.cassandra.index.sai.SSTableContext; +import org.apache.cassandra.index.sai.StorageAttachedIndex; import org.apache.cassandra.index.sai.disk.SSTableIndex; import org.apache.cassandra.index.sai.disk.v1.segment.Segment; import org.apache.cassandra.index.sai.disk.v1.segment.SegmentMetadata; @@ -40,7 +40,6 @@ import org.apache.cassandra.index.sai.iterators.KeyRangeIterator; import org.apache.cassandra.index.sai.iterators.KeyRangeUnionIterator; import org.apache.cassandra.index.sai.plan.Expression; import org.apache.cassandra.index.sai.utils.PrimaryKey; -import org.apache.cassandra.index.sai.utils.TypeUtil; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.io.util.FileUtils; import org.apache.cassandra.utils.Throwables; @@ -72,23 +71,23 @@ public class V1SSTableIndex extends SSTableIndex private PerColumnIndexFiles indexFiles; - public V1SSTableIndex(SSTableContext sstableContext, IndexContext indexContext) + public V1SSTableIndex(SSTableContext sstableContext, StorageAttachedIndex index) { - super(sstableContext, indexContext); + super(sstableContext, index); try { - this.indexFiles = new PerColumnIndexFiles(sstableContext.indexDescriptor, indexContext); + this.indexFiles = new PerColumnIndexFiles(sstableContext.indexDescriptor, indexTermType, indexIdentifier); ImmutableList.Builder segmentsBuilder = ImmutableList.builder(); - final MetadataSource source = MetadataSource.loadColumnMetadata(sstableContext.indexDescriptor, indexContext); + final MetadataSource source = MetadataSource.loadColumnMetadata(sstableContext.indexDescriptor, indexIdentifier); metadatas = SegmentMetadata.load(source, sstableContext.indexDescriptor.primaryKeyFactory); for (SegmentMetadata metadata : metadatas) { - segmentsBuilder.add(new Segment(indexContext, sstableContext, indexFiles, metadata)); + segmentsBuilder.add(new Segment(index, sstableContext, indexFiles, metadata)); } segments = segmentsBuilder.build(); @@ -99,8 +98,8 @@ public class V1SSTableIndex extends SSTableIndex this.bounds = AbstractBounds.bounds(minKey, true, maxKey, true); - this.minTerm = metadatas.stream().map(m -> m.minTerm).min(TypeUtil.comparator(indexContext.getValidator())).orElse(null); - this.maxTerm = metadatas.stream().map(m -> m.maxTerm).max(TypeUtil.comparator(indexContext.getValidator())).orElse(null); + this.minTerm = metadatas.stream().map(m -> m.minTerm).min(indexTermType.comparator()).orElse(null); + this.maxTerm = metadatas.stream().map(m -> m.maxTerm).max(indexTermType.comparator()).orElse(null); this.numRows = metadatas.stream().mapToLong(m -> m.numRows).sum(); @@ -193,16 +192,16 @@ public class V1SSTableIndex extends SSTableIndex for (SegmentMetadata metadata : metadatas) { - dataset.row(sstable.metadata().keyspace, indexContext.getIndexName(), sstable.getFilename(), metadata.rowIdOffset) + dataset.row(sstable.metadata().keyspace, indexIdentifier.indexName, sstable.getFilename(), metadata.rowIdOffset) .column(TABLE_NAME, sstable.descriptor.cfname) - .column(COLUMN_NAME, indexContext.getColumnName()) + .column(COLUMN_NAME, indexTermType.columnName()) .column(CELL_COUNT, metadata.numRows) .column(MIN_SSTABLE_ROW_ID, metadata.minSSTableRowId) .column(MAX_SSTABLE_ROW_ID, metadata.maxSSTableRowId) .column(START_TOKEN, tokenFactory.toString(metadata.minKey.token())) .column(END_TOKEN, tokenFactory.toString(metadata.maxKey.token())) - .column(MIN_TERM, indexContext.getValidator().getSerializer().deserialize(metadata.minTerm).toString()) - .column(MAX_TERM, indexContext.getValidator().getSerializer().deserialize(metadata.maxTerm).toString()) + .column(MIN_TERM, indexTermType.indexType().getSerializer().deserialize(metadata.minTerm).toString()) + .column(MAX_TERM, indexTermType.indexType().getSerializer().deserialize(metadata.maxTerm).toString()) .column(COMPONENT_METADATA, metadata.componentMetadatas.asMap()); } } diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/bbtree/BlockBalancedTreeIterator.java b/src/java/org/apache/cassandra/index/sai/disk/v1/bbtree/BlockBalancedTreeIterator.java index a650784754..cc53ee0a74 100644 --- a/src/java/org/apache/cassandra/index/sai/disk/v1/bbtree/BlockBalancedTreeIterator.java +++ b/src/java/org/apache/cassandra/index/sai/disk/v1/bbtree/BlockBalancedTreeIterator.java @@ -21,10 +21,9 @@ package org.apache.cassandra.index.sai.disk.v1.bbtree; import java.util.Iterator; import java.util.Map; -import org.apache.cassandra.db.marshal.AbstractType; import org.apache.cassandra.index.sai.postings.PostingList; +import org.apache.cassandra.index.sai.utils.IndexTermType; import org.apache.cassandra.index.sai.utils.TermsIterator; -import org.apache.cassandra.index.sai.utils.TypeUtil; import org.apache.cassandra.utils.Pair; import org.apache.cassandra.utils.bytecomparable.ByteComparable; import org.apache.cassandra.utils.bytecomparable.ByteSourceInverse; @@ -36,11 +35,11 @@ import org.apache.lucene.util.packed.PackedLongValues; */ public interface BlockBalancedTreeIterator extends Iterator> { - static BlockBalancedTreeIterator fromTermsIterator(final TermsIterator termsIterator, AbstractType termsComparator) + static BlockBalancedTreeIterator fromTermsIterator(final TermsIterator termsIterator, IndexTermType indexTermType) { return new BlockBalancedTreeIterator() { - final byte[] scratch = new byte[TypeUtil.fixedSizeOf(termsComparator)]; + final byte[] scratch = new byte[indexTermType.fixedSizeOf()]; @Override public boolean hasNext() diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/bbtree/BlockBalancedTreePostingsWriter.java b/src/java/org/apache/cassandra/index/sai/disk/v1/bbtree/BlockBalancedTreePostingsWriter.java index 6ba6894dc3..590528e782 100644 --- a/src/java/org/apache/cassandra/index/sai/disk/v1/bbtree/BlockBalancedTreePostingsWriter.java +++ b/src/java/org/apache/cassandra/index/sai/disk/v1/bbtree/BlockBalancedTreePostingsWriter.java @@ -29,7 +29,6 @@ import java.util.PriorityQueue; import java.util.TreeMap; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; - import javax.annotation.concurrent.NotThreadSafe; import com.google.common.base.Stopwatch; @@ -41,7 +40,7 @@ import org.slf4j.LoggerFactory; import org.agrona.collections.IntArrayList; import org.apache.cassandra.config.CassandraRelevantProperties; -import org.apache.cassandra.index.sai.IndexContext; +import org.apache.cassandra.index.sai.utils.IndexIdentifier; import org.apache.cassandra.index.sai.disk.io.IndexOutputWriter; import org.apache.cassandra.index.sai.disk.v1.postings.MergePostingList; import org.apache.cassandra.index.sai.disk.v1.postings.PackedLongsPostingList; @@ -127,7 +126,7 @@ public class BlockBalancedTreePostingsWriter implements BlockBalancedTreeWalker. * After writing out the postings, it writes a map of node ID -> postings file pointer for all * nodes with an attached postings list. It then returns the file pointer to this map. */ - public long finish(IndexOutputWriter out, List leafPostings, IndexContext indexContext) throws IOException + public long finish(IndexOutputWriter out, List leafPostings, IndexIdentifier indexIdentifier) throws IOException { checkState(leafPostings.size() == leafOffsetToNodeID.size(), "Expected equal number of postings lists (%s) and leaf offsets (%s).", @@ -150,7 +149,7 @@ public class BlockBalancedTreePostingsWriter implements BlockBalancedTreeWalker. Collection leafNodeIDs = leafOffsetToNodeID.values(); - logger.debug(indexContext.logMessage("Writing posting lists for {} internal and {} leaf balanced tree nodes. Leaf postings memory usage: {}."), + logger.debug(indexIdentifier.logMessage("Writing posting lists for {} internal and {} leaf balanced tree nodes. Leaf postings memory usage: {}."), internalNodeIDs.size(), leafNodeIDs.size(), FBUtilities.prettyPrintMemory(postingsRamBytesUsed)); long startFP = out.getFilePointer(); @@ -185,7 +184,7 @@ public class BlockBalancedTreePostingsWriter implements BlockBalancedTreeWalker. postingLists.clear(); } flushTime.stop(); - logger.debug(indexContext.logMessage("Flushed {} of posting lists for balanced tree nodes in {} ms."), + logger.debug(indexIdentifier.logMessage("Flushed {} of posting lists for balanced tree nodes in {} ms."), FBUtilities.prettyPrintMemory(out.getFilePointer() - startFP), flushTime.elapsed(TimeUnit.MILLISECONDS)); diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/bbtree/BlockBalancedTreeQueries.java b/src/java/org/apache/cassandra/index/sai/disk/v1/bbtree/BlockBalancedTreeQueries.java index 483fa5a99b..bb4b477c40 100644 --- a/src/java/org/apache/cassandra/index/sai/disk/v1/bbtree/BlockBalancedTreeQueries.java +++ b/src/java/org/apache/cassandra/index/sai/disk/v1/bbtree/BlockBalancedTreeQueries.java @@ -19,9 +19,8 @@ package org.apache.cassandra.index.sai.disk.v1.bbtree; import java.nio.ByteBuffer; -import org.apache.cassandra.db.marshal.AbstractType; import org.apache.cassandra.index.sai.plan.Expression; -import org.apache.cassandra.index.sai.utils.TypeUtil; +import org.apache.cassandra.index.sai.utils.IndexTermType; import org.apache.cassandra.utils.ByteArrayUtil; import org.apache.lucene.index.PointValues.Relation; @@ -44,33 +43,33 @@ public class BlockBalancedTreeQueries public static BlockBalancedTreeReader.IntersectVisitor balancedTreeQueryFrom(Expression expression, int bytesPerValue) { - if (expression.lower == null && expression.upper == null) + if (expression.lower() == null && expression.upper() == null) { return MATCH_ALL; } Bound lower = null ; - if (expression.lower != null) + if (expression.lower() != null) { - final byte[] lowerBound = toComparableBytes(bytesPerValue, expression.lower.value.encoded, expression.validator); - lower = new Bound(lowerBound, !expression.lower.inclusive); + final byte[] lowerBound = toComparableBytes(bytesPerValue, expression.lower().value.encoded, expression.getIndexTermType()); + lower = new Bound(lowerBound, !expression.lower().inclusive); } Bound upper = null; - if (expression.upper != null) + if (expression.upper() != null) { - final byte[] upperBound = toComparableBytes(bytesPerValue, expression.upper.value.encoded, expression.validator); - upper = new Bound(upperBound, !expression.upper.inclusive); + final byte[] upperBound = toComparableBytes(bytesPerValue, expression.upper().value.encoded, expression.getIndexTermType()); + upper = new Bound(upperBound, !expression.upper().inclusive); } return new RangeQueryVisitor(lower, upper); } - private static byte[] toComparableBytes(int bytesPerDim, ByteBuffer value, AbstractType type) + private static byte[] toComparableBytes(int bytesPerDim, ByteBuffer value, IndexTermType indexTermType) { - byte[] buffer = new byte[TypeUtil.fixedSizeOf(type)]; + byte[] buffer = new byte[indexTermType.fixedSizeOf()]; assert buffer.length == bytesPerDim; - TypeUtil.toComparableBytes(value, type, buffer); + indexTermType.toComparableBytes(value, buffer); return buffer; } diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/bbtree/BlockBalancedTreeReader.java b/src/java/org/apache/cassandra/index/sai/disk/v1/bbtree/BlockBalancedTreeReader.java index 39c313df54..0f14db390b 100644 --- a/src/java/org/apache/cassandra/index/sai/disk/v1/bbtree/BlockBalancedTreeReader.java +++ b/src/java/org/apache/cassandra/index/sai/disk/v1/bbtree/BlockBalancedTreeReader.java @@ -29,8 +29,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.cassandra.exceptions.QueryCancelledException; -import org.apache.cassandra.index.sai.IndexContext; import org.apache.cassandra.index.sai.QueryContext; +import org.apache.cassandra.index.sai.utils.IndexIdentifier; import org.apache.cassandra.index.sai.disk.io.IndexFileUtils; import org.apache.cassandra.index.sai.disk.io.SeekingRandomAccessInput; import org.apache.cassandra.index.sai.disk.v1.postings.FilteringPostingList; @@ -61,21 +61,21 @@ public class BlockBalancedTreeReader extends BlockBalancedTreeWalker implements private static final Comparator COMPARATOR = Comparator.comparingLong(PeekablePostingList::peek); - private final IndexContext indexContext; + private final IndexIdentifier indexIdentifier; private final FileHandle postingsFile; private final BlockBalancedTreePostingsIndex postingsIndex; private final int leafOrderMapBitsRequired; /** * Performs a blocking read. */ - public BlockBalancedTreeReader(IndexContext indexContext, + public BlockBalancedTreeReader(IndexIdentifier indexIdentifier, FileHandle treeIndexFile, long treeIndexRoot, FileHandle postingsFile, long treePostingsRoot) throws IOException { super(treeIndexFile, treeIndexRoot); - this.indexContext = indexContext; + this.indexIdentifier = indexIdentifier; this.postingsFile = postingsFile; this.postingsIndex = new BlockBalancedTreePostingsIndex(postingsFile, treePostingsRoot); leafOrderMapBitsRequired = DirectWriter.unsignedBitsRequired(maxValuesInLeafNode - 1); @@ -161,7 +161,7 @@ public class BlockBalancedTreeReader extends BlockBalancedTreeWalker implements catch (Throwable t) { if (!(t instanceof QueryCancelledException)) - logger.error(indexContext.logMessage("Balanced tree intersection failed on {}"), treeIndexFile.path(), t); + logger.error(indexIdentifier.logMessage("Balanced tree intersection failed on {}"), treeIndexFile.path(), t); closeOnException(); throw Throwables.cleaned(t); @@ -196,7 +196,7 @@ public class BlockBalancedTreeReader extends BlockBalancedTreeWalker implements else { if (logger.isTraceEnabled()) - logger.trace(indexContext.logMessage("[{}] Intersection completed in {} microseconds. {} leaf and internal posting lists hit."), + logger.trace(indexIdentifier.logMessage("[{}] Intersection completed in {} microseconds. {} leaf and internal posting lists hit."), treeIndexFile.path(), elapsedMicros, postingLists.size()); return MergePostingList.merge(postingLists, () -> FileUtils.close(postingsInput, postingsSummaryInput)); } @@ -215,7 +215,7 @@ public class BlockBalancedTreeReader extends BlockBalancedTreeWalker implements } if (state.atLeafNode()) - throw new CorruptIndexException(indexContext.logMessage(String.format("Leaf node %s does not have balanced tree postings.", state.nodeID)), ""); + throw new CorruptIndexException(indexIdentifier.logMessage(String.format("Leaf node %s does not have balanced tree postings.", state.nodeID)), ""); // Recurse on left subtree: state.pushLeft(); diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/bbtree/NumericIndexWriter.java b/src/java/org/apache/cassandra/index/sai/disk/v1/bbtree/NumericIndexWriter.java index 6ccc041956..b45ac3404a 100644 --- a/src/java/org/apache/cassandra/index/sai/disk/v1/bbtree/NumericIndexWriter.java +++ b/src/java/org/apache/cassandra/index/sai/disk/v1/bbtree/NumericIndexWriter.java @@ -26,9 +26,9 @@ import java.util.Map; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.MoreObjects; -import org.apache.cassandra.index.sai.IndexContext; import org.apache.cassandra.index.sai.disk.format.IndexComponent; import org.apache.cassandra.index.sai.disk.format.IndexDescriptor; +import org.apache.cassandra.index.sai.utils.IndexIdentifier; import org.apache.cassandra.index.sai.disk.io.IndexOutputWriter; import org.apache.cassandra.index.sai.disk.v1.segment.SegmentMetadata; import org.apache.lucene.store.IndexOutput; @@ -51,31 +51,31 @@ public class NumericIndexWriter private final BlockBalancedTreeWriter writer; private final IndexDescriptor indexDescriptor; - private final IndexContext indexContext; + private final IndexIdentifier indexIdentifier; private final int bytesPerValue; /** * @param maxSegmentRowId maximum possible segment row ID, used to create `maxRows` for the balanced tree */ public NumericIndexWriter(IndexDescriptor indexDescriptor, - IndexContext indexContext, + IndexIdentifier indexIdentifier, int bytesPerValue, long maxSegmentRowId) { - this(indexDescriptor, indexContext, MAX_POINTS_IN_LEAF_NODE, bytesPerValue, maxSegmentRowId); + this(indexDescriptor, indexIdentifier, MAX_POINTS_IN_LEAF_NODE, bytesPerValue, maxSegmentRowId); } @VisibleForTesting public NumericIndexWriter(IndexDescriptor indexDescriptor, - IndexContext indexContext, + IndexIdentifier indexIdentifier, int maxPointsInLeafNode, int bytesPerValue, long maxSegmentRowId) { - checkArgument(maxSegmentRowId >= 0, "[%s] maxSegmentRowId must be non-negative value, but got %s", indexContext.getIndexName(), maxSegmentRowId); + checkArgument(maxSegmentRowId >= 0, "[%s] maxSegmentRowId must be non-negative value, but got %s", indexIdentifier, maxSegmentRowId); this.indexDescriptor = indexDescriptor; - this.indexContext = indexContext; + this.indexIdentifier = indexIdentifier; this.bytesPerValue = bytesPerValue; this.writer = new BlockBalancedTreeWriter(bytesPerValue, maxPointsInLeafNode); } @@ -83,10 +83,7 @@ public class NumericIndexWriter @Override public String toString() { - return MoreObjects.toStringHelper(this) - .add("indexContext", indexContext) - .add("bytesPerValue", bytesPerValue) - .toString(); + return MoreObjects.toStringHelper(this).add("indexName", indexIdentifier).add("bytesPerValue", bytesPerValue).toString(); } private static class LeafCallback implements BlockBalancedTreeWriter.Callback @@ -126,7 +123,7 @@ public class NumericIndexWriter LeafCallback leafCallback = new LeafCallback(); - try (IndexOutput treeOutput = indexDescriptor.openPerIndexOutput(IndexComponent.BALANCED_TREE, indexContext, true)) + try (IndexOutput treeOutput = indexDescriptor.openPerIndexOutput(IndexComponent.BALANCED_TREE, indexIdentifier, true)) { // The SSTable balanced tree component file is opened in append mode, so our offset is the current file pointer. long treeOffset = treeOutput.getFilePointer(); @@ -148,8 +145,11 @@ public class NumericIndexWriter components.put(IndexComponent.BALANCED_TREE, treePosition, treeOffset, treeLength, attributes); } - try (BlockBalancedTreeWalker reader = new BlockBalancedTreeWalker(indexDescriptor.createPerIndexFileHandle(IndexComponent.BALANCED_TREE, indexContext, null), treePosition); - IndexOutputWriter postingsOutput = indexDescriptor.openPerIndexOutput(IndexComponent.POSTING_LISTS, indexContext, true)) + try (BlockBalancedTreeWalker reader = new BlockBalancedTreeWalker(indexDescriptor.createPerIndexFileHandle(IndexComponent.BALANCED_TREE, + indexIdentifier, + null), + treePosition); + IndexOutputWriter postingsOutput = indexDescriptor.openPerIndexOutput(IndexComponent.POSTING_LISTS, indexIdentifier, true)) { long postingsOffset = postingsOutput.getFilePointer(); @@ -157,7 +157,7 @@ public class NumericIndexWriter reader.traverse(postingsWriter); // The balanced tree postings writer already writes its own header & footer. - long postingsPosition = postingsWriter.finish(postingsOutput, leafCallback.leafPostings, indexContext); + long postingsPosition = postingsWriter.finish(postingsOutput, leafCallback.leafPostings, indexIdentifier); Map attributes = new LinkedHashMap<>(); attributes.put("num_leaf_postings", Integer.toString(postingsWriter.numLeafPostings)); diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/postings/PostingListRangeIterator.java b/src/java/org/apache/cassandra/index/sai/disk/v1/postings/PostingListRangeIterator.java index a1f77f3344..2a266718da 100644 --- a/src/java/org/apache/cassandra/index/sai/disk/v1/postings/PostingListRangeIterator.java +++ b/src/java/org/apache/cassandra/index/sai/disk/v1/postings/PostingListRangeIterator.java @@ -27,9 +27,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.cassandra.exceptions.QueryCancelledException; -import org.apache.cassandra.index.sai.IndexContext; import org.apache.cassandra.index.sai.QueryContext; import org.apache.cassandra.index.sai.disk.PrimaryKeyMap; +import org.apache.cassandra.index.sai.utils.IndexIdentifier; import org.apache.cassandra.index.sai.disk.v1.segment.IndexSegmentSearcherContext; import org.apache.cassandra.index.sai.iterators.KeyRangeIterator; import org.apache.cassandra.index.sai.postings.PostingList; @@ -62,7 +62,7 @@ public class PostingListRangeIterator extends KeyRangeIterator private final QueryContext queryContext; private final PostingList postingList; - private final IndexContext indexContext; + private final IndexIdentifier indexIdentifier; private final PrimaryKeyMap primaryKeyMap; private final long rowIdOffset; @@ -73,20 +73,20 @@ public class PostingListRangeIterator extends KeyRangeIterator * Create a direct PostingListRangeIterator where the underlying PostingList is materialised * immediately so the posting list size can be used. */ - public PostingListRangeIterator(IndexContext indexContext, + public PostingListRangeIterator(IndexIdentifier indexIdentifier, PrimaryKeyMap primaryKeyMap, IndexSegmentSearcherContext searcherContext) { super(searcherContext.minimumKey, searcherContext.maximumKey, searcherContext.count()); - this.indexContext = indexContext; + this.indexIdentifier = indexIdentifier; this.primaryKeyMap = primaryKeyMap; this.postingList = searcherContext.postingList; this.rowIdOffset = searcherContext.segmentRowIdOffset; this.queryContext = searcherContext.context; } - public PostingListRangeIterator(IndexContext indexContext, + public PostingListRangeIterator(IndexIdentifier indexIdentifier, PrimaryKeyMap primaryKeyMap, PostingList postingList, QueryContext queryContext) @@ -94,7 +94,7 @@ public class PostingListRangeIterator extends KeyRangeIterator super(primaryKeyMap.primaryKeyFromRowId(postingList.minimum()), primaryKeyMap.primaryKeyFromRowId(postingList.maximum()), postingList.size()); - this.indexContext = indexContext; + this.indexIdentifier = indexIdentifier; this.primaryKeyMap = primaryKeyMap; this.postingList = postingList; this.rowIdOffset = 0; @@ -131,7 +131,7 @@ public class PostingListRangeIterator extends KeyRangeIterator catch (Throwable t) { if (!(t instanceof QueryCancelledException)) - logger.error(indexContext.logMessage("Unable to provide next token!"), t); + logger.error(indexIdentifier.logMessage("Unable to provide next token!"), t); throw Throwables.cleaned(t); } @@ -143,7 +143,7 @@ public class PostingListRangeIterator extends KeyRangeIterator if (logger.isTraceEnabled()) { final long exhaustedInMills = timeToExhaust.stop().elapsed(TimeUnit.MILLISECONDS); - logger.trace(indexContext.logMessage("PostingListRangeIterator exhausted after {} ms"), exhaustedInMills); + logger.trace(indexIdentifier.logMessage("PostingListRangeIterator exhausted after {} ms"), exhaustedInMills); } FileUtils.closeQuietly(Arrays.asList(postingList, primaryKeyMap)); diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/postings/PostingsWriter.java b/src/java/org/apache/cassandra/index/sai/disk/v1/postings/PostingsWriter.java index d0cb5acbf2..545710c861 100644 --- a/src/java/org/apache/cassandra/index/sai/disk/v1/postings/PostingsWriter.java +++ b/src/java/org/apache/cassandra/index/sai/disk/v1/postings/PostingsWriter.java @@ -25,10 +25,10 @@ import javax.annotation.concurrent.NotThreadSafe; import com.google.common.annotations.VisibleForTesting; import org.agrona.collections.LongArrayList; -import org.apache.cassandra.index.sai.IndexContext; import org.apache.cassandra.index.sai.disk.ResettableByteBuffersIndexOutput; import org.apache.cassandra.index.sai.disk.format.IndexComponent; import org.apache.cassandra.index.sai.disk.format.IndexDescriptor; +import org.apache.cassandra.index.sai.utils.IndexIdentifier; import org.apache.cassandra.index.sai.disk.io.IndexOutputWriter; import org.apache.cassandra.index.sai.disk.v1.SAICodecUtils; import org.apache.cassandra.index.sai.postings.PostingList; @@ -105,9 +105,9 @@ public class PostingsWriter implements Closeable private long maxDelta; private long totalPostings; - public PostingsWriter(IndexDescriptor indexDescriptor, IndexContext indexContext) throws IOException + public PostingsWriter(IndexDescriptor indexDescriptor, IndexIdentifier indexIdentifier) throws IOException { - this(indexDescriptor, indexContext, BLOCK_SIZE); + this(indexDescriptor, indexIdentifier, BLOCK_SIZE); } public PostingsWriter(IndexOutputWriter dataOutput) throws IOException @@ -116,9 +116,9 @@ public class PostingsWriter implements Closeable } @VisibleForTesting - PostingsWriter(IndexDescriptor indexDescriptor, IndexContext indexContext, int blockSize) throws IOException + PostingsWriter(IndexDescriptor indexDescriptor, IndexIdentifier indexIdentifier, int blockSize) throws IOException { - this(indexDescriptor.openPerIndexOutput(IndexComponent.POSTING_LISTS, indexContext, true), blockSize); + this(indexDescriptor.openPerIndexOutput(IndexComponent.POSTING_LISTS, indexIdentifier, true), blockSize); } private PostingsWriter(IndexOutputWriter dataOutput, int blockSize) throws IOException diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/segment/IndexSegmentSearcher.java b/src/java/org/apache/cassandra/index/sai/disk/v1/segment/IndexSegmentSearcher.java index 36369728ec..96f389d794 100644 --- a/src/java/org/apache/cassandra/index/sai/disk/v1/segment/IndexSegmentSearcher.java +++ b/src/java/org/apache/cassandra/index/sai/disk/v1/segment/IndexSegmentSearcher.java @@ -22,8 +22,8 @@ import java.io.IOException; import org.apache.cassandra.db.PartitionPosition; import org.apache.cassandra.dht.AbstractBounds; -import org.apache.cassandra.index.sai.IndexContext; import org.apache.cassandra.index.sai.QueryContext; +import org.apache.cassandra.index.sai.StorageAttachedIndex; import org.apache.cassandra.index.sai.disk.PrimaryKeyMap; import org.apache.cassandra.index.sai.disk.v1.PerColumnIndexFiles; import org.apache.cassandra.index.sai.disk.v1.postings.PostingListRangeIterator; @@ -43,30 +43,30 @@ public abstract class IndexSegmentSearcher implements SegmentOrdering, Closeable final PrimaryKeyMap.Factory primaryKeyMapFactory; final PerColumnIndexFiles indexFiles; final SegmentMetadata metadata; - final IndexContext indexContext; + final StorageAttachedIndex index; IndexSegmentSearcher(PrimaryKeyMap.Factory primaryKeyMapFactory, PerColumnIndexFiles perIndexFiles, SegmentMetadata segmentMetadata, - IndexContext indexContext) + StorageAttachedIndex index) { this.primaryKeyMapFactory = primaryKeyMapFactory; this.indexFiles = perIndexFiles; this.metadata = segmentMetadata; - this.indexContext = indexContext; + this.index = index; } public static IndexSegmentSearcher open(PrimaryKeyMap.Factory primaryKeyMapFactory, PerColumnIndexFiles indexFiles, SegmentMetadata segmentMetadata, - IndexContext indexContext) throws IOException + StorageAttachedIndex index) throws IOException { - if (indexContext.isVector()) - return new VectorIndexSegmentSearcher(primaryKeyMapFactory, indexFiles, segmentMetadata, indexContext); - else if (indexContext.isLiteral()) - return new LiteralIndexSegmentSearcher(primaryKeyMapFactory, indexFiles, segmentMetadata, indexContext); + if (index.termType().isVector()) + return new VectorIndexSegmentSearcher(primaryKeyMapFactory, indexFiles, segmentMetadata, index); + else if (index.termType().isLiteral()) + return new LiteralIndexSegmentSearcher(primaryKeyMapFactory, indexFiles, segmentMetadata, index); else - return new NumericIndexSegmentSearcher(primaryKeyMapFactory, indexFiles, segmentMetadata, indexContext); + return new NumericIndexSegmentSearcher(primaryKeyMapFactory, indexFiles, segmentMetadata, index); } /** @@ -95,6 +95,6 @@ public abstract class IndexSegmentSearcher implements SegmentOrdering, Closeable queryContext, PeekablePostingList.makePeekable(postingList)); - return new PostingListRangeIterator(indexContext, primaryKeyMapFactory.newPerSSTablePrimaryKeyMap(), searcherContext); + return new PostingListRangeIterator(index.identifier(), primaryKeyMapFactory.newPerSSTablePrimaryKeyMap(), searcherContext); } } diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/segment/LiteralIndexSegmentSearcher.java b/src/java/org/apache/cassandra/index/sai/disk/v1/segment/LiteralIndexSegmentSearcher.java index 17ac4b43e0..4a79efecd9 100644 --- a/src/java/org/apache/cassandra/index/sai/disk/v1/segment/LiteralIndexSegmentSearcher.java +++ b/src/java/org/apache/cassandra/index/sai/disk/v1/segment/LiteralIndexSegmentSearcher.java @@ -27,8 +27,8 @@ import org.slf4j.LoggerFactory; import org.apache.cassandra.db.PartitionPosition; import org.apache.cassandra.dht.AbstractBounds; -import org.apache.cassandra.index.sai.IndexContext; import org.apache.cassandra.index.sai.QueryContext; +import org.apache.cassandra.index.sai.StorageAttachedIndex; import org.apache.cassandra.index.sai.disk.PrimaryKeyMap; import org.apache.cassandra.index.sai.disk.format.IndexComponent; import org.apache.cassandra.index.sai.disk.v1.PerColumnIndexFiles; @@ -52,23 +52,20 @@ public class LiteralIndexSegmentSearcher extends IndexSegmentSearcher LiteralIndexSegmentSearcher(PrimaryKeyMap.Factory primaryKeyMapFactory, PerColumnIndexFiles perIndexFiles, SegmentMetadata segmentMetadata, - IndexContext indexContext) throws IOException + StorageAttachedIndex index) throws IOException { - super(primaryKeyMapFactory, perIndexFiles, segmentMetadata, indexContext); + super(primaryKeyMapFactory, perIndexFiles, segmentMetadata, index); long root = metadata.getIndexRoot(IndexComponent.TERMS_DATA); assert root >= 0; - perColumnEventListener = (QueryEventListener.TrieIndexEventListener)indexContext.getColumnQueryMetrics(); + perColumnEventListener = (QueryEventListener.TrieIndexEventListener)index.columnQueryMetrics(); Map map = metadata.componentMetadatas.get(IndexComponent.TERMS_DATA).attributes; String footerPointerString = map.get(SAICodecUtils.FOOTER_POINTER); long footerPointer = footerPointerString == null ? -1 : Long.parseLong(footerPointerString); - reader = new LiteralIndexSegmentTermsReader(indexContext, - indexFiles.termsData(), - indexFiles.postingLists(), - root, footerPointer); + reader = new LiteralIndexSegmentTermsReader(index.identifier(), indexFiles.termsData(), indexFiles.postingLists(), root, footerPointer); } @Override @@ -82,12 +79,12 @@ public class LiteralIndexSegmentSearcher extends IndexSegmentSearcher public KeyRangeIterator search(Expression expression, AbstractBounds keyRange, QueryContext queryContext) throws IOException { if (logger.isTraceEnabled()) - logger.trace(indexContext.logMessage("Searching on expression '{}'..."), expression); + logger.trace(index.identifier().logMessage("Searching on expression '{}'..."), expression); - if (!expression.getOp().isEquality()) - throw new IllegalArgumentException(indexContext.logMessage("Unsupported expression: " + expression)); + if (!expression.getIndexOperator().isEquality()) + throw new IllegalArgumentException(index.identifier().logMessage("Unsupported expression: " + expression)); - final ByteComparable term = ByteComparable.fixedLength(expression.lower.value.encoded); + final ByteComparable term = ByteComparable.fixedLength(expression.lower().value.encoded); QueryEventListener.TrieIndexEventListener listener = MulticastQueryEventListeners.of(queryContext, perColumnEventListener); return toPrimaryKeyIterator(reader.exactMatch(term, listener, queryContext), queryContext); } @@ -95,9 +92,7 @@ public class LiteralIndexSegmentSearcher extends IndexSegmentSearcher @Override public String toString() { - return MoreObjects.toStringHelper(this) - .add("indexContext", indexContext) - .toString(); + return MoreObjects.toStringHelper(this).add("index", index).toString(); } @Override diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/segment/LiteralIndexSegmentTermsReader.java b/src/java/org/apache/cassandra/index/sai/disk/v1/segment/LiteralIndexSegmentTermsReader.java index abca7c95cb..6c6f81cddf 100644 --- a/src/java/org/apache/cassandra/index/sai/disk/v1/segment/LiteralIndexSegmentTermsReader.java +++ b/src/java/org/apache/cassandra/index/sai/disk/v1/segment/LiteralIndexSegmentTermsReader.java @@ -26,8 +26,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.cassandra.exceptions.QueryCancelledException; -import org.apache.cassandra.index.sai.IndexContext; import org.apache.cassandra.index.sai.QueryContext; +import org.apache.cassandra.index.sai.utils.IndexIdentifier; import org.apache.cassandra.index.sai.disk.io.IndexFileUtils; import org.apache.cassandra.index.sai.disk.v1.postings.PostingsReader; import org.apache.cassandra.index.sai.disk.v1.trie.TrieTermsDictionaryReader; @@ -57,18 +57,18 @@ public class LiteralIndexSegmentTermsReader implements Closeable { private static final Logger logger = LoggerFactory.getLogger(LiteralIndexSegmentTermsReader.class); - private final IndexContext indexContext; + private final IndexIdentifier indexIdentifier; private final FileHandle termDictionaryFile; private final FileHandle postingsFile; private final long termDictionaryRoot; - public LiteralIndexSegmentTermsReader(IndexContext indexContext, + public LiteralIndexSegmentTermsReader(IndexIdentifier indexIdentifier, FileHandle termsData, FileHandle postingLists, long root, long termsFooterPointer) throws IOException { - this.indexContext = indexContext; + this.indexIdentifier = indexIdentifier; termDictionaryFile = termsData; postingsFile = postingLists; termDictionaryRoot = root; @@ -137,7 +137,7 @@ public class LiteralIndexSegmentTermsReader implements Closeable catch (Throwable e) { if (!(e instanceof QueryCancelledException)) - logger.error(indexContext.logMessage("Failed to execute term query"), e); + logger.error(indexIdentifier.logMessage("Failed to execute term query"), e); closeOnException(); throw Throwables.cleaned(e); diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/segment/NumericIndexSegmentSearcher.java b/src/java/org/apache/cassandra/index/sai/disk/v1/segment/NumericIndexSegmentSearcher.java index fabf6408b0..137e635fd0 100644 --- a/src/java/org/apache/cassandra/index/sai/disk/v1/segment/NumericIndexSegmentSearcher.java +++ b/src/java/org/apache/cassandra/index/sai/disk/v1/segment/NumericIndexSegmentSearcher.java @@ -26,8 +26,8 @@ import org.slf4j.LoggerFactory; import org.apache.cassandra.db.PartitionPosition; import org.apache.cassandra.dht.AbstractBounds; -import org.apache.cassandra.index.sai.IndexContext; import org.apache.cassandra.index.sai.QueryContext; +import org.apache.cassandra.index.sai.StorageAttachedIndex; import org.apache.cassandra.index.sai.disk.PrimaryKeyMap; import org.apache.cassandra.index.sai.disk.format.IndexComponent; import org.apache.cassandra.index.sai.disk.v1.PerColumnIndexFiles; @@ -53,23 +53,23 @@ public class NumericIndexSegmentSearcher extends IndexSegmentSearcher NumericIndexSegmentSearcher(PrimaryKeyMap.Factory primaryKeyMapFactory, PerColumnIndexFiles perIndexFiles, SegmentMetadata segmentMetadata, - IndexContext indexContext) throws IOException + StorageAttachedIndex index) throws IOException { - super(primaryKeyMapFactory, perIndexFiles, segmentMetadata, indexContext); + super(primaryKeyMapFactory, perIndexFiles, segmentMetadata, index); final long treePosition = metadata.getIndexRoot(IndexComponent.BALANCED_TREE); if (treePosition < 0) - throw new CorruptIndexException(indexContext.logMessage("The tree position is less than zero."), IndexComponent.BALANCED_TREE.name); + throw new CorruptIndexException(index.identifier().logMessage("The tree position is less than zero."), IndexComponent.BALANCED_TREE.name); final long postingsPosition = metadata.getIndexRoot(IndexComponent.POSTING_LISTS); if (postingsPosition < 0) - throw new CorruptIndexException(indexContext.logMessage("The postings position is less than zero."), IndexComponent.BALANCED_TREE.name); + throw new CorruptIndexException(index.identifier().logMessage("The postings position is less than zero."), IndexComponent.BALANCED_TREE.name); - treeReader = new BlockBalancedTreeReader(indexContext, + treeReader = new BlockBalancedTreeReader(index.identifier(), indexFiles.balancedTree(), treePosition, indexFiles.postingLists(), postingsPosition); - perColumnEventListener = (QueryEventListener.BalancedTreeEventListener)indexContext.getColumnQueryMetrics(); + perColumnEventListener = (QueryEventListener.BalancedTreeEventListener)index.columnQueryMetrics(); } @Override @@ -82,9 +82,9 @@ public class NumericIndexSegmentSearcher extends IndexSegmentSearcher public KeyRangeIterator search(Expression exp, AbstractBounds keyRange, QueryContext context) throws IOException { if (logger.isTraceEnabled()) - logger.trace(indexContext.logMessage("Searching on expression '{}'..."), exp); + logger.trace(index.identifier().logMessage("Searching on expression '{}'..."), exp); - if (exp.getOp().isEqualityOrRange()) + if (exp.getIndexOperator().isEqualityOrRange()) { final BlockBalancedTreeReader.IntersectVisitor query = balancedTreeQueryFrom(exp, treeReader.getBytesPerValue()); QueryEventListener.BalancedTreeEventListener listener = MulticastQueryEventListeners.of(context, perColumnEventListener); @@ -92,7 +92,7 @@ public class NumericIndexSegmentSearcher extends IndexSegmentSearcher } else { - throw new IllegalArgumentException(indexContext.logMessage("Unsupported expression during index query: " + exp)); + throw new IllegalArgumentException(index.identifier().logMessage("Unsupported expression during index query: " + exp)); } } @@ -100,7 +100,7 @@ public class NumericIndexSegmentSearcher extends IndexSegmentSearcher public String toString() { return MoreObjects.toStringHelper(this) - .add("indexContext", indexContext) + .add("index", index) .add("count", treeReader.getPointCount()) .add("bytesPerValue", treeReader.getBytesPerValue()) .toString(); diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/segment/Segment.java b/src/java/org/apache/cassandra/index/sai/disk/v1/segment/Segment.java index 095a5b6245..6e08551bd1 100644 --- a/src/java/org/apache/cassandra/index/sai/disk/v1/segment/Segment.java +++ b/src/java/org/apache/cassandra/index/sai/disk/v1/segment/Segment.java @@ -27,13 +27,13 @@ import org.apache.cassandra.db.PartitionPosition; import org.apache.cassandra.dht.AbstractBounds; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; -import org.apache.cassandra.index.sai.IndexContext; import org.apache.cassandra.index.sai.QueryContext; import org.apache.cassandra.index.sai.SSTableContext; +import org.apache.cassandra.index.sai.StorageAttachedIndex; import org.apache.cassandra.index.sai.disk.PrimaryKeyMap; import org.apache.cassandra.index.sai.disk.v1.PerColumnIndexFiles; -import org.apache.cassandra.index.sai.plan.Expression; import org.apache.cassandra.index.sai.iterators.KeyRangeIterator; +import org.apache.cassandra.index.sai.plan.Expression; import org.apache.cassandra.index.sai.utils.PrimaryKey; import org.apache.cassandra.io.util.FileUtils; @@ -54,7 +54,7 @@ public class Segment implements SegmentOrdering, Closeable private final IndexSegmentSearcher index; - public Segment(IndexContext indexContext, SSTableContext sstableContext, PerColumnIndexFiles indexFiles, SegmentMetadata metadata) throws IOException + public Segment(StorageAttachedIndex index, SSTableContext sstableContext, PerColumnIndexFiles indexFiles, SegmentMetadata metadata) throws IOException { this.minKeyBound = metadata.minKey.token().minKeyBound(); this.maxKeyBound = metadata.maxKey.token().maxKeyBound(); @@ -62,7 +62,7 @@ public class Segment implements SegmentOrdering, Closeable this.primaryKeyMapFactory = sstableContext.primaryKeyMapFactory; this.metadata = metadata; - this.index = IndexSegmentSearcher.open(primaryKeyMapFactory, indexFiles, metadata, indexContext); + this.index = IndexSegmentSearcher.open(primaryKeyMapFactory, indexFiles, metadata, index); } @VisibleForTesting diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/segment/SegmentBuilder.java b/src/java/org/apache/cassandra/index/sai/disk/v1/segment/SegmentBuilder.java index 98af217a65..2d95219be5 100644 --- a/src/java/org/apache/cassandra/index/sai/disk/v1/segment/SegmentBuilder.java +++ b/src/java/org/apache/cassandra/index/sai/disk/v1/segment/SegmentBuilder.java @@ -26,8 +26,6 @@ import com.google.common.annotations.VisibleForTesting; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.cassandra.db.marshal.AbstractType; -import org.apache.cassandra.index.sai.IndexContext; import org.apache.cassandra.index.sai.disk.format.IndexDescriptor; import org.apache.cassandra.index.sai.disk.v1.IndexWriterConfig; import org.apache.cassandra.index.sai.disk.v1.bbtree.BlockBalancedTreeRamBuffer; @@ -35,9 +33,10 @@ import org.apache.cassandra.index.sai.disk.v1.bbtree.NumericIndexWriter; import org.apache.cassandra.index.sai.disk.v1.trie.LiteralIndexWriter; import org.apache.cassandra.index.sai.disk.v1.vector.OnHeapGraph; import org.apache.cassandra.index.sai.memory.RAMStringIndexer; +import org.apache.cassandra.index.sai.utils.IndexIdentifier; +import org.apache.cassandra.index.sai.utils.IndexTermType; import org.apache.cassandra.index.sai.utils.NamedMemoryLimiter; import org.apache.cassandra.index.sai.utils.PrimaryKey; -import org.apache.cassandra.index.sai.utils.TypeUtil; import org.apache.cassandra.utils.FastByteOperations; import org.apache.lucene.util.BytesRefBuilder; @@ -74,7 +73,7 @@ public abstract class SegmentBuilder private ByteBuffer minTerm; private ByteBuffer maxTerm; - final AbstractType termComparator; + final IndexTermType indexTermType; long totalBytesAllocated; int rowCount = 0; int maxSegmentRowId = -1; @@ -84,12 +83,12 @@ public abstract class SegmentBuilder private final byte[] scratch; private final BlockBalancedTreeRamBuffer trieBuffer; - public BlockBalancedTreeSegmentBuilder(AbstractType termComparator, NamedMemoryLimiter limiter) + public BlockBalancedTreeSegmentBuilder(IndexTermType indexTermType, NamedMemoryLimiter limiter) { - super(termComparator, limiter); + super(indexTermType, limiter); - scratch = new byte[TypeUtil.fixedSizeOf(termComparator)]; - trieBuffer = new BlockBalancedTreeRamBuffer(TypeUtil.fixedSizeOf(termComparator)); + scratch = new byte[indexTermType.fixedSizeOf()]; + trieBuffer = new BlockBalancedTreeRamBuffer(indexTermType.fixedSizeOf()); totalBytesAllocated = this.trieBuffer.memoryUsed(); } @@ -102,16 +101,16 @@ public abstract class SegmentBuilder @Override protected long addInternal(ByteBuffer term, int segmentRowId) { - TypeUtil.toComparableBytes(term, termComparator, scratch); + indexTermType.toComparableBytes(term, scratch); return trieBuffer.add(segmentRowId, scratch); } @Override - protected SegmentMetadata.ComponentMetadataMap flushInternal(IndexDescriptor indexDescriptor, IndexContext indexContext) throws IOException + protected SegmentMetadata.ComponentMetadataMap flushInternal(IndexDescriptor indexDescriptor, IndexIdentifier indexIdentifier) throws IOException { NumericIndexWriter writer = new NumericIndexWriter(indexDescriptor, - indexContext, - TypeUtil.fixedSizeOf(termComparator), + indexIdentifier, + indexTermType.fixedSizeOf(), maxSegmentRowId); return writer.writeCompleteSegment(trieBuffer.iterator()); } @@ -123,9 +122,9 @@ public abstract class SegmentBuilder final BytesRefBuilder stringBuffer = new BytesRefBuilder(); - public RAMStringSegmentBuilder(AbstractType termComparator, NamedMemoryLimiter limiter) + public RAMStringSegmentBuilder(IndexTermType indexTermType, NamedMemoryLimiter limiter) { - super(termComparator, limiter); + super(indexTermType, limiter); ramIndexer = new RAMStringIndexer(); totalBytesAllocated = ramIndexer.estimatedBytesUsed(); @@ -145,9 +144,9 @@ public abstract class SegmentBuilder } @Override - protected SegmentMetadata.ComponentMetadataMap flushInternal(IndexDescriptor indexDescriptor, IndexContext indexContext) throws IOException + protected SegmentMetadata.ComponentMetadataMap flushInternal(IndexDescriptor indexDescriptor, IndexIdentifier indexIdentifier) throws IOException { - try (LiteralIndexWriter writer = new LiteralIndexWriter(indexDescriptor, indexContext)) + try (LiteralIndexWriter writer = new LiteralIndexWriter(indexDescriptor, indexIdentifier)) { return writer.writeCompleteSegment(ramIndexer.getTermsWithPostings()); } @@ -167,10 +166,10 @@ public abstract class SegmentBuilder { private final OnHeapGraph graphIndex; - public VectorSegmentBuilder(AbstractType termComparator, NamedMemoryLimiter limiter, IndexWriterConfig indexWriterConfig) + public VectorSegmentBuilder(IndexTermType indexTermType, NamedMemoryLimiter limiter, IndexWriterConfig indexWriterConfig) { - super(termComparator, limiter); - graphIndex = new OnHeapGraph<>(termComparator, indexWriterConfig, false); + super(indexTermType, limiter); + graphIndex = new OnHeapGraph<>(indexTermType.indexType(), indexWriterConfig, false); } @Override @@ -186,9 +185,9 @@ public abstract class SegmentBuilder } @Override - protected SegmentMetadata.ComponentMetadataMap flushInternal(IndexDescriptor indexDescriptor, IndexContext indexContext) throws IOException + protected SegmentMetadata.ComponentMetadataMap flushInternal(IndexDescriptor indexDescriptor, IndexIdentifier indexIdentifier) throws IOException { - return graphIndex.writeData(indexDescriptor, indexContext, p -> p); + return graphIndex.writeData(indexDescriptor, indexIdentifier, p -> p); } } @@ -197,27 +196,27 @@ public abstract class SegmentBuilder return ACTIVE_BUILDER_COUNT.get(); } - private SegmentBuilder(AbstractType termComparator, NamedMemoryLimiter limiter) + private SegmentBuilder(IndexTermType indexTermType, NamedMemoryLimiter limiter) { - this.termComparator = termComparator; + this.indexTermType = indexTermType; this.limiter = limiter; lastValidSegmentRowID = testLastValidSegmentRowId >= 0 ? testLastValidSegmentRowId : LAST_VALID_SEGMENT_ROW_ID; minimumFlushBytes = limiter.limitBytes() / ACTIVE_BUILDER_COUNT.incrementAndGet(); } - public SegmentMetadata flush(IndexDescriptor indexDescriptor, IndexContext indexContext) throws IOException + public SegmentMetadata flush(IndexDescriptor indexDescriptor, IndexIdentifier indexIdentifier) throws IOException { assert !flushed : "Cannot flush an already flushed segment"; flushed = true; if (getRowCount() == 0) { - logger.warn(indexContext.logMessage("No rows to index during flush of SSTable {}."), indexDescriptor.sstableDescriptor); + logger.warn(indexIdentifier.logMessage("No rows to index during flush of SSTable {}."), indexDescriptor.sstableDescriptor); return null; } - SegmentMetadata.ComponentMetadataMap indexMetas = flushInternal(indexDescriptor, indexContext); + SegmentMetadata.ComponentMetadataMap indexMetas = flushInternal(indexDescriptor, indexIdentifier); return new SegmentMetadata(segmentRowIdOffset, rowCount, minSSTableRowId, maxSSTableRowId, minKey, maxKey, minTerm, maxTerm, indexMetas); } @@ -234,8 +233,8 @@ public abstract class SegmentBuilder minKey = key; maxKey = key; - minTerm = TypeUtil.min(term, minTerm, termComparator); - maxTerm = TypeUtil.max(term, maxTerm, termComparator); + minTerm = indexTermType.min(term, minTerm); + maxTerm = indexTermType.max(term, maxTerm); if (rowCount == 0) { @@ -282,11 +281,11 @@ public abstract class SegmentBuilder * 2. It releases the builder's memory against its limiter. * 3. It defensively marks the builder inactive to make sure nothing bad happens if we try to close it twice. * - * @param indexContext an {@link IndexContext} used for creating logging messages + * @param indexIdentifier an {@link IndexIdentifier} used for creating logging messages * * @return the number of bytes used by the memory limiter after releasing this builder */ - public long release(IndexContext indexContext) + public long release(IndexIdentifier indexIdentifier) { if (active) { @@ -296,7 +295,7 @@ public abstract class SegmentBuilder return used; } - logger.warn(indexContext.logMessage("Attempted to release storage-attached index segment builder memory after builder marked inactive.")); + logger.warn(indexIdentifier.logMessage("Attempted to release storage-attached index segment builder memory after builder marked inactive.")); return limiter.currentBytesUsed(); } @@ -304,7 +303,7 @@ public abstract class SegmentBuilder protected abstract long addInternal(ByteBuffer term, int segmentRowId); - protected abstract SegmentMetadata.ComponentMetadataMap flushInternal(IndexDescriptor indexDescriptor, IndexContext indexContext) throws IOException; + protected abstract SegmentMetadata.ComponentMetadataMap flushInternal(IndexDescriptor indexDescriptor, IndexIdentifier indexIdentifier) throws IOException; public int getRowCount() { diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/segment/VectorIndexSegmentSearcher.java b/src/java/org/apache/cassandra/index/sai/disk/v1/segment/VectorIndexSegmentSearcher.java index 13e829bda4..dc6b8fa706 100644 --- a/src/java/org/apache/cassandra/index/sai/disk/v1/segment/VectorIndexSegmentSearcher.java +++ b/src/java/org/apache/cassandra/index/sai/disk/v1/segment/VectorIndexSegmentSearcher.java @@ -33,8 +33,8 @@ import io.github.jbellis.jvector.util.SparseFixedBitSet; import org.agrona.collections.IntArrayList; import org.apache.cassandra.db.PartitionPosition; import org.apache.cassandra.dht.AbstractBounds; -import org.apache.cassandra.index.sai.IndexContext; import org.apache.cassandra.index.sai.QueryContext; +import org.apache.cassandra.index.sai.StorageAttachedIndex; import org.apache.cassandra.index.sai.VectorQueryContext; import org.apache.cassandra.index.sai.disk.PrimaryKeyMap; import org.apache.cassandra.index.sai.disk.v1.PerColumnIndexFiles; @@ -49,7 +49,6 @@ import org.apache.cassandra.index.sai.postings.PostingList; import org.apache.cassandra.index.sai.utils.AtomicRatio; import org.apache.cassandra.index.sai.utils.PrimaryKey; import org.apache.cassandra.index.sai.utils.RangeUtil; -import org.apache.cassandra.index.sai.utils.TypeUtil; import org.apache.cassandra.tracing.Tracing; import static java.lang.Math.max; @@ -71,13 +70,13 @@ public class VectorIndexSegmentSearcher extends IndexSegmentSearcher VectorIndexSegmentSearcher(PrimaryKeyMap.Factory primaryKeyMapFactory, PerColumnIndexFiles perIndexFiles, SegmentMetadata segmentMetadata, - IndexContext indexContext) throws IOException + StorageAttachedIndex index) throws IOException { - super(primaryKeyMapFactory, perIndexFiles, segmentMetadata, indexContext); - graph = new DiskAnn(segmentMetadata.componentMetadatas, perIndexFiles, indexContext); + super(primaryKeyMapFactory, perIndexFiles, segmentMetadata, index); + graph = new DiskAnn(segmentMetadata.componentMetadatas, perIndexFiles, index.indexWriterConfig()); cachedBitSets = ThreadLocal.withInitial(() -> new SparseFixedBitSet(graph.size())); globalBruteForceRows = Integer.MAX_VALUE; - optimizeFor = indexContext.getIndexWriterConfig().getOptimizeFor(); + optimizeFor = index.indexWriterConfig().getOptimizeFor(); } @Override @@ -92,17 +91,17 @@ public class VectorIndexSegmentSearcher extends IndexSegmentSearcher int limit = context.vectorContext().limit(); if (logger.isTraceEnabled()) - logger.trace(indexContext.logMessage("Searching on expression '{}'..."), exp); + logger.trace(index.identifier().logMessage("Searching on expression '{}'..."), exp); - if (exp.getOp() != Expression.IndexOperator.ANN) - throw new IllegalArgumentException(indexContext.logMessage("Unsupported expression during ANN index query: " + exp)); + if (exp.getIndexOperator() != Expression.IndexOperator.ANN) + throw new IllegalArgumentException(index.identifier().logMessage("Unsupported expression during ANN index query: " + exp)); int topK = optimizeFor.topKFor(limit); BitsOrPostingList bitsOrPostingList = bitsOrPostingListForKeyRange(context.vectorContext(), keyRange, topK); if (bitsOrPostingList.skipANN()) return toPrimaryKeyIterator(bitsOrPostingList.postingList(), context); - float[] queryVector = TypeUtil.decomposeVector(indexContext, exp.lower.value.raw.duplicate()); + float[] queryVector = index.termType().decomposeVector(exp.lower().value.raw.duplicate()); var vectorPostings = graph.search(queryVector, topK, limit, bitsOrPostingList.getBits()); if (bitsOrPostingList.expectedNodesVisited >= 0) updateExpectedNodes(vectorPostings.getVisitedCount(), bitsOrPostingList.expectedNodesVisited); @@ -258,7 +257,7 @@ public class VectorIndexSegmentSearcher extends IndexSegmentSearcher return toPrimaryKeyIterator(new IntArrayPostingList(rowIds.toIntArray()), context); // else ask the index to perform a search limited to the bits we created - float[] queryVector = TypeUtil.decomposeVector(indexContext, expression.lower.value.raw.duplicate()); + float[] queryVector = index.termType().decomposeVector(expression.lower().value.raw.duplicate()); var results = graph.search(queryVector, topK, limit, bits); updateExpectedNodes(results.getVisitedCount(), expectedNodesVisited(topK, maxSegmentRowId, graph.size())); return toPrimaryKeyIterator(results, context); @@ -304,9 +303,7 @@ public class VectorIndexSegmentSearcher extends IndexSegmentSearcher @Override public String toString() { - return MoreObjects.toStringHelper(this) - .add("indexContext", indexContext) - .toString(); + return MoreObjects.toStringHelper(this).add("index", index).toString(); } @Override diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/trie/LiteralIndexWriter.java b/src/java/org/apache/cassandra/index/sai/disk/v1/trie/LiteralIndexWriter.java index e52a05e170..d05f006d49 100644 --- a/src/java/org/apache/cassandra/index/sai/disk/v1/trie/LiteralIndexWriter.java +++ b/src/java/org/apache/cassandra/index/sai/disk/v1/trie/LiteralIndexWriter.java @@ -25,14 +25,14 @@ import javax.annotation.concurrent.NotThreadSafe; import org.apache.commons.lang3.mutable.MutableLong; -import org.apache.cassandra.index.sai.IndexContext; -import org.apache.cassandra.index.sai.postings.PostingList; -import org.apache.cassandra.index.sai.utils.TermsIterator; import org.apache.cassandra.index.sai.disk.format.IndexComponent; import org.apache.cassandra.index.sai.disk.format.IndexDescriptor; -import org.apache.cassandra.index.sai.disk.v1.segment.SegmentMetadata; -import org.apache.cassandra.index.sai.disk.v1.postings.PostingsWriter; +import org.apache.cassandra.index.sai.utils.IndexIdentifier; import org.apache.cassandra.index.sai.disk.v1.SAICodecUtils; +import org.apache.cassandra.index.sai.disk.v1.postings.PostingsWriter; +import org.apache.cassandra.index.sai.disk.v1.segment.SegmentMetadata; +import org.apache.cassandra.index.sai.postings.PostingList; +import org.apache.cassandra.index.sai.utils.TermsIterator; import org.apache.cassandra.utils.bytecomparable.ByteComparable; /** @@ -45,10 +45,10 @@ public class LiteralIndexWriter implements Closeable private final PostingsWriter postingsWriter; private long postingsAdded; - public LiteralIndexWriter(IndexDescriptor indexDescriptor, IndexContext indexContext) throws IOException + public LiteralIndexWriter(IndexDescriptor indexDescriptor, IndexIdentifier indexIdentifier) throws IOException { - this.termsDictionaryWriter = new TrieTermsDictionaryWriter(indexDescriptor, indexContext); - this.postingsWriter = new PostingsWriter(indexDescriptor, indexContext); + this.termsDictionaryWriter = new TrieTermsDictionaryWriter(indexDescriptor, indexIdentifier); + this.postingsWriter = new PostingsWriter(indexDescriptor, indexIdentifier); } /** diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/trie/TrieTermsDictionaryWriter.java b/src/java/org/apache/cassandra/index/sai/disk/v1/trie/TrieTermsDictionaryWriter.java index 373e695e92..df6b278a06 100644 --- a/src/java/org/apache/cassandra/index/sai/disk/v1/trie/TrieTermsDictionaryWriter.java +++ b/src/java/org/apache/cassandra/index/sai/disk/v1/trie/TrieTermsDictionaryWriter.java @@ -23,9 +23,9 @@ import javax.annotation.concurrent.NotThreadSafe; import org.apache.commons.lang3.mutable.MutableLong; -import org.apache.cassandra.index.sai.IndexContext; import org.apache.cassandra.index.sai.disk.format.IndexComponent; import org.apache.cassandra.index.sai.disk.format.IndexDescriptor; +import org.apache.cassandra.index.sai.utils.IndexIdentifier; import org.apache.cassandra.index.sai.disk.io.IndexOutputWriter; import org.apache.cassandra.index.sai.disk.v1.SAICodecUtils; import org.apache.cassandra.io.tries.IncrementalDeepTrieWriterPageAware; @@ -34,7 +34,7 @@ import org.apache.cassandra.utils.bytecomparable.ByteComparable; /** * Writes terms dictionary to disk in a trie format (see {@link IncrementalTrieWriter}). - * + *

    * Allows for variable-length keys. Trie values are 64-bit offsets to the posting file, pointing to the beginning of * summary block for that postings list. */ @@ -45,9 +45,9 @@ public class TrieTermsDictionaryWriter implements Closeable private final IndexOutputWriter termDictionaryOutput; private final long startOffset; - TrieTermsDictionaryWriter(IndexDescriptor indexDescriptor, IndexContext indexContext) throws IOException + TrieTermsDictionaryWriter(IndexDescriptor indexDescriptor, IndexIdentifier indexIdentifier) throws IOException { - termDictionaryOutput = indexDescriptor.openPerIndexOutput(IndexComponent.TERMS_DATA, indexContext, true); + termDictionaryOutput = indexDescriptor.openPerIndexOutput(IndexComponent.TERMS_DATA, indexIdentifier, true); startOffset = termDictionaryOutput.getFilePointer(); SAICodecUtils.writeHeader(termDictionaryOutput); diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/vector/CheckpointingIterator.java b/src/java/org/apache/cassandra/index/sai/disk/v1/vector/CheckpointingIterator.java deleted file mode 100644 index 154009d445..0000000000 --- a/src/java/org/apache/cassandra/index/sai/disk/v1/vector/CheckpointingIterator.java +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.index.sai.disk.v1.vector; - -import com.google.common.collect.Iterables; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import org.apache.cassandra.index.sai.QueryContext; -import org.apache.cassandra.index.sai.disk.SSTableIndex; -import org.apache.cassandra.index.sai.iterators.KeyRangeIterator; -import org.apache.cassandra.index.sai.utils.PrimaryKey; -import org.apache.cassandra.io.util.FileUtils; - -/** - * A {@link KeyRangeIterator}, used when returning results from a vector search, that checks for query timeouts - * during the iteration process. - */ -public class CheckpointingIterator extends KeyRangeIterator -{ - private static final Logger logger = LoggerFactory.getLogger(CheckpointingIterator.class); - - private final QueryContext context; - private final KeyRangeIterator union; - private final Iterable referencedIndexes; - - public CheckpointingIterator(KeyRangeIterator wrapped, - Iterable referencedIndexes, - Iterable referencedAnnIndexesInHybridSearch, - QueryContext queryContext) - { - super(wrapped.getMinimum(), wrapped.getMaximum(), wrapped.getCount()); - - this.union = wrapped; - if (referencedAnnIndexesInHybridSearch != null) - this.referencedIndexes = Iterables.concat(referencedIndexes, referencedAnnIndexesInHybridSearch); - else - this.referencedIndexes = referencedIndexes; - this.context = queryContext; - } - - @Override - protected PrimaryKey computeNext() - { - try - { - return union.hasNext() ? union.next() : endOfData(); - } - finally - { - context.checkpoint(); - } - } - - @Override - protected void performSkipTo(PrimaryKey nextKey) - { - try - { - union.skipTo(nextKey); - } - finally - { - context.checkpoint(); - } - } - - @Override - public void close() - { - FileUtils.closeQuietly(union); - referencedIndexes.forEach(CheckpointingIterator::releaseQuietly); - } - - private static void releaseQuietly(SSTableIndex index) - { - try - { - index.release(); - } - catch (Throwable e) - { - logger.error(index.getIndexContext().logMessage(String.format("Failed to release index on SSTable %s", index.getSSTable())), e); - } - } -} diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/vector/DiskAnn.java b/src/java/org/apache/cassandra/index/sai/disk/v1/vector/DiskAnn.java index 4a9247a788..93f50df54f 100644 --- a/src/java/org/apache/cassandra/index/sai/disk/v1/vector/DiskAnn.java +++ b/src/java/org/apache/cassandra/index/sai/disk/v1/vector/DiskAnn.java @@ -34,8 +34,8 @@ import io.github.jbellis.jvector.graph.SearchResult.NodeScore; import io.github.jbellis.jvector.pq.CompressedVectors; import io.github.jbellis.jvector.util.Bits; import io.github.jbellis.jvector.vector.VectorSimilarityFunction; -import org.apache.cassandra.index.sai.IndexContext; import org.apache.cassandra.index.sai.disk.format.IndexComponent; +import org.apache.cassandra.index.sai.disk.v1.IndexWriterConfig; import org.apache.cassandra.index.sai.disk.v1.PerColumnIndexFiles; import org.apache.cassandra.index.sai.disk.v1.postings.VectorPostingList; import org.apache.cassandra.index.sai.disk.v1.segment.SegmentMetadata; @@ -52,9 +52,9 @@ public class DiskAnn implements AutoCloseable // only one of these will be not null private final CompressedVectors compressedVectors; - public DiskAnn(SegmentMetadata.ComponentMetadataMap componentMetadatas, PerColumnIndexFiles indexFiles, IndexContext context) throws IOException + public DiskAnn(SegmentMetadata.ComponentMetadataMap componentMetadatas, PerColumnIndexFiles indexFiles, IndexWriterConfig config) throws IOException { - similarityFunction = context.getIndexWriterConfig().getSimilarityFunction(); + similarityFunction = config.getSimilarityFunction(); SegmentMetadata.ComponentMetadata termsMetadata = componentMetadatas.get(IndexComponent.TERMS_DATA); graphHandle = indexFiles.termsData(); diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/vector/OnHeapGraph.java b/src/java/org/apache/cassandra/index/sai/disk/v1/vector/OnHeapGraph.java index 06aecf0751..9030581083 100644 --- a/src/java/org/apache/cassandra/index/sai/disk/v1/vector/OnHeapGraph.java +++ b/src/java/org/apache/cassandra/index/sai/disk/v1/vector/OnHeapGraph.java @@ -49,9 +49,9 @@ import io.github.jbellis.jvector.vector.VectorSimilarityFunction; import org.apache.cassandra.db.marshal.AbstractType; import org.apache.cassandra.db.marshal.VectorType; import org.apache.cassandra.exceptions.InvalidRequestException; -import org.apache.cassandra.index.sai.IndexContext; import org.apache.cassandra.index.sai.disk.format.IndexComponent; import org.apache.cassandra.index.sai.disk.format.IndexDescriptor; +import org.apache.cassandra.index.sai.utils.IndexIdentifier; import org.apache.cassandra.index.sai.disk.io.IndexFileUtils; import org.apache.cassandra.index.sai.disk.v1.IndexWriterConfig; import org.apache.cassandra.index.sai.disk.v1.SAICodecUtils; @@ -280,7 +280,7 @@ public class OnHeapGraph return keyQueue; } - public SegmentMetadata.ComponentMetadataMap writeData(IndexDescriptor indexDescriptor, IndexContext indexContext, Function postingTransformer) throws IOException + public SegmentMetadata.ComponentMetadataMap writeData(IndexDescriptor indexDescriptor, IndexIdentifier indexIdentifier, Function postingTransformer) throws IOException { int nInProgress = builder.insertsInProgress(); assert nInProgress == 0 : String.format("Attempting to write graph while %d inserts are in progress", nInProgress); @@ -292,9 +292,9 @@ public class OnHeapGraph postingsMap.keySet().size(), vectorValues.size()); logger.debug("Writing graph with {} rows and {} distinct vectors", postingsMap.values().stream().mapToInt(VectorPostings::size).sum(), vectorValues.size()); - try (var pqOutput = IndexFileUtils.instance.openOutput(indexDescriptor.fileFor(IndexComponent.COMPRESSED_VECTORS, indexContext), true); - var postingsOutput = IndexFileUtils.instance.openOutput(indexDescriptor.fileFor(IndexComponent.POSTING_LISTS, indexContext), true); - var indexOutput = IndexFileUtils.instance.openOutput(indexDescriptor.fileFor(IndexComponent.TERMS_DATA, indexContext), true)) + try (var pqOutput = IndexFileUtils.instance.openOutput(indexDescriptor.fileFor(IndexComponent.COMPRESSED_VECTORS, indexIdentifier), true); + var postingsOutput = IndexFileUtils.instance.openOutput(indexDescriptor.fileFor(IndexComponent.POSTING_LISTS, indexIdentifier), true); + var indexOutput = IndexFileUtils.instance.openOutput(indexDescriptor.fileFor(IndexComponent.TERMS_DATA, indexIdentifier), true)) { SAICodecUtils.writeHeader(pqOutput); SAICodecUtils.writeHeader(postingsOutput); diff --git a/src/java/org/apache/cassandra/index/sai/memory/MemoryIndex.java b/src/java/org/apache/cassandra/index/sai/memory/MemoryIndex.java index 3e7662707d..4307727b88 100644 --- a/src/java/org/apache/cassandra/index/sai/memory/MemoryIndex.java +++ b/src/java/org/apache/cassandra/index/sai/memory/MemoryIndex.java @@ -22,9 +22,10 @@ import org.apache.cassandra.db.Clustering; import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.db.PartitionPosition; import org.apache.cassandra.dht.AbstractBounds; -import org.apache.cassandra.index.sai.IndexContext; import org.apache.cassandra.index.sai.QueryContext; +import org.apache.cassandra.index.sai.StorageAttachedIndex; import org.apache.cassandra.index.sai.disk.format.IndexDescriptor; +import org.apache.cassandra.index.sai.utils.IndexIdentifier; import org.apache.cassandra.index.sai.disk.v1.segment.SegmentMetadata; import org.apache.cassandra.index.sai.iterators.KeyRangeIterator; import org.apache.cassandra.index.sai.plan.Expression; @@ -40,11 +41,11 @@ import java.util.function.Function; public abstract class MemoryIndex implements MemtableOrdering { - protected final IndexContext indexContext; + protected final StorageAttachedIndex index; - protected MemoryIndex(IndexContext indexContext) + protected MemoryIndex(StorageAttachedIndex index) { - this.indexContext = indexContext; + this.index = index; } public abstract long add(DecoratedKey key, Clustering clustering, ByteBuffer value); @@ -65,6 +66,6 @@ public abstract class MemoryIndex implements MemtableOrdering public abstract Iterator> iterator(); public abstract SegmentMetadata.ComponentMetadataMap writeDirect(IndexDescriptor indexDescriptor, - IndexContext indexContext, + IndexIdentifier indexIdentifier, Function postingTransformer) throws IOException; } diff --git a/src/java/org/apache/cassandra/index/sai/memory/MemtableIndex.java b/src/java/org/apache/cassandra/index/sai/memory/MemtableIndex.java index 21509067d0..d124af651e 100644 --- a/src/java/org/apache/cassandra/index/sai/memory/MemtableIndex.java +++ b/src/java/org/apache/cassandra/index/sai/memory/MemtableIndex.java @@ -29,26 +29,27 @@ import org.apache.cassandra.db.Clustering; import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.db.PartitionPosition; import org.apache.cassandra.dht.AbstractBounds; -import org.apache.cassandra.index.sai.IndexContext; import org.apache.cassandra.index.sai.QueryContext; +import org.apache.cassandra.index.sai.StorageAttachedIndex; import org.apache.cassandra.index.sai.disk.format.IndexDescriptor; +import org.apache.cassandra.index.sai.utils.IndexIdentifier; import org.apache.cassandra.index.sai.disk.v1.segment.SegmentMetadata; +import org.apache.cassandra.index.sai.iterators.KeyRangeIterator; import org.apache.cassandra.index.sai.plan.Expression; import org.apache.cassandra.index.sai.utils.PrimaryKey; import org.apache.cassandra.index.sai.utils.PrimaryKeys; -import org.apache.cassandra.index.sai.iterators.KeyRangeIterator; import org.apache.cassandra.utils.Pair; import org.apache.cassandra.utils.bytecomparable.ByteComparable; public class MemtableIndex implements MemtableOrdering { - private final MemoryIndex index; + private final MemoryIndex memoryIndex; private final LongAdder writeCount = new LongAdder(); private final LongAdder estimatedMemoryUsed = new LongAdder(); - public MemtableIndex(IndexContext indexContext) + public MemtableIndex(StorageAttachedIndex index) { - this.index = indexContext.isVector() ? new VectorMemoryIndex(indexContext) : new TrieMemoryIndex(indexContext); + this.memoryIndex = index.termType().isVector() ? new VectorMemoryIndex(index) : new TrieMemoryIndex(index); } public long writeCount() @@ -63,17 +64,17 @@ public class MemtableIndex implements MemtableOrdering public boolean isEmpty() { - return index.isEmpty(); + return memoryIndex.isEmpty(); } public ByteBuffer getMinTerm() { - return index.getMinTerm(); + return memoryIndex.getMinTerm(); } public ByteBuffer getMaxTerm() { - return index.getMaxTerm(); + return memoryIndex.getMaxTerm(); } public long index(DecoratedKey key, Clustering clustering, ByteBuffer value) @@ -81,7 +82,7 @@ public class MemtableIndex implements MemtableOrdering if (value == null || value.remaining() == 0) return 0; - long ram = index.add(key, clustering, value); + long ram = memoryIndex.add(key, clustering, value); writeCount.increment(); estimatedMemoryUsed.add(ram); return ram; @@ -89,29 +90,29 @@ public class MemtableIndex implements MemtableOrdering public long update(DecoratedKey key, Clustering clustering, ByteBuffer oldValue, ByteBuffer newValue) { - return index.update(key, clustering, oldValue, newValue); + return memoryIndex.update(key, clustering, oldValue, newValue); } public KeyRangeIterator search(QueryContext queryContext, Expression expression, AbstractBounds keyRange) { - return index.search(queryContext, expression, keyRange); + return memoryIndex.search(queryContext, expression, keyRange); } public Iterator> iterator() { - return index.iterator(); + return memoryIndex.iterator(); } public SegmentMetadata.ComponentMetadataMap writeDirect(IndexDescriptor indexDescriptor, - IndexContext indexContext, + IndexIdentifier indexIdentifier, Function postingTransformer) throws IOException { - return index.writeDirect(indexDescriptor, indexContext, postingTransformer); + return memoryIndex.writeDirect(indexDescriptor, indexIdentifier, postingTransformer); } @Override public KeyRangeIterator limitToTopResults(List primaryKeys, Expression expression, int limit) { - return index.limitToTopResults(primaryKeys, expression, limit); + return memoryIndex.limitToTopResults(primaryKeys, expression, limit); } } diff --git a/src/java/org/apache/cassandra/index/sai/memory/MemtableIndexManager.java b/src/java/org/apache/cassandra/index/sai/memory/MemtableIndexManager.java index c845059f5c..36e48a04b1 100644 --- a/src/java/org/apache/cassandra/index/sai/memory/MemtableIndexManager.java +++ b/src/java/org/apache/cassandra/index/sai/memory/MemtableIndexManager.java @@ -36,8 +36,8 @@ import org.apache.cassandra.db.lifecycle.LifecycleNewTracker; import org.apache.cassandra.db.memtable.Memtable; import org.apache.cassandra.db.rows.Row; import org.apache.cassandra.dht.AbstractBounds; -import org.apache.cassandra.index.sai.IndexContext; import org.apache.cassandra.index.sai.QueryContext; +import org.apache.cassandra.index.sai.StorageAttachedIndex; import org.apache.cassandra.index.sai.plan.Expression; import org.apache.cassandra.index.sai.iterators.KeyRangeIterator; import org.apache.cassandra.index.sai.iterators.KeyRangeUnionIterator; @@ -47,12 +47,12 @@ import org.apache.cassandra.utils.FBUtilities; public class MemtableIndexManager { - private final IndexContext indexContext; + private final StorageAttachedIndex index; private final ConcurrentMap liveMemtableIndexMap; - public MemtableIndexManager(IndexContext indexContext) + public MemtableIndexManager(StorageAttachedIndex index) { - this.indexContext = indexContext; + this.index = index; this.liveMemtableIndexMap = new ConcurrentHashMap<>(); } @@ -64,15 +64,15 @@ public class MemtableIndexManager // call to computeIfAbsent() if it's not. (see https://bugs.openjdk.java.net/browse/JDK-8161372) MemtableIndex target = (current != null) ? current - : liveMemtableIndexMap.computeIfAbsent(mt, memtable -> new MemtableIndex(indexContext)); + : liveMemtableIndexMap.computeIfAbsent(mt, memtable -> new MemtableIndex(index)); long start = Clock.Global.nanoTime(); long bytes = 0; - if (indexContext.isNonFrozenCollection()) + if (index.termType().isNonFrozenCollection()) { - Iterator bufferIterator = indexContext.getValuesOf(row, FBUtilities.nowInSeconds()); + Iterator bufferIterator = index.termType().valuesOf(row, FBUtilities.nowInSeconds()); if (bufferIterator != null) { while (bufferIterator.hasNext()) @@ -84,16 +84,16 @@ public class MemtableIndexManager } else { - ByteBuffer value = indexContext.getValueOf(key, row, FBUtilities.nowInSeconds()); + ByteBuffer value = index.termType().valueOf(key, row, FBUtilities.nowInSeconds()); bytes += target.index(key, row.clustering(), value); } - indexContext.getIndexMetrics().memtableIndexWriteLatency.update(Clock.Global.nanoTime() - start, TimeUnit.NANOSECONDS); + index.indexMetrics().memtableIndexWriteLatency.update(Clock.Global.nanoTime() - start, TimeUnit.NANOSECONDS); return bytes; } public long update(DecoratedKey key, Row oldRow, Row newRow, Memtable memtable) { - if (!indexContext.isVector()) + if (!index.termType().isVector()) { return index(key, newRow, memtable); } @@ -102,8 +102,8 @@ public class MemtableIndexManager if (target == null) return 0; - ByteBuffer oldValue = indexContext.getValueOf(key, oldRow, FBUtilities.nowInSeconds()); - ByteBuffer newValue = indexContext.getValueOf(key, newRow, FBUtilities.nowInSeconds()); + ByteBuffer oldValue = index.termType().valueOf(key, oldRow, FBUtilities.nowInSeconds()); + ByteBuffer newValue = index.termType().valueOf(key, newRow, FBUtilities.nowInSeconds()); return target.update(key, oldRow.clustering(), oldValue, newValue); } diff --git a/src/java/org/apache/cassandra/index/sai/memory/TrieMemoryIndex.java b/src/java/org/apache/cassandra/index/sai/memory/TrieMemoryIndex.java index 843076394b..9b60e2b374 100644 --- a/src/java/org/apache/cassandra/index/sai/memory/TrieMemoryIndex.java +++ b/src/java/org/apache/cassandra/index/sai/memory/TrieMemoryIndex.java @@ -26,7 +26,6 @@ import java.util.SortedSet; import java.util.concurrent.atomic.LongAdder; import java.util.function.Function; -import org.apache.cassandra.index.sai.QueryContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -34,20 +33,20 @@ import io.netty.util.concurrent.FastThreadLocal; import org.apache.cassandra.db.Clustering; import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.db.PartitionPosition; -import org.apache.cassandra.db.marshal.AbstractType; import org.apache.cassandra.db.memtable.TrieMemtable; import org.apache.cassandra.db.tries.InMemoryTrie; import org.apache.cassandra.db.tries.Trie; import org.apache.cassandra.dht.AbstractBounds; -import org.apache.cassandra.index.sai.IndexContext; +import org.apache.cassandra.index.sai.QueryContext; +import org.apache.cassandra.index.sai.StorageAttachedIndex; import org.apache.cassandra.index.sai.analyzer.AbstractAnalyzer; import org.apache.cassandra.index.sai.disk.format.IndexDescriptor; import org.apache.cassandra.index.sai.disk.v1.segment.SegmentMetadata; +import org.apache.cassandra.index.sai.iterators.KeyRangeIterator; import org.apache.cassandra.index.sai.plan.Expression; +import org.apache.cassandra.index.sai.utils.IndexIdentifier; import org.apache.cassandra.index.sai.utils.PrimaryKey; import org.apache.cassandra.index.sai.utils.PrimaryKeys; -import org.apache.cassandra.index.sai.iterators.KeyRangeIterator; -import org.apache.cassandra.index.sai.utils.TypeUtil; import org.apache.cassandra.utils.Pair; import org.apache.cassandra.utils.bytecomparable.ByteComparable; import org.apache.cassandra.utils.bytecomparable.ByteSource; @@ -65,22 +64,18 @@ public class TrieMemoryIndex extends MemoryIndex private final InMemoryTrie data; private final PrimaryKeysReducer primaryKeysReducer; - private final AbstractAnalyzer.AnalyzerFactory analyzerFactory; - private final AbstractType validator; private final boolean isLiteral; private ByteBuffer minTerm; private ByteBuffer maxTerm; - public TrieMemoryIndex(IndexContext indexContext) + public TrieMemoryIndex(StorageAttachedIndex index) { - super(indexContext); + super(index); this.data = new InMemoryTrie<>(TrieMemtable.BUFFER_TYPE); this.primaryKeysReducer = new PrimaryKeysReducer(); // The use of the analyzer is within a synchronized block so can be considered thread-safe - this.analyzerFactory = indexContext.getAnalyzerFactory(); - this.validator = indexContext.getValidator(); - this.isLiteral = TypeUtil.isLiteral(validator); + this.isLiteral = index.termType().isLiteral(); } /** @@ -94,53 +89,37 @@ public class TrieMemoryIndex extends MemoryIndex @Override public synchronized long add(DecoratedKey key, Clustering clustering, ByteBuffer value) { - AbstractAnalyzer analyzer = analyzerFactory.create(); - try + value = index.termType().asIndexBytes(value); + final PrimaryKey primaryKey = index.hasClustering() ? index.keyFactory().create(key, clustering) + : index.keyFactory().create(key); + final long initialSizeOnHeap = data.sizeOnHeap(); + final long initialSizeOffHeap = data.sizeOffHeap(); + final long reducerHeapSize = primaryKeysReducer.heapAllocations(); + + if (index.hasAnalyzer()) { - value = TypeUtil.asIndexBytes(value, validator); - analyzer.reset(value); - final PrimaryKey primaryKey = indexContext.hasClustering() ? indexContext.keyFactory().create(key, clustering) - : indexContext.keyFactory().create(key); - final long initialSizeOnHeap = data.sizeOnHeap(); - final long initialSizeOffHeap = data.sizeOffHeap(); - final long reducerHeapSize = primaryKeysReducer.heapAllocations(); - - while (analyzer.hasNext()) + AbstractAnalyzer analyzer = index.analyzer(); + try { - final ByteBuffer term = analyzer.next(); - if (!indexContext.validateMaxTermSize(key, term, false)) - continue; - - setMinMaxTerm(term.duplicate()); - - final ByteComparable comparableBytes = asComparableBytes(term); - - try + analyzer.reset(value); + while (analyzer.hasNext()) { - if (term.limit() <= MAX_RECURSIVE_KEY_LENGTH) - { - data.putRecursive(comparableBytes, primaryKey, primaryKeysReducer); - } - else - { - data.apply(Trie.singleton(comparableBytes, primaryKey), primaryKeysReducer); - } - } - catch (InMemoryTrie.SpaceExhaustedException e) - { - throw new RuntimeException(e); + addTerm(primaryKey, analyzer.next()); } } - - long onHeap = data.sizeOnHeap(); - long offHeap = data.sizeOffHeap(); - long heapAllocations = primaryKeysReducer.heapAllocations(); - return (onHeap - initialSizeOnHeap) + (offHeap - initialSizeOffHeap) + (heapAllocations - reducerHeapSize); + finally + { + analyzer.end(); + } } - finally + else { - analyzer.end(); + addTerm(primaryKey, value); } + long onHeap = data.sizeOnHeap(); + long offHeap = data.sizeOffHeap(); + long heapAllocations = primaryKeysReducer.heapAllocations(); + return (onHeap - initialSizeOnHeap) + (offHeap - initialSizeOffHeap) + (heapAllocations - reducerHeapSize); } @Override @@ -162,7 +141,7 @@ public class TrieMemoryIndex extends MemoryIndex if (logger.isTraceEnabled()) logger.trace("Searching memtable index on expression '{}'...", expression); - switch (expression.getOp()) + switch (expression.getIndexOperator()) { case EQ: case CONTAINS_KEY: @@ -204,7 +183,7 @@ public class TrieMemoryIndex extends MemoryIndex @Override public SegmentMetadata.ComponentMetadataMap writeDirect(IndexDescriptor indexDescriptor, - IndexContext indexContext, + IndexIdentifier indexIdentifier, Function postingTransformer) { throw new UnsupportedOperationException(); @@ -228,18 +207,44 @@ public class TrieMemoryIndex extends MemoryIndex return maxTerm; } + private void addTerm(PrimaryKey primaryKey, ByteBuffer term) + { + if (index.validateMaxTermSize(primaryKey.partitionKey(), term, false)) + { + setMinMaxTerm(term.duplicate()); + + final ByteComparable comparableBytes = asComparableBytes(term); + + try + { + if (term.limit() <= MAX_RECURSIVE_KEY_LENGTH) + { + data.putRecursive(comparableBytes, primaryKey, primaryKeysReducer); + } + else + { + data.apply(Trie.singleton(comparableBytes, primaryKey), primaryKeysReducer); + } + } + catch (InMemoryTrie.SpaceExhaustedException e) + { + throw new RuntimeException(e); + } + } + } + private void setMinMaxTerm(ByteBuffer term) { assert term != null; - minTerm = TypeUtil.min(term, minTerm, indexContext.getValidator()); - maxTerm = TypeUtil.max(term, maxTerm, indexContext.getValidator()); + minTerm = index.termType().min(term, minTerm); + maxTerm = index.termType().max(term, maxTerm); } private ByteComparable asComparableBytes(ByteBuffer input) { return isLiteral ? version -> terminated(ByteSource.of(input, version)) - : version -> TypeUtil.asComparableBytes(input, validator, version); + : version -> index.termType().asComparableBytes(input, version); } private ByteComparable decode(ByteComparable term) @@ -271,8 +276,8 @@ public class TrieMemoryIndex extends MemoryIndex private KeyRangeIterator exactMatch(Expression expression, AbstractBounds keyRange) { - ByteComparable comparableMatch = expression.lower == null ? ByteComparable.EMPTY - : asComparableBytes(expression.lower.value.encoded); + ByteComparable comparableMatch = expression.lower() == null ? ByteComparable.EMPTY + : asComparableBytes(expression.lower().value.encoded); PrimaryKeys primaryKeys = data.get(comparableMatch); return primaryKeys == null ? KeyRangeIterator.empty() : new FilteringInMemoryKeyRangeIterator(primaryKeys.keys(), keyRange); @@ -348,10 +353,10 @@ public class TrieMemoryIndex extends MemoryIndex { ByteComparable lowerBound, upperBound; boolean lowerInclusive, upperInclusive; - if (expression.lower != null) + if (expression.lower() != null) { - lowerBound = asComparableBytes(expression.lower.value.encoded); - lowerInclusive = expression.lower.inclusive; + lowerBound = asComparableBytes(expression.lower().value.encoded); + lowerInclusive = expression.lower().inclusive; } else { @@ -359,10 +364,10 @@ public class TrieMemoryIndex extends MemoryIndex lowerInclusive = false; } - if (expression.upper != null) + if (expression.upper() != null) { - upperBound = asComparableBytes(expression.upper.value.encoded); - upperInclusive = expression.upper.inclusive; + upperBound = asComparableBytes(expression.upper().value.encoded); + upperInclusive = expression.upper().inclusive; } else { diff --git a/src/java/org/apache/cassandra/index/sai/memory/VectorMemoryIndex.java b/src/java/org/apache/cassandra/index/sai/memory/VectorMemoryIndex.java index 12b21d2205..52ae0b8baa 100644 --- a/src/java/org/apache/cassandra/index/sai/memory/VectorMemoryIndex.java +++ b/src/java/org/apache/cassandra/index/sai/memory/VectorMemoryIndex.java @@ -37,10 +37,11 @@ import org.apache.cassandra.db.Clustering; import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.db.PartitionPosition; import org.apache.cassandra.dht.AbstractBounds; -import org.apache.cassandra.index.sai.IndexContext; import org.apache.cassandra.index.sai.QueryContext; +import org.apache.cassandra.index.sai.StorageAttachedIndex; import org.apache.cassandra.index.sai.VectorQueryContext; import org.apache.cassandra.index.sai.disk.format.IndexDescriptor; +import org.apache.cassandra.index.sai.utils.IndexIdentifier; import org.apache.cassandra.index.sai.disk.v1.segment.SegmentMetadata; import org.apache.cassandra.index.sai.disk.v1.vector.OnHeapGraph; import org.apache.cassandra.index.sai.iterators.KeyRangeIterator; @@ -49,7 +50,6 @@ import org.apache.cassandra.index.sai.plan.Expression; import org.apache.cassandra.index.sai.utils.PrimaryKey; import org.apache.cassandra.index.sai.utils.PrimaryKeys; import org.apache.cassandra.index.sai.utils.RangeUtil; -import org.apache.cassandra.index.sai.utils.TypeUtil; import org.apache.cassandra.tracing.Tracing; import org.apache.cassandra.utils.Pair; import org.apache.cassandra.utils.bytecomparable.ByteComparable; @@ -69,20 +69,20 @@ public class VectorMemoryIndex extends MemoryIndex private final NavigableSet primaryKeys = new ConcurrentSkipListSet<>(); - public VectorMemoryIndex(IndexContext indexContext) + public VectorMemoryIndex(StorageAttachedIndex index) { - super(indexContext); - this.graph = new OnHeapGraph<>(indexContext.getValidator(), indexContext.getIndexWriterConfig()); + super(index); + this.graph = new OnHeapGraph<>(index.termType().indexType(), index.indexWriterConfig()); } @Override public synchronized long add(DecoratedKey key, Clustering clustering, ByteBuffer value) { - if (value == null || value.remaining() == 0 || !indexContext.validateMaxTermSize(key, value, false)) + if (value == null || value.remaining() == 0 || !index.validateMaxTermSize(key, value, false)) return 0; - var primaryKey = indexContext.hasClustering() ? indexContext.keyFactory().create(key, clustering) - : indexContext.keyFactory().create(key); + var primaryKey = index.hasClustering() ? index.keyFactory().create(key, clustering) + : index.keyFactory().create(key); return index(primaryKey, value); } @@ -117,8 +117,8 @@ public class VectorMemoryIndex extends MemoryIndex long bytesUsed = 0; if (different) { - var primaryKey = indexContext.hasClustering() ? indexContext.keyFactory().create(key, clustering) - : indexContext.keyFactory().create(key); + var primaryKey = index.hasClustering() ? index.keyFactory().create(key, clustering) + : index.keyFactory().create(key); // update bounds because only rows with vectors are included in the key bounds, // so if the vector was null before, we won't have included it updateKeyBounds(primaryKey); @@ -151,12 +151,12 @@ public class VectorMemoryIndex extends MemoryIndex @Override public KeyRangeIterator search(QueryContext queryContext, Expression expr, AbstractBounds keyRange) { - assert expr.getOp() == Expression.IndexOperator.ANN : "Only ANN is supported for vector search, received " + expr.getOp(); + assert expr.getIndexOperator() == Expression.IndexOperator.ANN : "Only ANN is supported for vector search, received " + expr.getIndexOperator(); VectorQueryContext vectorQueryContext = queryContext.vectorContext(); - var buffer = expr.lower.value.raw; - float[] qv = TypeUtil.decomposeVector(indexContext, buffer); + var buffer = expr.lower().value.raw; + float[] qv = index.termType().decomposeVector(buffer); Bits bits; if (!RangeUtil.coversFullRing(keyRange)) @@ -168,8 +168,8 @@ public class VectorMemoryIndex extends MemoryIndex // if right token is MAX (Long.MIN_VALUE), there is no upper bound boolean isMaxToken = keyRange.right.getToken().isMinimum(); // max token - PrimaryKey left = indexContext.keyFactory().create(keyRange.left.getToken()); // lower bound - PrimaryKey right = isMaxToken ? null : indexContext.keyFactory().create(keyRange.right.getToken()); // upper bound + PrimaryKey left = index.keyFactory().create(keyRange.left.getToken()); // lower bound + PrimaryKey right = isMaxToken ? null : index.keyFactory().create(keyRange.right.getToken()); // upper bound Set resultKeys = isMaxToken ? primaryKeys.tailSet(left, leftInclusive) : primaryKeys.subSet(left, leftInclusive, right, rightInclusive); if (!vectorQueryContext.getShadowedPrimaryKeys().isEmpty()) @@ -220,8 +220,8 @@ public class VectorMemoryIndex extends MemoryIndex return new KeyRangeListIterator(minimumKey, maximumKey, results); } - ByteBuffer buffer = expression.lower.value.raw; - float[] qv = TypeUtil.decomposeVector(indexContext, buffer); + ByteBuffer buffer = expression.lower().value.raw; + float[] qv = index.termType().decomposeVector(buffer); var bits = new KeyFilteringBits(results); var keyQueue = graph.search(qv, limit, bits); if (keyQueue.isEmpty()) @@ -232,7 +232,7 @@ public class VectorMemoryIndex extends MemoryIndex private int maxBruteForceRows(int limit, int nPermittedOrdinals, int graphSize) { int expectedNodesVisited = expectedNodesVisited(limit, nPermittedOrdinals, graphSize); - int expectedComparisons = indexContext.getIndexWriterConfig().getMaximumNodeConnections() * expectedNodesVisited; + int expectedComparisons = index.indexWriterConfig().getMaximumNodeConnections() * expectedNodesVisited; // in-memory comparisons are cheaper than pulling a row off disk and then comparing // VSTODO this is dramatically oversimplified // larger dimension should increase this, because comparisons are more expensive @@ -266,10 +266,10 @@ public class VectorMemoryIndex extends MemoryIndex } public SegmentMetadata.ComponentMetadataMap writeDirect(IndexDescriptor indexDescriptor, - IndexContext indexContext, + IndexIdentifier indexIdentifier, Function postingTransformer) throws IOException { - return graph.writeData(indexDescriptor, indexContext, postingTransformer); + return graph.writeData(indexDescriptor, indexIdentifier, postingTransformer); } @Override diff --git a/src/java/org/apache/cassandra/index/sai/metrics/AbstractMetrics.java b/src/java/org/apache/cassandra/index/sai/metrics/AbstractMetrics.java index d9fbb8cc72..90956e4b32 100644 --- a/src/java/org/apache/cassandra/index/sai/metrics/AbstractMetrics.java +++ b/src/java/org/apache/cassandra/index/sai/metrics/AbstractMetrics.java @@ -20,6 +20,7 @@ package org.apache.cassandra.index.sai.metrics; import java.util.ArrayList; import java.util.List; +import org.apache.cassandra.index.sai.utils.IndexIdentifier; import org.apache.cassandra.metrics.CassandraMetricsRegistry; import org.apache.cassandra.metrics.DefaultNameFactory; @@ -35,6 +36,11 @@ public abstract class AbstractMetrics private final String scope; protected final List tracked = new ArrayList<>(); + AbstractMetrics(IndexIdentifier indexIdentifier, String scope) + { + this(indexIdentifier.keyspaceName, indexIdentifier.tableName, indexIdentifier.indexName, scope); + } + AbstractMetrics(String keyspace, String table, String scope) { this(keyspace, table, null, scope); diff --git a/src/java/org/apache/cassandra/index/sai/metrics/ColumnQueryMetrics.java b/src/java/org/apache/cassandra/index/sai/metrics/ColumnQueryMetrics.java index 7259ecc495..f922231d0b 100644 --- a/src/java/org/apache/cassandra/index/sai/metrics/ColumnQueryMetrics.java +++ b/src/java/org/apache/cassandra/index/sai/metrics/ColumnQueryMetrics.java @@ -21,15 +21,15 @@ import java.util.concurrent.TimeUnit; import com.codahale.metrics.Meter; import com.codahale.metrics.Timer; -import org.apache.cassandra.index.sai.IndexContext; +import org.apache.cassandra.index.sai.utils.IndexIdentifier; import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics; public abstract class ColumnQueryMetrics extends AbstractMetrics { - protected ColumnQueryMetrics(IndexContext indexContext) + protected ColumnQueryMetrics(IndexIdentifier indexIdentifier) { - super(indexContext.getKeyspace(), indexContext.getTable(), indexContext.getIndexName(), "ColumnQueryMetrics"); + super(indexIdentifier, "ColumnQueryMetrics"); } public static class TrieIndexMetrics extends ColumnQueryMetrics implements QueryEventListener.TrieIndexEventListener @@ -43,9 +43,9 @@ public abstract class ColumnQueryMetrics extends AbstractMetrics private final QueryEventListener.PostingListEventListener postingsListener; - public TrieIndexMetrics(IndexContext indexContext) + public TrieIndexMetrics(IndexIdentifier indexIdentifier) { - super(indexContext); + super(indexIdentifier); termsTraversalTotalTime = Metrics.timer(createMetricName("TermsLookupLatency")); @@ -83,9 +83,9 @@ public abstract class ColumnQueryMetrics extends AbstractMetrics private final QueryEventListener.PostingListEventListener postingsListener; - public BalancedTreeIndexMetrics(IndexContext indexContext) + public BalancedTreeIndexMetrics(IndexIdentifier indexIdentifier) { - super(indexContext); + super(indexIdentifier); intersectionLatency = Metrics.timer(createMetricName("BalancedTreeIntersectionLatency")); intersectionEarlyExits = Metrics.meter(createMetricName("BalancedTreeIntersectionEarlyExits")); diff --git a/src/java/org/apache/cassandra/index/sai/metrics/IndexMetrics.java b/src/java/org/apache/cassandra/index/sai/metrics/IndexMetrics.java index edba3c1cd3..575fb79b9b 100644 --- a/src/java/org/apache/cassandra/index/sai/metrics/IndexMetrics.java +++ b/src/java/org/apache/cassandra/index/sai/metrics/IndexMetrics.java @@ -21,7 +21,7 @@ import com.codahale.metrics.Counter; import com.codahale.metrics.Gauge; import com.codahale.metrics.Histogram; import com.codahale.metrics.Timer; -import org.apache.cassandra.index.sai.IndexContext; +import org.apache.cassandra.index.sai.StorageAttachedIndex; import org.apache.cassandra.index.sai.memory.MemtableIndexManager; import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics; @@ -40,11 +40,10 @@ public class IndexMetrics extends AbstractMetrics public final Histogram compactionSegmentCellsPerSecond; public final Histogram compactionSegmentBytesPerSecond; - public IndexMetrics(IndexContext indexContext) + public IndexMetrics(StorageAttachedIndex index, MemtableIndexManager memtableIndexManager) { - super(indexContext.getKeyspace(), indexContext.getTable(), indexContext.getIndexName(), "IndexMetrics"); + super(index.identifier(), "IndexMetrics"); - MemtableIndexManager memtableIndexManager = indexContext.getMemtableIndexManager(); memtableIndexWriteLatency = Metrics.timer(createMetricName("MemtableIndexWriteLatency")); compactionSegmentCellsPerSecond = Metrics.histogram(createMetricName("CompactionSegmentCellsPerSecond"), false); compactionSegmentBytesPerSecond = Metrics.histogram(createMetricName("CompactionSegmentBytesPerSecond"), false); @@ -54,10 +53,10 @@ public class IndexMetrics extends AbstractMetrics compactionCount = Metrics.counter(createMetricName("CompactionCount")); memtableIndexFlushErrors = Metrics.counter(createMetricName("MemtableIndexFlushErrors")); segmentFlushErrors = Metrics.counter(createMetricName("CompactionSegmentFlushErrors")); - Metrics.register(createMetricName("SSTableCellCount"), (Gauge) indexContext::getCellCount); + Metrics.register(createMetricName("SSTableCellCount"), (Gauge) index::cellCount); Metrics.register(createMetricName("LiveMemtableIndexWriteCount"), (Gauge) memtableIndexManager::liveMemtableWriteCount); Metrics.register(createMetricName("MemtableIndexBytes"), (Gauge) memtableIndexManager::estimatedMemIndexMemoryUsed); - Metrics.register(createMetricName("DiskUsedBytes"), (Gauge) indexContext::diskUsage); - Metrics.register(createMetricName("IndexFileCacheBytes"), (Gauge) indexContext::indexFileCacheSize); + Metrics.register(createMetricName("DiskUsedBytes"), (Gauge) index::diskUsage); + Metrics.register(createMetricName("IndexFileCacheBytes"), (Gauge) index::indexFileCacheSize); } } diff --git a/src/java/org/apache/cassandra/index/sai/plan/Expression.java b/src/java/org/apache/cassandra/index/sai/plan/Expression.java index 99f7d46af9..262fc3a9cc 100644 --- a/src/java/org/apache/cassandra/index/sai/plan/Expression.java +++ b/src/java/org/apache/cassandra/index/sai/plan/Expression.java @@ -26,14 +26,54 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.cassandra.cql3.Operator; -import org.apache.cassandra.db.marshal.AbstractType; -import org.apache.cassandra.index.sai.IndexContext; +import org.apache.cassandra.index.sai.StorageAttachedIndex; import org.apache.cassandra.index.sai.analyzer.AbstractAnalyzer; -import org.apache.cassandra.index.sai.utils.TypeUtil; +import org.apache.cassandra.index.sai.utils.IndexTermType; -public class Expression +/** + * An {@link Expression} is an internal representation of an index query operation. They are built from + * CQL {@link Operator} and {@link ByteBuffer} value pairs for a single column. + *

    + * Each {@link Expression} consists of an {@link IndexOperator} and optional lower and upper {@link Bound}s. + *

    + * The {@link IndexedExpression} has a backing {@link StorageAttachedIndex} for the index query but order to support + * CQL expressions on columns that do not have indexes or use operators that are not supported by the index there is + * an {@link UnindexedExpression} that does not provide a {@link StorageAttachedIndex} and can only be used for + * post-filtering + */ +public abstract class Expression { - private static final Logger logger = LoggerFactory.getLogger(Expression.class); + Logger logger = LoggerFactory.getLogger(Expression.class); + + private final IndexTermType indexTermType; + protected IndexOperator operator; + + public Bound lower, upper; + // The upperInclusive and lowerInclusive flags are maintained separately to the inclusive flags + // in the upper and lower bounds because the upper and lower bounds have their inclusivity relaxed + // if the datatype being filtered is rounded in the index. These flags are used in the post-filtering + // process to remove values equal to the bounds. + public boolean upperInclusive, lowerInclusive; + + Expression(IndexTermType indexTermType) + { + this.indexTermType = indexTermType; + } + + public static Expression create(StorageAttachedIndex index) + { + return new IndexedExpression(index); + } + + public static Expression create(IndexTermType indexTermType) + { + return new UnindexedExpression(indexTermType); + } + + public static boolean supportsOperator(Operator operator) + { + return IndexOperator.valueOf(operator) != null; + } public enum IndexOperator { @@ -77,25 +117,32 @@ public class Expression } } - public final AbstractAnalyzer.AnalyzerFactory analyzerFactory; + public abstract boolean isNotIndexed(); - public final IndexContext context; - public final AbstractType validator; + public abstract StorageAttachedIndex getIndex(); - protected IndexOperator operator; + abstract boolean hasAnalyzer(); - public Bound lower, upper; - // The upperInclusive and lowerInclusive flags are maintained separately to the inclusive flags - // in the upper and lower bounds because the upper and lower bounds have their inclusivity relaxed - // if the datatype being filtered is rounded in the index. These flags are used in the post-filtering - // process to remove values equal to the bounds. - public boolean upperInclusive, lowerInclusive; + abstract AbstractAnalyzer getAnalyzer(); - public Expression(IndexContext indexContext) + public IndexOperator getIndexOperator() { - this.context = indexContext; - this.analyzerFactory = indexContext.getAnalyzerFactory(); - this.validator = indexContext.getValidator(); + return operator; + } + + public IndexTermType getIndexTermType() + { + return indexTermType; + } + + public Bound lower() + { + return lower; + } + + public Bound upper() + { + return upper; } /** @@ -113,19 +160,19 @@ public class Expression // range search is always inclusive, otherwise we run the risk of // missing values that are within the exclusive range but are rejected // because their rounded value is the same as the value being queried. - lowerInclusive = upperInclusive = TypeUtil.supportsRounding(validator); + lowerInclusive = upperInclusive = indexTermType.supportsRounding(); switch (op) { case EQ: case CONTAINS: case CONTAINS_KEY: - lower = new Bound(value, validator, true); + lower = new Bound(value, indexTermType, true); upper = lower; operator = IndexOperator.valueOf(op); break; case LTE: - if (context.getDefinition().isReversedType()) + if (indexTermType.isReversed()) { this.lowerInclusive = true; lowerInclusive = true; @@ -137,14 +184,14 @@ public class Expression } case LT: operator = IndexOperator.RANGE; - if (context.getDefinition().isReversedType()) - lower = new Bound(value, validator, lowerInclusive); + if (indexTermType.isReversed()) + lower = new Bound(value, indexTermType, lowerInclusive); else - upper = new Bound(value, validator, upperInclusive); + upper = new Bound(value, indexTermType, upperInclusive); break; case GTE: - if (context.getDefinition().isReversedType()) + if (indexTermType.isReversed()) { this.upperInclusive = true; upperInclusive = true; @@ -156,20 +203,20 @@ public class Expression } case GT: operator = IndexOperator.RANGE; - if (context.getDefinition().isReversedType()) - upper = new Bound(value, validator, upperInclusive); + if (indexTermType.isReversed()) + upper = new Bound(value, indexTermType, upperInclusive); else - lower = new Bound(value, validator, lowerInclusive); + lower = new Bound(value, indexTermType, lowerInclusive); break; case ANN: operator = IndexOperator.ANN; - lower = new Bound(value, validator, true); + lower = new Bound(value, indexTermType, true); upper = lower; break; + default: + throw new IllegalArgumentException("Index does not support the " + op + " operator"); } - assert operator != null; - return this; } @@ -180,26 +227,26 @@ public class Expression { // If the expression represents an ANN ordering then we return true because the actual result // is approximate and will rarely / never match the expression value - if (validator.isVector()) + if (indexTermType.isVector()) return true; - if (!TypeUtil.isValid(columnValue, validator)) + if (!indexTermType.isValid(columnValue)) { - logger.error(context.logMessage("Value is not valid for indexed column {} with {}"), context.getColumnName(), validator); + logger.error("Value is not valid for indexed column {} with {}", indexTermType.columnName(), indexTermType.indexType()); return false; } - Value value = new Value(columnValue, validator); + Value value = new Value(columnValue, indexTermType); if (lower != null) { // suffix check - if (TypeUtil.isLiteral(validator)) + if (indexTermType.isLiteral()) return validateStringValue(value.raw, lower.value.raw); else { // range or (not-)equals - (mainly) for numeric values - int cmp = TypeUtil.comparePostFilter(lower.value, value, validator); + int cmp = indexTermType.comparePostFilter(lower.value, value); // in case of EQ lower == upper if (operator == IndexOperator.EQ || operator == IndexOperator.CONTAINS_KEY || operator == IndexOperator.CONTAINS_VALUE) @@ -213,12 +260,12 @@ public class Expression if (upper != null && lower != upper) { // string (prefix or suffix) check - if (TypeUtil.isLiteral(validator)) + if (indexTermType.isLiteral()) return validateStringValue(value.raw, upper.value.raw); else { // range - mainly for numeric values - int cmp = TypeUtil.comparePostFilter(upper.value, value, validator); + int cmp = indexTermType.comparePostFilter(upper.value, value); return (cmp > 0 || (cmp == 0 && upperInclusive)); } } @@ -228,41 +275,45 @@ public class Expression private boolean validateStringValue(ByteBuffer columnValue, ByteBuffer requestedValue) { - AbstractAnalyzer analyzer = analyzerFactory.create(); - analyzer.reset(columnValue.duplicate()); - try + if (hasAnalyzer()) { - while (analyzer.hasNext()) + AbstractAnalyzer analyzer = getAnalyzer(); + analyzer.reset(columnValue.duplicate()); + try { - final ByteBuffer term = analyzer.next(); - - boolean isMatch = false; - switch (operator) + while (analyzer.hasNext()) { - case EQ: - case CONTAINS_KEY: - case CONTAINS_VALUE: - isMatch = validator.compare(term, requestedValue) == 0; - break; - case RANGE: - isMatch = isLowerSatisfiedBy(term) && isUpperSatisfiedBy(term); - break; + if (termMatches(analyzer.next(), requestedValue)) + return true; } - - if (isMatch) - return true; + return false; + } + finally + { + analyzer.end(); } - return false; } - finally + else { - analyzer.end(); + return termMatches(columnValue, requestedValue); } } - public IndexOperator getOp() + private boolean termMatches(ByteBuffer term, ByteBuffer requestedValue) { - return operator; + boolean isMatch = false; + switch (operator) + { + case EQ: + case CONTAINS_KEY: + case CONTAINS_VALUE: + isMatch = indexTermType.compare(term, requestedValue) == 0; + break; + case RANGE: + isMatch = isLowerSatisfiedBy(term) && isUpperSatisfiedBy(term); + break; + } + return isMatch; } private boolean hasLower() @@ -280,7 +331,7 @@ public class Expression if (!hasLower()) return true; - int cmp = validator.compare(value, lower.value.raw); + int cmp = indexTermType.indexType().compare(value, lower.value.raw); return cmp > 0 || cmp == 0 && lower.inclusive; } @@ -289,7 +340,7 @@ public class Expression if (!hasUpper()) return true; - int cmp = validator.compare(value, upper.value.raw); + int cmp = indexTermType.indexType().compare(value, upper.value.raw); return cmp < 0 || cmp == 0 && upper.inclusive; } @@ -297,20 +348,19 @@ public class Expression public String toString() { return String.format("Expression{name: %s, op: %s, lower: (%s, %s), upper: (%s, %s)}", - context.getColumnName(), + indexTermType.columnName(), operator, - lower == null ? "null" : validator.getString(lower.value.raw), + lower == null ? "null" : indexTermType.asString(lower.value.raw), lower != null && lower.inclusive, - upper == null ? "null" : validator.getString(upper.value.raw), + upper == null ? "null" : indexTermType.asString(upper.value.raw), upper != null && upper.inclusive); } @Override public int hashCode() { - return new HashCodeBuilder().append(context.getColumnName()) + return new HashCodeBuilder().append(indexTermType) .append(operator) - .append(validator) .append(lower).append(upper).build(); } @@ -325,13 +375,79 @@ public class Expression Expression o = (Expression) other; - return Objects.equals(context.getColumnName(), o.context.getColumnName()) - && validator.equals(o.validator) + return Objects.equals(indexTermType, o.indexTermType) && operator == o.operator && Objects.equals(lower, o.lower) && Objects.equals(upper, o.upper); } + public static class IndexedExpression extends Expression + { + private final StorageAttachedIndex index; + + public IndexedExpression(StorageAttachedIndex index) + { + super(index.termType()); + this.index = index; + } + + @Override + public boolean isNotIndexed() + { + return false; + } + + @Override + public StorageAttachedIndex getIndex() + { + return index; + } + + @Override + boolean hasAnalyzer() + { + return index.hasAnalyzer(); + } + + @Override + AbstractAnalyzer getAnalyzer() + { + return index.analyzer(); + } + } + + public static class UnindexedExpression extends Expression + { + private UnindexedExpression(IndexTermType indexTermType) + { + super(indexTermType); + } + + @Override + public boolean isNotIndexed() + { + return true; + } + + @Override + public StorageAttachedIndex getIndex() + { + throw new UnsupportedOperationException(); + } + + @Override + boolean hasAnalyzer() + { + return false; + } + + @Override + AbstractAnalyzer getAnalyzer() + { + throw new UnsupportedOperationException(); + } + } + /** * A representation of a column value in its raw and encoded form. */ @@ -340,10 +456,10 @@ public class Expression public final ByteBuffer raw; public final ByteBuffer encoded; - public Value(ByteBuffer value, AbstractType type) + public Value(ByteBuffer value, IndexTermType indexTermType) { this.raw = value; - this.encoded = TypeUtil.asIndexBytes(value, type); + this.encoded = indexTermType.asIndexBytes(value); } @Override @@ -371,9 +487,9 @@ public class Expression public final Value value; public final boolean inclusive; - public Bound(ByteBuffer value, AbstractType type, boolean inclusive) + public Bound(ByteBuffer value, IndexTermType indexTermType, boolean inclusive) { - this.value = new Value(value, type); + this.value = new Value(value, indexTermType); this.inclusive = inclusive; } diff --git a/src/java/org/apache/cassandra/index/sai/plan/FilterTree.java b/src/java/org/apache/cassandra/index/sai/plan/FilterTree.java index 52927f285c..8c1ab0fe03 100644 --- a/src/java/org/apache/cassandra/index/sai/plan/FilterTree.java +++ b/src/java/org/apache/cassandra/index/sai/plan/FilterTree.java @@ -28,7 +28,6 @@ import com.google.common.collect.ListMultimap; import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.db.rows.Row; import org.apache.cassandra.db.rows.Unfiltered; -import org.apache.cassandra.index.sai.utils.TypeUtil; import org.apache.cassandra.schema.ColumnMetadata; import org.apache.cassandra.schema.ColumnMetadata.Kind; import org.apache.cassandra.utils.FBUtilities; @@ -37,7 +36,7 @@ import static org.apache.cassandra.index.sai.plan.Operation.BooleanOperator; /** * Tree-like structure to filter base table data using indexed expressions and non-user-defined filters. - * + *

    * This is needed because: * 1. SAI doesn't index tombstones, base data may have been shadowed. * 2. Replica filter protecting may fetch data that doesn't match index expressions. @@ -48,8 +47,7 @@ public class FilterTree protected final ListMultimap expressions; protected final List children = new ArrayList<>(); - FilterTree(BooleanOperator operation, - ListMultimap expressions) + FilterTree(BooleanOperator operation, ListMultimap expressions) { this.op = operation; this.expressions = expressions; @@ -95,14 +93,14 @@ public class FilterTree { Expression filter = filterIterator.previous(); - if (TypeUtil.isNonFrozenCollection(column.type)) + if (filter.getIndexTermType().isNonFrozenCollection()) { - Iterator valueIterator = filter.context.getValuesOf(row, now); + Iterator valueIterator = filter.getIndexTermType().valuesOf(row, now); result = op.apply(result, collectionMatch(valueIterator, filter)); } else { - ByteBuffer value = filter.context.getValueOf(key, row, now); + ByteBuffer value = filter.getIndexTermType().valueOf(key, row, now); result = op.apply(result, singletonMatch(value, filter)); } diff --git a/src/java/org/apache/cassandra/index/sai/plan/Operation.java b/src/java/org/apache/cassandra/index/sai/plan/Operation.java index 51c63476e7..eb56b9c132 100644 --- a/src/java/org/apache/cassandra/index/sai/plan/Operation.java +++ b/src/java/org/apache/cassandra/index/sai/plan/Operation.java @@ -31,11 +31,15 @@ import com.google.common.collect.Iterables; import com.google.common.collect.ListMultimap; import org.apache.cassandra.cql3.Operator; +import org.apache.cassandra.cql3.statements.schema.IndexTarget; import org.apache.cassandra.db.filter.RowFilter; -import org.apache.cassandra.index.sai.IndexContext; +import org.apache.cassandra.db.marshal.AbstractType; +import org.apache.cassandra.db.marshal.CollectionType; +import org.apache.cassandra.exceptions.InvalidRequestException; +import org.apache.cassandra.index.sai.StorageAttachedIndex; import org.apache.cassandra.index.sai.analyzer.AbstractAnalyzer; import org.apache.cassandra.index.sai.iterators.KeyRangeIterator; -import org.apache.cassandra.index.sai.utils.TypeUtil; +import org.apache.cassandra.index.sai.utils.IndexTermType; import org.apache.cassandra.schema.ColumnMetadata; public class Operation @@ -58,8 +62,7 @@ public class Operation } @VisibleForTesting - protected static ListMultimap buildIndexExpressions(QueryController controller, - BooleanOperator booleanOperator, + protected static ListMultimap buildIndexExpressions(QueryController queryController, List expressions) { ListMultimap analyzed = ArrayListMultimap.create(); @@ -72,40 +75,66 @@ public class Operation return cmp == 0 ? -Integer.compare(getPriority(a.operator()), getPriority(b.operator())) : cmp; }); - for (final RowFilter.Expression e : expressions) + for (final RowFilter.Expression expression : expressions) { - IndexContext indexContext = controller.getContext(e); - List perColumn = analyzed.get(e.column()); + if (Expression.supportsOperator(expression.operator())) + { + StorageAttachedIndex index = queryController.indexFor(expression); - AbstractAnalyzer analyzer = indexContext.getAnalyzerFactory().create(); + List perColumn = analyzed.get(expression.column()); + + if (index == null) + buildUnindexedExpression(queryController, expression, perColumn); + else + buildIndexedExpression(index, expression, perColumn); + } + } + + return analyzed; + } + + private static void buildUnindexedExpression(QueryController queryController, + RowFilter.Expression expression, + List perColumn) + { + IndexTermType indexTermType = IndexTermType.create(expression.column(), + queryController.metadata().partitionKeyColumns(), + determineIndexTargetType(expression)); + if (indexTermType.isMultiExpression(expression)) + { + perColumn.add(Expression.create(indexTermType).add(expression.operator(), expression.getIndexValue().duplicate())); + } + else + { + Expression range; + if (perColumn.size() == 0) + { + range = Expression.create(indexTermType); + perColumn.add(range); + } + else + { + range = Iterables.getLast(perColumn); + } + range.add(expression.operator(), expression.getIndexValue().duplicate()); + } + } + + private static void buildIndexedExpression(StorageAttachedIndex index, RowFilter.Expression expression, List perColumn) + { + if (index.hasAnalyzer()) + { + AbstractAnalyzer analyzer = index.analyzer(); try { - analyzer.reset(e.getIndexValue().duplicate()); + analyzer.reset(expression.getIndexValue().duplicate()); - // EQ can have multiple expressions e.g. text = "Hello World", - // becomes text = "Hello" OR text = "World" because "space" is always interpreted as a split point (by analyzer), - // CONTAINS/CONTAINS_KEY are always treated as multiple expressions since they currently only targetting - // collections. - boolean isMultiExpression = false; - switch (e.operator()) - { - case EQ: - // EQ operator will always be a multiple expression because it is being used by - // map entries - isMultiExpression = indexContext.isNonFrozenCollection(); - break; - - case CONTAINS: - case CONTAINS_KEY: - isMultiExpression = true; - break; - } - if (isMultiExpression) + if (index.termType().isMultiExpression(expression)) { while (analyzer.hasNext()) { final ByteBuffer token = analyzer.next(); - perColumn.add(new Expression(indexContext).add(e.operator(), token.duplicate())); + perColumn.add(Expression.create(index).add(expression.operator(), token.duplicate())); } } else @@ -114,9 +143,9 @@ public class Operation // not-equals is combined with the range iff operator is AND. { Expression range; - if (perColumn.size() == 0 || booleanOperator != BooleanOperator.AND) + if (perColumn.size() == 0) { - range = new Expression(indexContext); + range = Expression.create(index); perColumn.add(range); } else @@ -124,18 +153,18 @@ public class Operation range = Iterables.getLast(perColumn); } - if (!TypeUtil.isLiteral(indexContext.getValidator())) - { - range.add(e.operator(), e.getIndexValue().duplicate()); - } - else + if (index.termType().isLiteral()) { while (analyzer.hasNext()) { ByteBuffer term = analyzer.next(); - range.add(e.operator(), term.duplicate()); + range.add(expression.operator(), term.duplicate()); } } + else + { + range.add(expression.operator(), expression.getIndexValue().duplicate()); + } } } finally @@ -143,8 +172,59 @@ public class Operation analyzer.end(); } } + else + { + if (index.termType().isMultiExpression(expression)) + { + perColumn.add(Expression.create(index).add(expression.operator(), expression.getIndexValue().duplicate())); + } + else + { + Expression range; + if (perColumn.size() == 0) + { + range = Expression.create(index); + perColumn.add(range); + } + else + { + range = Iterables.getLast(perColumn); + } + range.add(expression.operator(), expression.getIndexValue().duplicate()); + } + } + } - return analyzed; + /** + * Determines the {@link IndexTarget.Type} for the expression. In this case we are only interested in map types and + * the operator being used in the expression. + */ + private static IndexTarget.Type determineIndexTargetType(RowFilter.Expression expression) + { + AbstractType type = expression.column().type; + IndexTarget.Type indexTargetType = IndexTarget.Type.SIMPLE; + if (type.isCollection() && type.isMultiCell()) + { + CollectionType collection = ((CollectionType) type); + if (collection.kind == CollectionType.Kind.MAP) + { + switch (expression.operator()) + { + case EQ: + indexTargetType = IndexTarget.Type.KEYS_AND_VALUES; + break; + case CONTAINS: + indexTargetType = IndexTarget.Type.VALUES; + break; + case CONTAINS_KEY: + indexTargetType = IndexTarget.Type.KEYS; + break; + default: + throw new InvalidRequestException("Invalid operator"); + } + } + } + return indexTargetType; } private static int getPriority(Operator op) @@ -300,7 +380,7 @@ public class Operation @Override public void analyze(List expressionList, QueryController controller) { - expressionMap = buildIndexExpressions(controller, BooleanOperator.AND, expressionList); + expressionMap = buildIndexExpressions(controller, expressionList); } @Override @@ -330,7 +410,7 @@ public class Operation @Override public void analyze(List expressionList, QueryController controller) { - expressionMap = buildIndexExpressions(controller, BooleanOperator.AND, expressionList); + expressionMap = buildIndexExpressions(controller, expressionList); } @Override diff --git a/src/java/org/apache/cassandra/index/sai/plan/QueryController.java b/src/java/org/apache/cassandra/index/sai/plan/QueryController.java index b236c61db4..ebd8fc20d0 100644 --- a/src/java/org/apache/cassandra/index/sai/plan/QueryController.java +++ b/src/java/org/apache/cassandra/index/sai/plan/QueryController.java @@ -29,7 +29,6 @@ import java.util.stream.Collectors; import com.google.common.collect.Lists; import org.apache.cassandra.cql3.Operator; -import org.apache.cassandra.cql3.statements.schema.IndexTarget; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.DataRange; import org.apache.cassandra.db.DecoratedKey; @@ -45,7 +44,6 @@ import org.apache.cassandra.db.filter.RowFilter; import org.apache.cassandra.db.rows.UnfilteredRowIterator; import org.apache.cassandra.dht.AbstractBounds; import org.apache.cassandra.dht.Range; -import org.apache.cassandra.index.sai.IndexContext; import org.apache.cassandra.index.sai.QueryContext; import org.apache.cassandra.index.sai.StorageAttachedIndex; import org.apache.cassandra.index.sai.VectorQueryContext; @@ -133,24 +131,16 @@ public class QueryController return ranges; } - /** - * @return indexed {@code IndexContext} if index is found; otherwise return non-indexed {@code IndexContext}. - */ - public IndexContext getContext(RowFilter.Expression expression) + public StorageAttachedIndex indexFor(RowFilter.Expression expression) { Set indexes = cfs.indexManager.getBestIndexFor(expression, StorageAttachedIndex.class); + return indexes.isEmpty() ? null : indexes.iterator().next(); + } - return indexes.isEmpty() ? new IndexContext(cfs.getKeyspaceName(), - cfs.getTableName(), - cfs.metadata().partitionKeyType, - cfs.getPartitioner(), - cfs.getComparator(), - expression.column(), - expression.operator().isContainsKey() - ? IndexTarget.Type.KEYS - : IndexTarget.Type.VALUES, - null) - : indexes.iterator().next().getIndexContext(); + public boolean hasAnalyzer(RowFilter.Expression expression) + { + StorageAttachedIndex index = indexFor(expression); + return index != null && index.hasAnalyzer(); } public UnfilteredRowIterator queryStorage(PrimaryKey key, ReadExecutionController executionController) @@ -193,7 +183,7 @@ public class QueryController public KeyRangeIterator.Builder getIndexQueryResults(Collection expressions) { // VSTODO move ANN out of expressions and into its own abstraction? That will help get generic ORDER BY support - expressions = expressions.stream().filter(e -> e.getOp() != Expression.IndexOperator.ANN).collect(Collectors.toList()); + expressions = expressions.stream().filter(e -> e.getIndexOperator() != Expression.IndexOperator.ANN).collect(Collectors.toList()); KeyRangeIterator.Builder builder = KeyRangeIntersectionIterator.builder(expressions.size()); @@ -250,10 +240,11 @@ public class QueryController public KeyRangeIterator getTopKRows(RowFilter.Expression expression) { assert expression.operator() == Operator.ANN; - var planExpression = new Expression(getContext(expression)) - .add(Operator.ANN, expression.getIndexValue().duplicate()); + StorageAttachedIndex index = indexFor(expression); + assert index != null; + var planExpression = Expression.create(index).add(Operator.ANN, expression.getIndexValue().duplicate()); // search memtable before referencing sstable indexes; otherwise we may miss newly flushed memtable index - KeyRangeIterator memtableResults = getContext(expression).getMemtableIndexManager().searchMemtableIndexes(queryContext, planExpression, mergeRange); + KeyRangeIterator memtableResults = index.memtableIndexManager().searchMemtableIndexes(queryContext, planExpression, mergeRange); QueryViewBuilder.QueryView queryView = new QueryViewBuilder(Collections.singleton(planExpression), mergeRange).build(); @@ -286,11 +277,13 @@ public class QueryController // eagerly can save some work when going from PK to row id for on disk segments. // Since the result is shared with multiple streams, we use an unmodifiable list. var sourceKeys = rawSourceKeys.stream().filter(vectorQueryContext::shouldInclude).collect(Collectors.toList()); - var planExpression = new Expression(this.getContext(expression)); + StorageAttachedIndex index = indexFor(expression); + assert index != null : "Cannot do ANN ordering on an unindexed column"; + var planExpression = Expression.create(index); planExpression.add(Operator.ANN, expression.getIndexValue().duplicate()); // search memtable before referencing sstable indexes; otherwise we may miss newly flushed memtable index - KeyRangeIterator memtableResults = this.getContext(expression).getMemtableIndexManager().limitToTopResults(queryContext, sourceKeys, planExpression); + KeyRangeIterator memtableResults = index.memtableIndexManager().limitToTopResults(queryContext, sourceKeys, planExpression); QueryViewBuilder.QueryView queryView = new QueryViewBuilder(Collections.singleton(planExpression), mergeRange).build(); try @@ -298,10 +291,10 @@ public class QueryController List sstableIntersections = queryView.view .stream() .flatMap(pair -> pair.right.stream()) - .map(index -> { + .map(idx -> { try { - return index.limitToTopKResults(queryContext, sourceKeys, planExpression); + return idx.limitToTopKResults(queryContext, sourceKeys, planExpression); } catch (IOException e) { diff --git a/src/java/org/apache/cassandra/index/sai/plan/QueryViewBuilder.java b/src/java/org/apache/cassandra/index/sai/plan/QueryViewBuilder.java index 8557ef5193..0a9691fb3c 100644 --- a/src/java/org/apache/cassandra/index/sai/plan/QueryViewBuilder.java +++ b/src/java/org/apache/cassandra/index/sai/plan/QueryViewBuilder.java @@ -102,7 +102,7 @@ public class QueryViewBuilder { // Non-index column query should only act as FILTER BY for satisfiedBy(Row) method // because otherwise it likely to go through the whole index. - if (expression.context.isNotIndexed()) + if (expression.isNotIndexed()) continue; // If we didn't get a most selective expression then none of the @@ -126,7 +126,7 @@ public class QueryViewBuilder // Finally, we select all the sstable indexes for this expression that // have overlapping keys with the sstable indexes of the most selective // and have a term range that is satisfied by the expression. - View view = expression.context.getView(); + View view = expression.getIndex().view(); Set indexes = new TreeSet<>(SSTableIndex.COMPARATOR); indexes.addAll(view.match(expression) .stream() @@ -157,10 +157,10 @@ public class QueryViewBuilder for (Expression expression : expressions) { - if (expression.context.isNotIndexed()) + if (expression.isNotIndexed()) continue; - View view = expression.context.getView(); + View view = expression.getIndex().view(); NavigableSet indexes = new TreeSet<>(SSTableIndex.COMPARATOR); indexes.addAll(selectIndexesInRange(view.match(expression))); diff --git a/src/java/org/apache/cassandra/index/sai/plan/StorageAttachedIndexQueryPlan.java b/src/java/org/apache/cassandra/index/sai/plan/StorageAttachedIndexQueryPlan.java index 0f1b8aacde..ff55011986 100644 --- a/src/java/org/apache/cassandra/index/sai/plan/StorageAttachedIndexQueryPlan.java +++ b/src/java/org/apache/cassandra/index/sai/plan/StorageAttachedIndexQueryPlan.java @@ -55,7 +55,7 @@ public class StorageAttachedIndexQueryPlan implements Index.QueryPlan this.postIndexFilter = postIndexFilter; this.filterOperation = filterOperation; this.indexes = indexes; - this.isTopK = indexes.stream().anyMatch(i -> i instanceof StorageAttachedIndex && ((StorageAttachedIndex) i).getIndexContext().isVector()); + this.isTopK = indexes.stream().anyMatch(i -> i instanceof StorageAttachedIndex && ((StorageAttachedIndex) i).termType().isVector()); } @Nullable @@ -66,12 +66,25 @@ public class StorageAttachedIndexQueryPlan implements Index.QueryPlan { ImmutableSet.Builder selectedIndexesBuilder = ImmutableSet.builder(); + RowFilter preIndexFilter = rowFilter; + RowFilter postIndexFilter = rowFilter; + for (RowFilter.Expression expression : rowFilter) { - // we ignore user-defined expressions here because we don't have a way to translate their #isSatifiedBy - // method, they will be included in the filter returned by QueryPlan#postIndexQueryFilter() - if (expression.isUserDefined()) + // we ignore any expressions here (currently IN and user-defined expressions) where we don't have a way to + // translate their #isSatifiedBy method, they will be included in the filter returned by QueryPlan#postIndexQueryFilter() + // + // Note: For both the pre- and post-filters we need to check that the expression exists before removing it + // because the without method assert if the expression doesn't exist. This can be the case if we are given + // a duplicate expression - a = 1 and a = 1. The without method removes all instances of the expression. + if (expression.operator().isIN() || expression.isUserDefined()) + { + if (preIndexFilter.getExpressions().contains(expression)) + preIndexFilter = preIndexFilter.without(expression); continue; + } + if (postIndexFilter.getExpressions().contains(expression)) + postIndexFilter = postIndexFilter.without(expression); for (StorageAttachedIndex index : indexes) { @@ -86,13 +99,7 @@ public class StorageAttachedIndexQueryPlan implements Index.QueryPlan if (selectedIndexes.isEmpty()) return null; - /* - * postIndexFilter comprised by those expressions in the read command row filter that can't be handled by - * {@link FilterTree#satisfiedBy(Unfiltered, Row, boolean)}. This includes expressions targeted - * at {@link RowFilter.UserExpression}s. - */ - RowFilter postIndexFilter = rowFilter.restrict(RowFilter.Expression::isUserDefined); - return new StorageAttachedIndexQueryPlan(cfs, queryMetrics, postIndexFilter, rowFilter, selectedIndexes); + return new StorageAttachedIndexQueryPlan(cfs, queryMetrics, postIndexFilter, preIndexFilter, selectedIndexes); } @Override @@ -141,7 +148,7 @@ public class StorageAttachedIndexQueryPlan implements Index.QueryPlan /** * @return a filter with all the expressions that are user-defined or for a non-indexed partition key column - * + *

    * (currently index on partition columns is not supported, see {@link StorageAttachedIndex#validateOptions(Map, TableMetadata)}) */ @Override diff --git a/src/java/org/apache/cassandra/index/sai/plan/StorageAttachedIndexSearcher.java b/src/java/org/apache/cassandra/index/sai/plan/StorageAttachedIndexSearcher.java index 251e79d088..d94ac9118e 100644 --- a/src/java/org/apache/cassandra/index/sai/plan/StorageAttachedIndexSearcher.java +++ b/src/java/org/apache/cassandra/index/sai/plan/StorageAttachedIndexSearcher.java @@ -48,7 +48,6 @@ import org.apache.cassandra.dht.Token; import org.apache.cassandra.exceptions.RequestTimeoutException; import org.apache.cassandra.index.Index; import org.apache.cassandra.index.sai.QueryContext; -import org.apache.cassandra.index.sai.analyzer.AbstractAnalyzer; import org.apache.cassandra.index.sai.metrics.TableQueryMetrics; import org.apache.cassandra.index.sai.iterators.KeyRangeIterator; import org.apache.cassandra.index.sai.utils.PrimaryKey; @@ -84,16 +83,8 @@ public class StorageAttachedIndexSearcher implements Index.Searcher { for (RowFilter.Expression expression : queryController.filterOperation()) { - AbstractAnalyzer analyzer = queryController.getContext(expression).getAnalyzerFactory().create(); - try - { - if (analyzer.transformValue()) - return applyIndexFilter(fullResponse, Operation.buildFilter(queryController), queryContext); - } - finally - { - analyzer.end(); - } + if (queryController.hasAnalyzer(expression)) + return applyIndexFilter(fullResponse, Operation.buildFilter(queryController), queryContext); } // if no analyzer does transformation diff --git a/src/java/org/apache/cassandra/index/sai/plan/VectorTopKProcessor.java b/src/java/org/apache/cassandra/index/sai/plan/VectorTopKProcessor.java index 5ddc6968cf..fb22b949f2 100644 --- a/src/java/org/apache/cassandra/index/sai/plan/VectorTopKProcessor.java +++ b/src/java/org/apache/cassandra/index/sai/plan/VectorTopKProcessor.java @@ -43,12 +43,11 @@ import org.apache.cassandra.db.rows.Row; import org.apache.cassandra.db.rows.Unfiltered; import org.apache.cassandra.index.Index; import org.apache.cassandra.index.SecondaryIndexManager; -import org.apache.cassandra.index.sai.IndexContext; import org.apache.cassandra.index.sai.StorageAttachedIndex; import org.apache.cassandra.index.sai.utils.InMemoryPartitionIterator; import org.apache.cassandra.index.sai.utils.InMemoryUnfilteredPartitionIterator; +import org.apache.cassandra.index.sai.utils.IndexTermType; import org.apache.cassandra.index.sai.utils.PartitionInfo; -import org.apache.cassandra.index.sai.utils.TypeUtil; import org.apache.cassandra.schema.ColumnMetadata; import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.Pair; @@ -65,7 +64,8 @@ import org.apache.cassandra.utils.Pair; public class VectorTopKProcessor { private final ReadCommand command; - private final IndexContext indexContext; + private final StorageAttachedIndex index; + private final IndexTermType indexTermType; private final float[] queryVector; private final int limit; @@ -74,10 +74,11 @@ public class VectorTopKProcessor { this.command = command; - Pair annIndexAndExpression = findTopKIndexContext(); + Pair annIndexAndExpression = findTopKIndex(); Preconditions.checkNotNull(annIndexAndExpression); - this.indexContext = annIndexAndExpression.left; + this.index = annIndexAndExpression.left; + this.indexTermType = annIndexAndExpression.left().termType(); this.queryVector = annIndexAndExpression.right; this.limit = command.limits().count(); } @@ -142,7 +143,7 @@ public class VectorTopKProcessor */ private float getScoreForRow(DecoratedKey key, Row row) { - ColumnMetadata column = indexContext.getDefinition(); + ColumnMetadata column = indexTermType.columnMetadata(); if (column.isPrimaryKeyColumn() && key == null) return 0; @@ -153,17 +154,17 @@ public class VectorTopKProcessor if ((column.isClusteringColumn() || column.isRegular()) && row.isStatic()) return 0; - ByteBuffer value = indexContext.getValueOf(key, row, FBUtilities.nowInSeconds()); + ByteBuffer value = indexTermType.valueOf(key, row, FBUtilities.nowInSeconds()); if (value != null) { - float[] vector = TypeUtil.decomposeVector(indexContext, value); - return indexContext.getIndexWriterConfig().getSimilarityFunction().compare(vector, queryVector); + float[] vector = indexTermType.decomposeVector(value); + return index.indexWriterConfig().getSimilarityFunction().compare(vector, queryVector); } return 0; } - private Pair findTopKIndexContext() + private Pair findTopKIndex() { ColumnFamilyStore cfs = Keyspace.openAndGetStore(command.metadata()); @@ -172,9 +173,8 @@ public class VectorTopKProcessor StorageAttachedIndex sai = findVectorIndexFor(cfs.indexManager, expression); if (sai != null) { - - float[] qv = TypeUtil.decomposeVector(sai.getIndexContext(), expression.getIndexValue().duplicate()); - return Pair.create(sai.getIndexContext(), qv); + float[] qv = sai.termType().decomposeVector(expression.getIndexValue().duplicate()); + return Pair.create(sai, qv); } } diff --git a/src/java/org/apache/cassandra/index/sai/utils/IndexIdentifier.java b/src/java/org/apache/cassandra/index/sai/utils/IndexIdentifier.java new file mode 100644 index 0000000000..f324207d6b --- /dev/null +++ b/src/java/org/apache/cassandra/index/sai/utils/IndexIdentifier.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.index.sai.utils; + +import com.google.common.base.Objects; + +/** + * This is a simple wrapper around the index identity. Its primary purpose is to isolate classes that only need + * access to the identity from the main index classes. This is useful in testing but also makes it easier to pass + * the log message wrapper {@link #logMessage(String)} to classes that don't need any other information about the index. + */ +public class IndexIdentifier +{ + public final String keyspaceName; + public final String tableName; + public final String indexName; + + public IndexIdentifier(String keyspaceName, String tableName, String indexName) + { + this.keyspaceName = keyspaceName; + this.tableName = tableName; + this.indexName = indexName; + } + + /** + * A helper method for constructing consistent log messages for specific column indexes. + *

    + * Example: For the index "idx" in keyspace "ks" on table "tb", calling this method with the raw message + * "Flushing new index segment..." will produce... + *

    + * "[ks.tb.idx] Flushing new index segment..." + * + * @param message The raw content of a logging message, without information identifying it with an index. + * + * @return A log message with the proper keyspace, table and index name prepended to it. + */ + public String logMessage(String message) + { + // Index names are unique only within a keyspace. + return String.format("[%s.%s.%s] %s", keyspaceName, tableName, indexName, message); + } + + @Override + public String toString() + { + return String.format("%s.%s", keyspaceName, indexName); + } + + @Override + public int hashCode() + { + return Objects.hashCode(keyspaceName, tableName, indexName); + } + + @Override + public boolean equals(Object obj) + { + if (this == obj) return true; + if (obj == null || getClass() != obj.getClass()) return false; + IndexIdentifier other = (IndexIdentifier) obj; + return Objects.equal(keyspaceName, other.keyspaceName) && + Objects.equal(tableName, other.tableName) && + Objects.equal(indexName, other.indexName); + } +} diff --git a/src/java/org/apache/cassandra/index/sai/utils/IndexTermType.java b/src/java/org/apache/cassandra/index/sai/utils/IndexTermType.java new file mode 100644 index 0000000000..217680153f --- /dev/null +++ b/src/java/org/apache/cassandra/index/sai/utils/IndexTermType.java @@ -0,0 +1,868 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.index.sai.utils; + +import java.math.BigInteger; +import java.net.InetAddress; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.EnumSet; +import java.util.Iterator; +import java.util.List; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Stream; +import java.util.stream.StreamSupport; + +import com.google.common.base.MoreObjects; +import com.google.common.collect.ImmutableSet; + +import com.googlecode.concurrenttrees.radix.ConcurrentRadixTree; +import org.apache.cassandra.cql3.CQL3Type; +import org.apache.cassandra.cql3.Operator; +import org.apache.cassandra.cql3.statements.schema.IndexTarget; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.filter.RowFilter; +import org.apache.cassandra.db.marshal.AbstractType; +import org.apache.cassandra.db.marshal.AsciiType; +import org.apache.cassandra.db.marshal.BooleanType; +import org.apache.cassandra.db.marshal.ByteBufferAccessor; +import org.apache.cassandra.db.marshal.CollectionType; +import org.apache.cassandra.db.marshal.CompositeType; +import org.apache.cassandra.db.marshal.DecimalType; +import org.apache.cassandra.db.marshal.InetAddressType; +import org.apache.cassandra.db.marshal.IntegerType; +import org.apache.cassandra.db.marshal.LongType; +import org.apache.cassandra.db.marshal.StringType; +import org.apache.cassandra.db.marshal.UTF8Type; +import org.apache.cassandra.db.marshal.UUIDType; +import org.apache.cassandra.db.marshal.VectorType; +import org.apache.cassandra.db.rows.Cell; +import org.apache.cassandra.db.rows.ComplexColumnData; +import org.apache.cassandra.db.rows.Row; +import org.apache.cassandra.index.sai.plan.Expression; +import org.apache.cassandra.schema.ColumnMetadata; +import org.apache.cassandra.serializers.MarshalException; +import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.utils.FastByteOperations; +import org.apache.cassandra.utils.bytecomparable.ByteComparable; +import org.apache.cassandra.utils.bytecomparable.ByteSource; +import org.apache.cassandra.utils.bytecomparable.ByteSourceInverse; + +/** + * This class is a representation of an {@link AbstractType} as an indexable type. It is responsible for determining the + * capabilities of the type and provides helper methods for handling term values associated with the type. + */ +public class IndexTermType +{ + private static final Set> EQ_ONLY_TYPES = ImmutableSet.of(UTF8Type.instance, + AsciiType.instance, + BooleanType.instance, + UUIDType.instance); + + private static final byte[] IPV4_PREFIX = new byte[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, -1 }; + + /** + * DecimalType / BigDecimal values are indexed by truncating their asComparableBytes representation to this size, + * padding on the right with zero-value-bytes until this size is reached (if necessary). This causes + * false-positives that must be filtered in a separate step after hitting the index and reading the associated + * (full) values. + */ + private static final int DECIMAL_APPROXIMATION_BYTES = 24; + private static final int BIG_INTEGER_APPROXIMATION_BYTES = 20; + private static final int INET_ADDRESS_SIZE = 16; + private static final int DEFAULT_FIXED_LENGTH = 16; + + private enum Capability + { + STRING, + VECTOR, + INET_ADDRESS, + BIG_INTEGER, + BIG_DECIMAL, + LONG, + BOOLEAN, + LITERAL, + REVERSED, + FROZEN, + COLLECTION, + NON_FROZEN_COLLECTION, + COMPOSITE, + COMPOSITE_PARTITION + } + + private final ColumnMetadata columnMetadata; + private final IndexTarget.Type indexTargetType; + private final AbstractType indexType; + private final List subTypes; + private final AbstractType vectorElementType; + private final int vectorDimension; + private final EnumSet capabilities; + + /** + * Create an {@link IndexTermType} from a {@link ColumnMetadata} and {@link IndexTarget.Type}. + * + * @param columnMetadata the {@link ColumnMetadata} for the column being indexed + * @param partitionColumns the partition columns for the table this column belongs to. This is used for identifying + * if the {@code columnMetadata} is a partition column and if it belongs to a composite + * partition + * @param indexTargetType the {@link IndexTarget.Type} for the index + * + * @return the {@link IndexTermType} + */ + public static IndexTermType create(ColumnMetadata columnMetadata, List partitionColumns, IndexTarget.Type indexTargetType) + { + return new IndexTermType(columnMetadata, partitionColumns, indexTargetType); + } + + private IndexTermType(ColumnMetadata columnMetadata, List partitionColumns, IndexTarget.Type indexTargetType) + { + this.columnMetadata = columnMetadata; + this.indexTargetType = indexTargetType; + this.capabilities = calculateCapabilities(columnMetadata, partitionColumns, indexTargetType); + this.indexType = calculateIndexType(columnMetadata.type, capabilities, indexTargetType); + if (indexType.subTypes().isEmpty()) + { + this.subTypes = Collections.emptyList(); + } + else + { + List subTypes = new ArrayList<>(indexType.subTypes().size()); + for (AbstractType subType : indexType.subTypes()) + subTypes.add(new IndexTermType(columnMetadata.withNewType(subType), partitionColumns, indexTargetType)); + this.subTypes = Collections.unmodifiableList(subTypes); + } + if (isVector()) + { + VectorType vectorType = (VectorType) indexType; + vectorElementType = vectorType.elementType; + vectorDimension = vectorType.dimension; + } + else + { + vectorElementType = null; + vectorDimension = -1; + } + } + + /** + * Returns {@code true} if the index type is a literal type and will use a literal index. This applies to + * string types, frozen types, composite types and boolean type. + */ + public boolean isLiteral() + { + return capabilities.contains(Capability.LITERAL); + } + + /** + * Returns {@code true} if the index type is a string type. This is used to determine if the type supports + * analysis. + */ + public boolean isString() + { + return capabilities.contains(Capability.STRING); + } + + /** + * Returns {@code true} if the index type is a vector type. Note: being a vector type does not mean that the type + * is valid for indexing in that we don't check the element type and dimension constraints here. + */ + public boolean isVector() + { + return capabilities.contains(Capability.VECTOR); + } + + /** + * Returns {@code true} if the index type is reversed. This is only the case (currently) for clustering keys with + * descending ordering. + */ + public boolean isReversed() + { + return capabilities.contains(Capability.REVERSED); + } + + /** + * Returns {@code true} if the index type is frozen, e.g. the type is wrapped with {@code frozen}. + */ + public boolean isFrozen() + { + return capabilities.contains(Capability.FROZEN); + } + + /** + * Returns {@code true} if the index type is a non-frozen collection + */ + public boolean isNonFrozenCollection() + { + return capabilities.contains(Capability.NON_FROZEN_COLLECTION); + } + + /** + * Returns {@code true} if the index type is a frozen collection. This is the inverse of a non-frozen collection + * but this method is here for clarity. + */ + public boolean isFrozenCollection() + { + return capabilities.contains(Capability.COLLECTION) && capabilities.contains(Capability.FROZEN); + } + + /** + * Returns {@code true} if the index type is a composite type, e.g. it has the form {@code Composite} + */ + public boolean isComposite() + { + return capabilities.contains(Capability.COMPOSITE); + } + + /** + * Returns {@code true} if the {@link RowFilter.Expression} passed is backed by a non-frozen collection and the + * {@code Operator} is one that cannot be merged together. + */ + public boolean isMultiExpression(RowFilter.Expression expression) + { + boolean multiExpression = false; + switch (expression.operator()) + { + case EQ: + multiExpression = isNonFrozenCollection(); + break; + case CONTAINS: + case CONTAINS_KEY: + multiExpression = true; + break; + } + return multiExpression; + } + + /** + * Returns true if given buffer would pass the {@link AbstractType#validate(ByteBuffer)} + * check. False otherwise. + */ + public boolean isValid(ByteBuffer term) + { + try + { + indexType.validate(term); + return true; + } + catch (MarshalException e) + { + return false; + } + } + + public AbstractType indexType() + { + return indexType; + } + + public Collection subTypes() + { + return subTypes; + } + + public CQL3Type asCQL3Type() + { + return indexType.asCQL3Type(); + } + + public ColumnMetadata columnMetadata() + { + return columnMetadata; + } + + public String columnName() + { + return columnMetadata.name.toString(); + } + + public AbstractType vectorElementType() + { + assert isVector(); + + return vectorElementType; + } + + public int vectorDimension() + { + assert isVector(); + + return vectorDimension; + } + + public boolean dependsOn(ColumnMetadata columnMetadata) + { + return this.columnMetadata.compareTo(columnMetadata) == 0; + } + + /** + * Indicates if the type encoding supports rounding of the raw value. + *

    + * This is significant in range searches where we have to make all range + * queries inclusive when searching the indexes in order to avoid excluding + * rounded values. Excluded values are removed by post-filtering. + */ + public boolean supportsRounding() + { + return isBigInteger() || isBigDecimal(); + } + + /** + * Returns the value length for the given {@link AbstractType}, selecting 16 for types + * that officially use VARIABLE_LENGTH but are, in fact, of a fixed length. + */ + public int fixedSizeOf() + { + if (indexType.isValueLengthFixed()) + return indexType.valueLengthIfFixed(); + else if (isInetAddress()) + return INET_ADDRESS_SIZE; + else if (isBigInteger()) + return BIG_INTEGER_APPROXIMATION_BYTES; + else if (isBigDecimal()) + return DECIMAL_APPROXIMATION_BYTES; + return DEFAULT_FIXED_LENGTH; + } + + /** + * Allows overriding the default getString method for {@link CompositeType}. It is + * a requirement of the {@link ConcurrentRadixTree} that the keys are strings but + * the getString method of {@link CompositeType} does not return a string that compares + * in the same order as the underlying {@link ByteBuffer}. To get round this we convert + * the {@link CompositeType} bytes to a hex string. + */ + public String asString(ByteBuffer value) + { + if (isComposite()) + return ByteBufferUtil.bytesToHex(value); + return indexType.getString(value); + } + + /** + * The inverse of the above method. Overrides the fromString method on {@link CompositeType} + * in order to convert the hex string to bytes. + */ + public ByteBuffer fromString(String value) + { + if (isComposite()) + return ByteBufferUtil.hexToBytes(value); + return indexType.fromString(value); + } + + /** + * Returns the cell value from the {@link DecoratedKey} or {@link Row} for the {@link IndexTermType} based on the + * kind of column this {@link IndexTermType} is based on. + * + * @param key the {@link DecoratedKey} of the row + * @param row the {@link Row} containing the non-partition column data + * @param nowInSecs the time that the index write operation started + * + * @return a {@link ByteBuffer} containing the cell value + */ + public ByteBuffer valueOf(DecoratedKey key, Row row, long nowInSecs) + { + if (row == null) + return null; + + switch (columnMetadata.kind) + { + case PARTITION_KEY: + return isCompositePartition() ? CompositeType.extractComponent(key.getKey(), columnMetadata.position()) + : key.getKey(); + case CLUSTERING: + // skip indexing of static clustering when regular column is indexed + return row.isStatic() ? null : row.clustering().bufferAt(columnMetadata.position()); + + // treat static cell retrieval the same was as regular + // only if row kind is STATIC otherwise return null + case STATIC: + if (!row.isStatic()) + return null; + case REGULAR: + Cell cell = row.getCell(columnMetadata); + return cell == null || !cell.isLive(nowInSecs) ? null : cell.buffer(); + + default: + return null; + } + } + + /** + * Returns a value iterator for collection type {@link IndexTermType}s. + * + * @param row the {@link Row} containing the column data + * @param nowInSecs the time that the index write operation started + * + * @return an {@link Iterator} of the collection values + */ + public Iterator valuesOf(Row row, long nowInSecs) + { + if (row == null) + return null; + + switch (columnMetadata.kind) + { + // treat static cell retrieval the same was as regular + // only if row kind is STATIC otherwise return null + case STATIC: + if (!row.isStatic()) + return null; + case REGULAR: + return collectionIterator(row.getComplexColumnData(columnMetadata), nowInSecs); + + default: + return null; + } + } + + public Comparator comparator() + { + // Override the comparator for BigInteger, frozen collections and composite types + if (isBigInteger() || isBigDecimal() || isComposite() || isFrozen()) + return FastByteOperations::compareUnsigned; + + return indexType; + } + + /** + * Compare two terms based on their type. This is used in place of {@link AbstractType#compare(ByteBuffer, ByteBuffer)} + * so that the default comparison can be overridden for specific types. + *

    + * Note: This should be used for all term comparison + */ + public int compare(ByteBuffer b1, ByteBuffer b2) + { + if (isInetAddress()) + return compareInet(b1, b2); + // BigInteger values, frozen types and composite types (map entries) use compareUnsigned to maintain + // a consistent order between the in-memory index and the on-disk index. + else if (isBigInteger() || isBigDecimal() || isComposite() || isFrozen()) + return FastByteOperations.compareUnsigned(b1, b2); + + return indexType.compare(b1, b2 ); + } + + /** + * Returns the smaller of two {@code ByteBuffer} values, based on the result of {@link + * #compare(ByteBuffer, ByteBuffer)} comparision. + */ + public ByteBuffer min(ByteBuffer a, ByteBuffer b) + { + return a == null ? b : (b == null || compare(b, a) > 0) ? a : b; + } + + /** + * Returns the greater of two {@code ByteBuffer} values, based on the result of {@link + * #compare(ByteBuffer, ByteBuffer)} comparision. + */ + public ByteBuffer max(ByteBuffer a, ByteBuffer b) + { + return a == null ? b : (b == null || compare(b, a) < 0) ? a : b; + } + + /** + * This is used for value comparison in post-filtering - {@link Expression#isSatisfiedBy(ByteBuffer)}. + *

    + * This allows types to decide whether they should be compared based on their encoded value or their + * raw value. At present only {@link InetAddressType} values are compared by their encoded values to + * allow for ipv4 -> ipv6 equivalency in searches. + */ + public int comparePostFilter(Expression.Value requestedValue, Expression.Value columnValue) + { + if (isInetAddress()) + return compareInet(requestedValue.encoded, columnValue.encoded); + // Override comparisons for frozen collections and composite types (map entries) + else if (isComposite() || isFrozen()) + return FastByteOperations.compareUnsigned(requestedValue.raw, columnValue.raw); + + return indexType.compare(requestedValue.raw, columnValue.raw); + } + + /** + * Fills a byte array with the comparable bytes for a type. + *

    + * This method expects a {@code value} parameter generated by calling {@link #asIndexBytes(ByteBuffer)}. + * It is not generally safe to pass the output of other serialization methods to this method. For instance, it is + * not generally safe to pass the output of {@link AbstractType#decompose(Object)} as the {@code value} parameter + * (there are certain types for which this is technically OK, but that doesn't hold for all types). + * + * @param value a value buffer returned by {@link #asIndexBytes(ByteBuffer)} + * @param bytes this method's output + */ + public void toComparableBytes(ByteBuffer value, byte[] bytes) + { + if (isInetAddress()) + ByteBufferUtil.copyBytes(value, value.hasArray() ? value.arrayOffset() + value.position() : value.position(), bytes, 0, INET_ADDRESS_SIZE); + else if (isBigInteger()) + ByteBufferUtil.copyBytes(value, value.hasArray() ? value.arrayOffset() + value.position() : value.position(), bytes, 0, BIG_INTEGER_APPROXIMATION_BYTES); + else if (isBigDecimal()) + ByteBufferUtil.copyBytes(value, value.hasArray() ? value.arrayOffset() + value.position() : value.position(), bytes, 0, DECIMAL_APPROXIMATION_BYTES); + else + ByteSourceInverse.copyBytes(asComparableBytes(value, ByteComparable.Version.OSS50), bytes); + } + + public ByteSource asComparableBytes(ByteBuffer value, ByteComparable.Version version) + { + if (isInetAddress() || isBigInteger() || isBigDecimal()) + return ByteSource.optionalFixedLength(ByteBufferAccessor.instance, value); + // The LongType.asComparableBytes uses variableLengthInteger which doesn't play well with + // the balanced tree because it is expecting fixed length data. So for SAI we use a optionalSignedFixedLengthNumber + // to keep all comparable values the same length + else if (isLong()) + return ByteSource.optionalSignedFixedLengthNumber(ByteBufferAccessor.instance, value); + return indexType.asComparableBytes(value, version); + } + + /** + * Translates the external value of specific types into a format used by the index. + */ + public ByteBuffer asIndexBytes(ByteBuffer value) + { + if (value == null) + return null; + + if (isInetAddress()) + return encodeInetAddress(value); + else if (isBigInteger()) + return encodeBigInteger(value); + else if (isBigDecimal()) + return encodeDecimal(value); + return value; + } + + public float[] decomposeVector(ByteBuffer byteBuffer) + { + assert isVector(); + return ((VectorType) indexType).composeAsFloat(byteBuffer); + } + + public boolean supports(Operator operator) + { + if (operator == Operator.LIKE || + operator == Operator.LIKE_CONTAINS || + operator == Operator.LIKE_PREFIX || + operator == Operator.LIKE_MATCHES || + operator == Operator.LIKE_SUFFIX) return false; + + // ANN is only supported against vectors, and vector indexes only support ANN + if (operator == Operator.ANN) + return isVector(); + + Expression.IndexOperator indexOperator = Expression.IndexOperator.valueOf(operator); + + if (isNonFrozenCollection()) + { + if (indexTargetType == IndexTarget.Type.KEYS) return indexOperator == Expression.IndexOperator.CONTAINS_KEY; + if (indexTargetType == IndexTarget.Type.VALUES) return indexOperator == Expression.IndexOperator.CONTAINS_VALUE; + return indexTargetType == IndexTarget.Type.KEYS_AND_VALUES && indexOperator == Expression.IndexOperator.EQ; + } + + if (indexTargetType == IndexTarget.Type.FULL) + return indexOperator == Expression.IndexOperator.EQ; + + if (indexOperator != Expression.IndexOperator.EQ && EQ_ONLY_TYPES.contains(indexType)) return false; + + // RANGE only applicable to non-literal indexes + return (indexOperator != null) && !(isLiteral() && indexOperator == Expression.IndexOperator.RANGE); + } + + @Override + public String toString() + { + return MoreObjects.toStringHelper(this) + .add("column", columnMetadata) + .add("type", indexType) + .add("indexType", indexTargetType) + .toString(); + } + + @Override + public boolean equals(Object obj) + { + if (obj == this) + return true; + + if (!(obj instanceof IndexTermType)) + return false; + + IndexTermType other = (IndexTermType) obj; + + return Objects.equals(columnMetadata, other.columnMetadata) && (indexTargetType == other.indexTargetType); + } + + @Override + public int hashCode() + { + return Objects.hash(columnMetadata, indexTargetType); + } + + private EnumSet calculateCapabilities(ColumnMetadata columnMetadata, List partitionKeyColumns, IndexTarget.Type indexTargetType) + { + EnumSet capabilities = EnumSet.noneOf(Capability.class); + + if (partitionKeyColumns.contains(columnMetadata) && partitionKeyColumns.size() > 1) + capabilities.add(Capability.COMPOSITE_PARTITION); + + AbstractType type = columnMetadata.type; + + if (type.isReversed()) + capabilities.add(Capability.REVERSED); + + AbstractType baseType = type.unwrap(); + + if (baseType.isCollection()) + capabilities.add(Capability.COLLECTION); + + if (baseType.isCollection() && baseType.isMultiCell()) + capabilities.add(Capability.NON_FROZEN_COLLECTION); + + if (!baseType.subTypes().isEmpty() && !baseType.isMultiCell()) + capabilities.add(Capability.FROZEN); + + AbstractType indexType = calculateIndexType(baseType, capabilities, indexTargetType); + + if (indexType instanceof CompositeType) + capabilities.add(Capability.COMPOSITE); + else if (!indexType.subTypes().isEmpty() && !indexType.isMultiCell()) + capabilities.add(Capability.FROZEN); + + if (indexType instanceof StringType) + capabilities.add(Capability.STRING); + + if (indexType instanceof BooleanType) + capabilities.add(Capability.BOOLEAN); + + if (capabilities.contains(Capability.STRING) || + capabilities.contains(Capability.BOOLEAN) || + capabilities.contains(Capability.FROZEN) || + capabilities.contains(Capability.COMPOSITE)) + capabilities.add(Capability.LITERAL); + + if (indexType instanceof VectorType) + capabilities.add(Capability.VECTOR); + + if (indexType instanceof InetAddressType) + capabilities.add(Capability.INET_ADDRESS); + + if (indexType instanceof IntegerType) + capabilities.add(Capability.BIG_INTEGER); + + if (indexType instanceof DecimalType) + capabilities.add(Capability.BIG_DECIMAL); + + if (indexType instanceof LongType) + capabilities.add(Capability.LONG); + + return capabilities; + } + + private AbstractType calculateIndexType(AbstractType baseType, EnumSet capabilities, IndexTarget.Type indexTargetType) + { + return capabilities.contains(Capability.NON_FROZEN_COLLECTION) ? collectionCellValueType(baseType, indexTargetType) : baseType; + } + + private Iterator collectionIterator(ComplexColumnData cellData, long nowInSecs) + { + if (cellData == null) + return null; + + Stream stream = StreamSupport.stream(cellData.spliterator(), false) + .filter(cell -> cell != null && cell.isLive(nowInSecs)) + .map(this::cellValue); + + if (isInetAddress()) + stream = stream.sorted((c1, c2) -> compareInet(encodeInetAddress(c1), encodeInetAddress(c2))); + + return stream.iterator(); + } + + private ByteBuffer cellValue(Cell cell) + { + if (isNonFrozenCollection()) + { + switch (((CollectionType) columnMetadata.type).kind) + { + case LIST: + return cell.buffer(); + case SET: + return cell.path().get(0); + case MAP: + switch (indexTargetType) + { + case KEYS: + return cell.path().get(0); + case VALUES: + return cell.buffer(); + case KEYS_AND_VALUES: + return CompositeType.build(ByteBufferAccessor.instance, cell.path().get(0), cell.buffer()); + } + } + } + return cell.buffer(); + } + + private AbstractType collectionCellValueType(AbstractType type, IndexTarget.Type indexType) + { + CollectionType collection = ((CollectionType) type); + switch (collection.kind) + { + case LIST: + return collection.valueComparator(); + case SET: + return collection.nameComparator(); + case MAP: + switch (indexType) + { + case KEYS: + return collection.nameComparator(); + case VALUES: + return collection.valueComparator(); + case KEYS_AND_VALUES: + return CompositeType.getInstance(collection.nameComparator(), collection.valueComparator()); + } + default: + throw new IllegalArgumentException("Unsupported collection type: " + collection.kind); + } + } + + private boolean isCompositePartition() + { + return capabilities.contains(Capability.COMPOSITE_PARTITION); + } + + /** + * Returns true if given {@link AbstractType} is {@link InetAddressType} + */ + private boolean isInetAddress() + { + return capabilities.contains(Capability.INET_ADDRESS); + } + + /** + * Returns true if given {@link AbstractType} is {@link IntegerType} + */ + private boolean isBigInteger() + { + return capabilities.contains(Capability.BIG_INTEGER); + } + + /** + * Returns true if given {@link AbstractType} is {@link DecimalType} + */ + private boolean isBigDecimal() + { + return capabilities.contains(Capability.BIG_DECIMAL); + } + + private boolean isLong() + { + return capabilities.contains(Capability.LONG); + } + + /** + * Compares 2 InetAddress terms by ensuring that both addresses are represented as + * ipv6 addresses. + */ + private static int compareInet(ByteBuffer b1, ByteBuffer b2) + { + assert isIPv6(b1) && isIPv6(b2); + + return FastByteOperations.compareUnsigned(b1, b2); + } + + private static boolean isIPv6(ByteBuffer address) + { + return address.remaining() == INET_ADDRESS_SIZE; + } + + /** + * Encode a {@link InetAddress} into a fixed width 16 byte encoded value. + *

    + * The encoded value is byte comparable and prefix compressible. + *

    + * The encoding is done by converting ipv4 addresses to their ipv6 equivalent. + */ + private static ByteBuffer encodeInetAddress(ByteBuffer value) + { + if (value.remaining() == 4) + { + int position = value.hasArray() ? value.arrayOffset() + value.position() : value.position(); + ByteBuffer mapped = ByteBuffer.allocate(INET_ADDRESS_SIZE); + System.arraycopy(IPV4_PREFIX, 0, mapped.array(), 0, IPV4_PREFIX.length); + ByteBufferUtil.copyBytes(value, position, mapped, IPV4_PREFIX.length, value.remaining()); + return mapped; + } + return value; + } + + /** + * Encode a {@link BigInteger} into a fixed width 20 byte encoded value. The encoded value is byte comparable + * and prefix compressible. + *

    + * The format of the encoding is: + *

    + * The first 4 bytes contain the integer length of the {@link BigInteger} byte array + * with the top bit flipped for positive values. + *

    + * The remaining 16 bytes contain the 16 most significant bytes of the + * {@link BigInteger} byte array. + *

    + * For {@link BigInteger} values whose underlying byte array is less than + * 16 bytes, the encoded value is sign extended. + */ + public static ByteBuffer encodeBigInteger(ByteBuffer value) + { + int size = value.remaining(); + int position = value.hasArray() ? value.arrayOffset() + value.position() : value.position(); + byte[] bytes = new byte[BIG_INTEGER_APPROXIMATION_BYTES]; + if (size < BIG_INTEGER_APPROXIMATION_BYTES - Integer.BYTES) + { + ByteBufferUtil.copyBytes(value, position, bytes, bytes.length - size, size); + if ((bytes[bytes.length - size] & 0x80) != 0) + Arrays.fill(bytes, Integer.BYTES, bytes.length - size, (byte)0xff); + else + Arrays.fill(bytes, Integer.BYTES, bytes.length - size, (byte)0x00); + } + else + { + ByteBufferUtil.copyBytes(value, position, bytes, Integer.BYTES, BIG_INTEGER_APPROXIMATION_BYTES - Integer.BYTES); + } + if ((bytes[4] & 0x80) != 0) + { + size = -size; + } + bytes[0] = (byte)(size >> 24 & 0xff); + bytes[1] = (byte)(size >> 16 & 0xff); + bytes[2] = (byte)(size >> 8 & 0xff); + bytes[3] = (byte)(size & 0xff); + bytes[0] ^= 0x80; + return ByteBuffer.wrap(bytes); + } + + public static ByteBuffer encodeDecimal(ByteBuffer value) + { + ByteSource bs = DecimalType.instance.asComparableBytes(value, ByteComparable.Version.OSS50); + bs = ByteSource.cutOrRightPad(bs, DECIMAL_APPROXIMATION_BYTES, 0); + return ByteBuffer.wrap(ByteSourceInverse.readBytes(bs, DECIMAL_APPROXIMATION_BYTES)); + } +} diff --git a/src/java/org/apache/cassandra/index/sai/utils/TypeUtil.java b/src/java/org/apache/cassandra/index/sai/utils/TypeUtil.java deleted file mode 100644 index 27ffd0acf1..0000000000 --- a/src/java/org/apache/cassandra/index/sai/utils/TypeUtil.java +++ /dev/null @@ -1,522 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.cassandra.index.sai.utils; - -import java.math.BigInteger; -import java.net.InetAddress; -import java.nio.ByteBuffer; -import java.util.Arrays; -import java.util.Comparator; -import java.util.Iterator; -import java.util.stream.Stream; -import java.util.stream.StreamSupport; - -import com.googlecode.concurrenttrees.radix.ConcurrentRadixTree; -import org.apache.cassandra.cql3.statements.schema.IndexTarget; -import org.apache.cassandra.db.marshal.AbstractType; -import org.apache.cassandra.db.marshal.BooleanType; -import org.apache.cassandra.db.marshal.ByteBufferAccessor; -import org.apache.cassandra.db.marshal.CollectionType; -import org.apache.cassandra.db.marshal.CompositeType; -import org.apache.cassandra.db.marshal.DecimalType; -import org.apache.cassandra.db.marshal.InetAddressType; -import org.apache.cassandra.db.marshal.IntegerType; -import org.apache.cassandra.db.marshal.LongType; -import org.apache.cassandra.db.marshal.ReversedType; -import org.apache.cassandra.db.marshal.StringType; -import org.apache.cassandra.db.marshal.VectorType; -import org.apache.cassandra.db.rows.Cell; -import org.apache.cassandra.db.rows.ComplexColumnData; -import org.apache.cassandra.index.sai.IndexContext; -import org.apache.cassandra.index.sai.plan.Expression; -import org.apache.cassandra.schema.ColumnMetadata; -import org.apache.cassandra.serializers.MarshalException; -import org.apache.cassandra.utils.ByteBufferUtil; -import org.apache.cassandra.utils.FastByteOperations; -import org.apache.cassandra.utils.bytecomparable.ByteComparable; -import org.apache.cassandra.utils.bytecomparable.ByteSource; -import org.apache.cassandra.utils.bytecomparable.ByteSourceInverse; - -public class TypeUtil -{ - private static final byte[] IPV4_PREFIX = new byte[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, -1 }; - - /** - * DecimalType / BigDecimal values are indexed by truncating their asComparableBytes representation to this size, - * padding on the right with zero-value-bytes until this size is reached (if necessary). This causes - * false-positives that must be filtered in a separate step after hitting the index and reading the associated - * (full) values. - */ - public static final int DECIMAL_APPROXIMATION_BYTES = 24; - - public static final int BIG_INTEGER_APPROXIMATION_BYTES = 20; - - public static final int INET_ADDRESS_SIZE = 16; - - public static final int DEFAULT_FIXED_LENGTH = 16; - - private TypeUtil() {} - - /** - * Returns true if given buffer would pass the {@link AbstractType#validate(ByteBuffer)} - * check. False otherwise. - */ - public static boolean isValid(ByteBuffer term, AbstractType validator) - { - try - { - validator.validate(term); - return true; - } - catch (MarshalException e) - { - return false; - } - } - - /** - * Indicates if the type encoding supports rounding of the raw value. - * - * This is significant in range searches where we have to make all range - * queries inclusive when searching the indexes in order to avoid excluding - * rounded values. Excluded values are removed by post-filtering. - */ - public static boolean supportsRounding(AbstractType type) - { - return isBigInteger(type) || isBigDecimal(type); - } - - /** - * Returns the smaller of two {@code ByteBuffer} values, based on the result of {@link - * #compare(ByteBuffer, ByteBuffer, AbstractType)} comparision. - */ - public static ByteBuffer min(ByteBuffer a, ByteBuffer b, AbstractType type) - { - return a == null ? b : (b == null || compare(b, a, type) > 0) ? a : b; - } - - /** - * Returns the greater of two {@code ByteBuffer} values, based on the result of {@link - * #compare(ByteBuffer, ByteBuffer, AbstractType)} comparision. - */ - public static ByteBuffer max(ByteBuffer a, ByteBuffer b, AbstractType type) - { - return a == null ? b : (b == null || compare(b, a, type) < 0) ? a : b; - } - - /** - * Returns the value length for the given {@link AbstractType}, selecting 16 for types - * that officially use VARIABLE_LENGTH but are, in fact, of a fixed length. - */ - public static int fixedSizeOf(AbstractType type) - { - if (type.isValueLengthFixed()) - return type.valueLengthIfFixed(); - else if (isInetAddress(type)) - return INET_ADDRESS_SIZE; - else if (isBigInteger(type)) - return BIG_INTEGER_APPROXIMATION_BYTES; - else if (isBigDecimal(type)) - return DECIMAL_APPROXIMATION_BYTES; - return DEFAULT_FIXED_LENGTH; - } - - public static AbstractType cellValueType(ColumnMetadata columnMetadata, IndexTarget.Type indexType) - { - AbstractType type = columnMetadata.type; - if (isNonFrozenCollection(type)) - { - CollectionType collection = ((CollectionType) type); - switch (collection.kind) - { - case LIST: - return collection.valueComparator(); - case SET: - return collection.nameComparator(); - case MAP: - switch (indexType) - { - case KEYS: - return collection.nameComparator(); - case VALUES: - return collection.valueComparator(); - case KEYS_AND_VALUES: - return CompositeType.getInstance(collection.nameComparator(), collection.valueComparator()); - } - } - } - return type; - } - - /** - * Allows overriding the default getString method for {@link CompositeType}. It is - * a requirement of the {@link ConcurrentRadixTree} that the keys are strings but - * the getString method of {@link CompositeType} does not return a string that compares - * in the same order as the underlying {@link ByteBuffer}. To get round this we convert - * the {@link CompositeType} bytes to a hex string. - */ - public static String getString(ByteBuffer value, AbstractType type) - { - if (isComposite(type)) - return ByteBufferUtil.bytesToHex(value); - return type.getString(value); - } - - /** - * The inverse of the above method. Overrides the fromString method on {@link CompositeType} - * in order to convert the hex string to bytes. - */ - public static ByteBuffer fromString(String value, AbstractType type) - { - if (isComposite(type)) - return ByteBufferUtil.hexToBytes(value); - return type.fromString(value); - } - - public static ByteSource asComparableBytes(ByteBuffer value, AbstractType type, ByteComparable.Version version) - { - if (type instanceof InetAddressType || type instanceof IntegerType || type instanceof DecimalType) - return ByteSource.optionalFixedLength(ByteBufferAccessor.instance, value); - // The LongType.asComparableBytes uses variableLengthInteger which doesn't play well with - // the balanced tree because it is expecting fixed length data. So for SAI we use a optionalSignedFixedLengthNumber - // to keep all comparable values the same length - else if (type instanceof LongType) - return ByteSource.optionalSignedFixedLengthNumber(ByteBufferAccessor.instance, value); - return type.asComparableBytes(value, version); - } - - /** - * Fills a byte array with the comparable bytes for a type. - *

    - * This method expects a {@code value} parameter generated by calling {@link #asIndexBytes(ByteBuffer, AbstractType)}. - * It is not generally safe to pass the output of other serialization methods to this method. For instance, it is - * not generally safe to pass the output of {@link AbstractType#decompose(Object)} as the {@code value} parameter - * (there are certain types for which this is technically OK, but that doesn't hold for all types). - * - * @param value a value buffer returned by {@link #asIndexBytes(ByteBuffer, AbstractType)} - * @param type the type associated with the encoded {@code value} parameter - * @param bytes this method's output - */ - public static void toComparableBytes(ByteBuffer value, AbstractType type, byte[] bytes) - { - if (isInetAddress(type)) - ByteBufferUtil.copyBytes(value, value.hasArray() ? value.arrayOffset() + value.position() : value.position(), bytes, 0, INET_ADDRESS_SIZE); - else if (isBigInteger(type)) - ByteBufferUtil.copyBytes(value, value.hasArray() ? value.arrayOffset() + value.position() : value.position(), bytes, 0, BIG_INTEGER_APPROXIMATION_BYTES); - else if (isBigDecimal(type)) - ByteBufferUtil.copyBytes(value, value.hasArray() ? value.arrayOffset() + value.position() : value.position(), bytes, 0, DECIMAL_APPROXIMATION_BYTES); - else - ByteSourceInverse.copyBytes(asComparableBytes(value, type, ByteComparable.Version.OSS50), bytes); - } - - /** - * Translates the external value of specific types into a format used by the index. - */ - public static ByteBuffer asIndexBytes(ByteBuffer value, AbstractType type) - { - if (value == null) - return null; - - if (isInetAddress(type)) - return encodeInetAddress(value); - else if (isBigInteger(type)) - return encodeBigInteger(value); - else if (type instanceof DecimalType) - return encodeDecimal(value); - return value; - } - - public static float[] decomposeVector(IndexContext indexContext, ByteBuffer byteBuffer) - { - return ((VectorType)indexContext.getValidator()).composeAsFloat(byteBuffer); - } - - /** - * Compare two terms based on their type. This is used in place of {@link AbstractType#compare(ByteBuffer, ByteBuffer)} - * so that the default comparison can be overridden for specific types. - * - * Note: This should be used for all term comparison - */ - public static int compare(ByteBuffer b1, ByteBuffer b2, AbstractType type) - { - if (isInetAddress(type)) - return compareInet(b1, b2); - // BigInteger values, frozen types and composite types (map entries) use compareUnsigned to maintain - // a consistent order between the in-memory index and the on-disk index. - else if (isBigInteger(type) || isBigDecimal(type) || isCompositeOrFrozen(type)) - return FastByteOperations.compareUnsigned(b1, b2); - - return type.compare(b1, b2 ); - } - - /** - * This is used for value comparison in post-filtering - {@link Expression#isSatisfiedBy(ByteBuffer)}. - * - * This allows types to decide whether they should be compared based on their encoded value or their - * raw value. At present only {@link InetAddressType} values are compared by their encoded values to - * allow for ipv4 -> ipv6 equivalency in searches. - */ - public static int comparePostFilter(Expression.Value requestedValue, Expression.Value columnValue, AbstractType type) - { - if (isInetAddress(type)) - return compareInet(requestedValue.encoded, columnValue.encoded); - // Override comparisons for frozen collections and composite types (map entries) - else if (isCompositeOrFrozen(type)) - return FastByteOperations.compareUnsigned(requestedValue.raw, columnValue.raw); - - return type.compare(requestedValue.raw, columnValue.raw); - } - - public static Iterator collectionIterator(AbstractType validator, - ComplexColumnData cellData, - ColumnMetadata columnMetadata, - IndexTarget.Type indexType, - long nowInSecs) - { - if (cellData == null) - return null; - - Stream stream = StreamSupport.stream(cellData.spliterator(), false).filter(cell -> cell != null && cell.isLive(nowInSecs)) - .map(cell -> cellValue(columnMetadata, indexType, cell)); - - if (isInetAddress(validator)) - stream = stream.sorted((c1, c2) -> compareInet(encodeInetAddress(c1), encodeInetAddress(c2))); - - return stream.iterator(); - } - - public static Comparator comparator(AbstractType type) - { - // Override the comparator for BigInteger, frozen collections and composite types - if (isBigInteger(type) || isBigDecimal(type) || isCompositeOrFrozen(type)) - return FastByteOperations::compareUnsigned; - - return type; - } - - private static ByteBuffer cellValue(ColumnMetadata columnMetadata, IndexTarget.Type indexType, Cell cell) - { - if (columnMetadata.type.isCollection() && columnMetadata.type.isMultiCell()) - { - switch (((CollectionType) columnMetadata.type).kind) - { - case LIST: - return cell.buffer(); - case SET: - return cell.path().get(0); - case MAP: - switch (indexType) - { - case KEYS: - return cell.path().get(0); - case VALUES: - return cell.buffer(); - case KEYS_AND_VALUES: - return CompositeType.build(ByteBufferAccessor.instance, cell.path().get(0), cell.buffer()); - } - } - } - return cell.buffer(); - } - - /** - * Compares 2 InetAddress terms by ensuring that both addresses are represented as - * ipv6 addresses. - */ - private static int compareInet(ByteBuffer b1, ByteBuffer b2) - { - assert isIPv6(b1) && isIPv6(b2); - - return FastByteOperations.compareUnsigned(b1, b2); - } - - private static boolean isIPv6(ByteBuffer address) - { - return address.remaining() == INET_ADDRESS_SIZE; - } - - /** - * Encode a {@link InetAddress} into a fixed width 16 byte encoded value. - * - * The encoded value is byte comparable and prefix compressible. - * - * The encoding is done by converting ipv4 addresses to their ipv6 equivalent. - */ - private static ByteBuffer encodeInetAddress(ByteBuffer value) - { - if (value.remaining() == 4) - { - int position = value.hasArray() ? value.arrayOffset() + value.position() : value.position(); - ByteBuffer mapped = ByteBuffer.allocate(INET_ADDRESS_SIZE); - System.arraycopy(IPV4_PREFIX, 0, mapped.array(), 0, IPV4_PREFIX.length); - ByteBufferUtil.copyBytes(value, position, mapped, IPV4_PREFIX.length, value.remaining()); - return mapped; - } - return value; - } - - /** - * Encode a {@link BigInteger} into a fixed width 20 byte encoded value. - * - * The encoded value is byte comparable and prefix compressible. - * - * The format of the encoding is: - * - * The first 4 bytes contain the integer length of the {@link BigInteger} byte array - * with the top bit flipped for positive values. - * - * The remaining 16 bytes contain the 16 most significant bytes of the - * {@link BigInteger} byte array. - * - * For {@link BigInteger} values whose underlying byte array is less than - * 16 bytes, the encoded value is sign extended. - */ - public static ByteBuffer encodeBigInteger(ByteBuffer value) - { - int size = value.remaining(); - int position = value.hasArray() ? value.arrayOffset() + value.position() : value.position(); - byte[] bytes = new byte[BIG_INTEGER_APPROXIMATION_BYTES]; - if (size < BIG_INTEGER_APPROXIMATION_BYTES - Integer.BYTES) - { - ByteBufferUtil.copyBytes(value, position, bytes, bytes.length - size, size); - if ((bytes[bytes.length - size] & 0x80) != 0) - Arrays.fill(bytes, Integer.BYTES, bytes.length - size, (byte)0xff); - else - Arrays.fill(bytes, Integer.BYTES, bytes.length - size, (byte)0x00); - } - else - { - ByteBufferUtil.copyBytes(value, position, bytes, Integer.BYTES, BIG_INTEGER_APPROXIMATION_BYTES - Integer.BYTES); - } - if ((bytes[4] & 0x80) != 0) - { - size = -size; - } - bytes[0] = (byte)(size >> 24 & 0xff); - bytes[1] = (byte)(size >> 16 & 0xff); - bytes[2] = (byte)(size >> 8 & 0xff); - bytes[3] = (byte)(size & 0xff); - bytes[0] ^= 0x80; - return ByteBuffer.wrap(bytes); - } - - /** - * Returns true if values of the given {@link AbstractType} should be indexed as literals. - */ - public static boolean isLiteral(AbstractType type) - { - return isString(type) || isCompositeOrFrozen(type) || baseType(type) instanceof BooleanType; - } - - /** - * Returns true if given {@link AbstractType} is based on a string, e.g. UTF8 or Ascii - */ - public static boolean isString(AbstractType type) - { - type = baseType(type); - return type instanceof StringType; - } - - /** - * Returns true if given {@link AbstractType} is a Composite(map entry) or frozen. - */ - public static boolean isCompositeOrFrozen(AbstractType type) - { - type = baseType(type); - return type instanceof CompositeType || isFrozen(type); - } - - /** - * Returns true if given {@link AbstractType} is frozen. - */ - public static boolean isFrozen(AbstractType type) - { - type = baseType(type); - return !type.subTypes().isEmpty() && !type.isMultiCell(); - } - - /** - * Returns true if given {@link AbstractType} is a frozen collection. - */ - public static boolean isFrozenCollection(AbstractType type) - { - type = baseType(type); - return type.isCollection() && !type.isMultiCell(); - } - - /** - * Returns true if given {@link AbstractType} is a non-frozen collection. - */ - public static boolean isNonFrozenCollection(AbstractType type) - { - type = baseType(type); - return type.isCollection() && type.isMultiCell(); - } - - /** - * Returns true if given {@link AbstractType} is {@link InetAddressType} - */ - private static boolean isInetAddress(AbstractType type) - { - type = baseType(type); - return type instanceof InetAddressType; - } - - /** - * Returns true if given {@link AbstractType} is {@link IntegerType} - */ - private static boolean isBigInteger(AbstractType type) - { - type = baseType(type); - return type instanceof IntegerType; - } - - /** - * Returns true if given {@link AbstractType} is {@link DecimalType} - */ - private static boolean isBigDecimal(AbstractType type) - { - type = baseType(type); - return type instanceof DecimalType; - } - - /** - * Returns true if given {@link AbstractType} is {@link CompositeType} - */ - public static boolean isComposite(AbstractType type) - { - type = baseType(type); - return type instanceof CompositeType; - } - - /** - * @return base type if given type is reversed, otherwise return itself - */ - private static AbstractType baseType(AbstractType type) - { - return type.isReversed() ? ((ReversedType) type).baseType : type; - } - - public static ByteBuffer encodeDecimal(ByteBuffer value) - { - ByteSource bs = DecimalType.instance.asComparableBytes(value, ByteComparable.Version.OSS50); - bs = ByteSource.cutOrRightPad(bs, DECIMAL_APPROXIMATION_BYTES, 0); - return ByteBuffer.wrap(ByteSourceInverse.readBytes(bs, DECIMAL_APPROXIMATION_BYTES)); - } -} diff --git a/src/java/org/apache/cassandra/index/sai/view/IndexViewManager.java b/src/java/org/apache/cassandra/index/sai/view/IndexViewManager.java index 42cb14dc0c..a6237b45dc 100644 --- a/src/java/org/apache/cassandra/index/sai/view/IndexViewManager.java +++ b/src/java/org/apache/cassandra/index/sai/view/IndexViewManager.java @@ -28,9 +28,9 @@ import java.util.concurrent.atomic.AtomicReference; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.cassandra.index.sai.IndexContext; import org.apache.cassandra.index.sai.IndexValidation; import org.apache.cassandra.index.sai.SSTableContext; +import org.apache.cassandra.index.sai.StorageAttachedIndex; import org.apache.cassandra.index.sai.disk.SSTableIndex; import org.apache.cassandra.index.sai.StorageAttachedIndexGroup; import org.apache.cassandra.io.sstable.format.SSTableReader; @@ -38,7 +38,7 @@ import org.apache.cassandra.utils.Pair; /** * Maintain an atomic view for read requests, so that requests can read all data during concurrent compactions. - * + *

    * All per-column {@link SSTableIndex} updates should be proxied by {@link StorageAttachedIndexGroup} to make * sure per-sstable {@link SSTableContext} are in-sync. */ @@ -46,16 +46,16 @@ public class IndexViewManager { private static final Logger logger = LoggerFactory.getLogger(IndexViewManager.class); - private final IndexContext context; + private final StorageAttachedIndex index; private final AtomicReference view = new AtomicReference<>(); - public IndexViewManager(IndexContext context) + public IndexViewManager(StorageAttachedIndex index) { - this.context = context; - this.view.set(new View(context, Collections.emptySet())); + this.index = index; + this.view.set(new View(index.termType(), Collections.emptySet())); } - public View getView() + public View view() { return view.get(); } @@ -72,7 +72,7 @@ public class IndexViewManager public Collection update(Collection oldSSTables, Collection newSSTableContexts, IndexValidation validation) { // Valid indexes on the left and invalid SSTable contexts on the right... - Pair, Collection> indexes = context.getBuiltIndexes(newSSTableContexts, validation); + Pair, Collection> indexes = getBuiltIndexes(newSSTableContexts, validation); View currentView, newView; Collection newViewIndexes = new HashSet<>(); @@ -104,14 +104,14 @@ public class IndexViewManager newViewIndexes.add(sstableIndex); } - newView = new View(context, newViewIndexes); + newView = new View(index.termType(), newViewIndexes); } while (!view.compareAndSet(currentView, newView)); releasableIndexes.forEach(SSTableIndex::release); if (logger.isTraceEnabled()) - logger.trace(context.logMessage("There are now {} active SSTable indexes."), view.get().getIndexes().size()); + logger.trace(index.identifier().logMessage("There are now {} active SSTable indexes."), view.get().getIndexes().size()); return indexes.right; } @@ -139,11 +139,70 @@ public class IndexViewManager */ public void invalidate() { - View previousView = view.getAndSet(new View(context, Collections.emptyList())); + View previousView = view.getAndSet(new View(index.termType(), Collections.emptyList())); for (SSTableIndex index : previousView) { index.markObsolete(); } } + + /** + * @return the indexes that are built on the given SSTables on the left and corrupted indexes' + * corresponding contexts on the right + */ + private Pair, Collection> getBuiltIndexes(Collection sstableContexts, IndexValidation validation) + { + Set valid = new HashSet<>(sstableContexts.size()); + Set invalid = new HashSet<>(); + + for (SSTableContext sstableContext : sstableContexts) + { + if (sstableContext.sstable.isMarkedCompacted()) + continue; + + if (!sstableContext.indexDescriptor.isPerColumnIndexBuildComplete(index.identifier())) + { + logger.debug(index.identifier().logMessage("An on-disk index build for SSTable {} has not completed."), sstableContext.descriptor()); + continue; + } + + if (sstableContext.indexDescriptor.isIndexEmpty(index.termType(), index.identifier())) + { + logger.debug(index.identifier().logMessage("No on-disk index was built for SSTable {} because the SSTable " + + "had no indexable rows for the index."), sstableContext.descriptor()); + continue; + } + + try + { + if (validation != IndexValidation.NONE) + { + if (!sstableContext.indexDescriptor.validatePerIndexComponents(index.termType(), index.identifier(), validation)) + { + invalid.add(sstableContext); + continue; + } + } + + SSTableIndex ssTableIndex = sstableContext.newSSTableIndex(index); + logger.debug(index.identifier().logMessage("Successfully created index for SSTable {}."), sstableContext.descriptor()); + + // Try to add new index to the set, if set already has such index, we'll simply release and move on. + // This covers situation when SSTable collection has the same SSTable multiple + // times because we don't know what kind of collection it actually is. + if (!valid.add(ssTableIndex)) + { + ssTableIndex.release(); + } + } + catch (Throwable e) + { + logger.warn(index.identifier().logMessage("Failed to update per-column components for SSTable {}"), sstableContext.descriptor(), e); + invalid.add(sstableContext); + } + } + + return Pair.create(valid, invalid); + } } diff --git a/src/java/org/apache/cassandra/index/sai/view/RangeTermTree.java b/src/java/org/apache/cassandra/index/sai/view/RangeTermTree.java index 93f07f99c4..b2b1803e37 100644 --- a/src/java/org/apache/cassandra/index/sai/view/RangeTermTree.java +++ b/src/java/org/apache/cassandra/index/sai/view/RangeTermTree.java @@ -26,11 +26,9 @@ import com.google.common.base.MoreObjects; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.cassandra.db.marshal.AbstractType; -import org.apache.cassandra.index.sai.IndexContext; import org.apache.cassandra.index.sai.disk.SSTableIndex; import org.apache.cassandra.index.sai.plan.Expression; -import org.apache.cassandra.index.sai.utils.TypeUtil; +import org.apache.cassandra.index.sai.utils.IndexTermType; import org.apache.cassandra.utils.Interval; import org.apache.cassandra.utils.IntervalTree; @@ -39,60 +37,59 @@ public class RangeTermTree private static final Logger logger = LoggerFactory.getLogger(RangeTermTree.class); protected final ByteBuffer min, max; - protected final AbstractType comparator; + protected final IndexTermType indexTermType; private final IntervalTree> rangeTree; - private RangeTermTree(ByteBuffer min, ByteBuffer max, IntervalTree> rangeTree, AbstractType comparator) + private RangeTermTree(ByteBuffer min, ByteBuffer max, IntervalTree> rangeTree, IndexTermType indexTermType) { this.min = min; this.max = max; this.rangeTree = rangeTree; - this.comparator = comparator; + this.indexTermType = indexTermType; } public List search(Expression e) { - ByteBuffer minTerm = e.lower == null ? min : e.lower.value.encoded; - ByteBuffer maxTerm = e.upper == null ? max : e.upper.value.encoded; + ByteBuffer minTerm = e.lower() == null ? min : e.lower().value.encoded; + ByteBuffer maxTerm = e.upper() == null ? max : e.upper().value.encoded; - return rangeTree.search(Interval.create(new Term(minTerm, comparator), - new Term(maxTerm, comparator), + return rangeTree.search(Interval.create(new Term(minTerm, indexTermType), + new Term(maxTerm, indexTermType), null)); } static class Builder { - private final AbstractType comparator; + private final IndexTermType indexTermType; private ByteBuffer min, max; final List> intervals = new ArrayList<>(); - protected Builder(AbstractType comparator) + protected Builder(IndexTermType indexTermType) { - this.comparator = comparator; + this.indexTermType = indexTermType; } public final void add(SSTableIndex index) { addIndex(index); - min = min == null || TypeUtil.compare(min, index.minTerm(), comparator) > 0 ? index.minTerm() : min; - max = max == null || TypeUtil.compare(max, index.maxTerm(), comparator) < 0 ? index.maxTerm() : max; + min = min == null || index.getIndexTermType().compare(min, index.minTerm()) > 0 ? index.minTerm() : min; + max = max == null || index.getIndexTermType().compare(max, index.maxTerm()) < 0 ? index.maxTerm() : max; } public void addIndex(SSTableIndex index) { Interval interval = - Interval.create(new Term(index.minTerm(), comparator), new Term(index.maxTerm(), comparator), index); + Interval.create(new Term(index.minTerm(), indexTermType), new Term(index.maxTerm(), indexTermType), index); if (logger.isTraceEnabled()) { - IndexContext context = index.getIndexContext(); - logger.trace(context.logMessage("Adding index for SSTable {} with minTerm={} and maxTerm={}..."), - index.getSSTable().descriptor, - comparator.compose(index.minTerm()), - comparator.compose(index.maxTerm())); + logger.trace(index.getIndexIdentifier().logMessage("Adding index for SSTable {} with minTerm={} and maxTerm={}..."), + index.getSSTable().descriptor, + indexTermType.indexType().compose(index.minTerm()), + indexTermType.indexType().compose(index.maxTerm())); } intervals.add(interval); @@ -100,7 +97,7 @@ public class RangeTermTree public RangeTermTree build() { - return new RangeTermTree(min, max, IntervalTree.build(intervals), comparator); + return new RangeTermTree(min, max, IntervalTree.build(intervals), indexTermType); } } @@ -111,24 +108,24 @@ public class RangeTermTree protected static class Term implements Comparable { private final ByteBuffer term; - private final AbstractType comparator; + private final IndexTermType indexTermType; - Term(ByteBuffer term, AbstractType comparator) + Term(ByteBuffer term, IndexTermType indexTermType) { this.term = term; - this.comparator = comparator; + this.indexTermType = indexTermType; } @Override public int compareTo(Term o) { - return TypeUtil.compare(term, o.term, comparator); + return indexTermType.compare(term, o.term); } @Override public String toString() { - return MoreObjects.toStringHelper(this).add("term", comparator.getString(term)).toString(); + return MoreObjects.toStringHelper(this).add("term", indexTermType.asString(term)).toString(); } } } diff --git a/src/java/org/apache/cassandra/index/sai/view/View.java b/src/java/org/apache/cassandra/index/sai/view/View.java index 4b528bd6ad..323eb24a7f 100644 --- a/src/java/org/apache/cassandra/index/sai/view/View.java +++ b/src/java/org/apache/cassandra/index/sai/view/View.java @@ -23,16 +23,15 @@ import java.util.HashMap; import java.util.Iterator; import java.util.Map; -import org.apache.cassandra.db.marshal.AbstractType; -import org.apache.cassandra.index.sai.IndexContext; import org.apache.cassandra.index.sai.disk.SSTableIndex; import org.apache.cassandra.index.sai.plan.Expression; +import org.apache.cassandra.index.sai.utils.IndexTermType; import org.apache.cassandra.io.sstable.Descriptor; import org.apache.cassandra.io.sstable.format.SSTableReader; /** * The View is an immutable, point in time, view of the avalailable {@link SSTableIndex}es for an index. - * + *

    * The view maintains a {@link RangeTermTree} for querying the view by value range. This is used by the * {@link org.apache.cassandra.index.sai.plan.QueryViewBuilder} to select the set of {@link SSTableIndex}es * to perform a query without needing to query indexes that are known not to contain to the requested @@ -44,13 +43,11 @@ public class View implements Iterable private final RangeTermTree rangeTermTree; - public View(IndexContext context, Collection indexes) + public View(IndexTermType indexTermType, Collection indexes) { this.view = new HashMap<>(); - AbstractType termValidator = context.getValidator(); - - RangeTermTree.Builder rangeTermTreeBuilder = new RangeTermTree.Builder(termValidator); + RangeTermTree.Builder rangeTermTreeBuilder = new RangeTermTree.Builder(indexTermType); for (SSTableIndex sstableIndex : indexes) { @@ -67,7 +64,7 @@ public class View implements Iterable */ public Collection match(Expression expression) { - if (expression.getOp() == Expression.IndexOperator.ANN) + if (expression.getIndexOperator() == Expression.IndexOperator.ANN) return getIndexes(); return rangeTermTree.search(expression); diff --git a/src/java/org/apache/cassandra/index/sai/virtual/ColumnIndexesSystemView.java b/src/java/org/apache/cassandra/index/sai/virtual/ColumnIndexesSystemView.java index 7430ce3412..2d2e5e2634 100644 --- a/src/java/org/apache/cassandra/index/sai/virtual/ColumnIndexesSystemView.java +++ b/src/java/org/apache/cassandra/index/sai/virtual/ColumnIndexesSystemView.java @@ -27,9 +27,7 @@ import org.apache.cassandra.db.virtual.SimpleDataSet; import org.apache.cassandra.db.virtual.VirtualTable; import org.apache.cassandra.dht.LocalPartitioner; import org.apache.cassandra.exceptions.InvalidRequestException; -import org.apache.cassandra.index.Index; import org.apache.cassandra.index.SecondaryIndexManager; -import org.apache.cassandra.index.sai.IndexContext; import org.apache.cassandra.index.sai.StorageAttachedIndex; import org.apache.cassandra.index.sai.StorageAttachedIndexGroup; import org.apache.cassandra.schema.Schema; @@ -92,19 +90,18 @@ public class ColumnIndexesSystemView extends AbstractVirtualTable if (group != null) { - for (Index index : group.getIndexes()) - { - IndexContext context = ((StorageAttachedIndex) index).getIndexContext(); - String indexName = context.getIndexName(); + group.getIndexes().forEach(i -> { + StorageAttachedIndex index = (StorageAttachedIndex) i; + String indexName = index.identifier().indexName; dataset.row(ks, indexName) .column(TABLE_NAME, cfs.name) - .column(COLUMN_NAME, context.getColumnName()) + .column(COLUMN_NAME, index.termType().columnName()) .column(IS_QUERYABLE, manager.isIndexQueryable(index)) .column(IS_BUILDING, manager.isIndexBuilding(indexName)) - .column(IS_STRING, context.isLiteral()) - .column(ANALYZER, context.getAnalyzerFactory().toString()); - } + .column(IS_STRING, index.termType().isLiteral()) + .column(ANALYZER, index.hasAnalyzer() ? index.analyzer().toString() : "NoOpAnalyzer"); + }); } } } diff --git a/src/java/org/apache/cassandra/index/sai/virtual/SSTableIndexesSystemView.java b/src/java/org/apache/cassandra/index/sai/virtual/SSTableIndexesSystemView.java index 4d72d01db1..7bc5426195 100644 --- a/src/java/org/apache/cassandra/index/sai/virtual/SSTableIndexesSystemView.java +++ b/src/java/org/apache/cassandra/index/sai/virtual/SSTableIndexesSystemView.java @@ -27,8 +27,6 @@ import org.apache.cassandra.db.virtual.VirtualTable; import org.apache.cassandra.dht.AbstractBounds; import org.apache.cassandra.dht.LocalPartitioner; import org.apache.cassandra.dht.Token; -import org.apache.cassandra.index.Index; -import org.apache.cassandra.index.sai.IndexContext; import org.apache.cassandra.index.sai.StorageAttachedIndex; import org.apache.cassandra.index.sai.StorageAttachedIndexGroup; import org.apache.cassandra.index.sai.disk.SSTableIndex; @@ -99,19 +97,18 @@ public class SSTableIndexesSystemView extends AbstractVirtualTable { Token.TokenFactory tokenFactory = cfs.metadata().partitioner.getTokenFactory(); - for (Index index : group.getIndexes()) - { - IndexContext indexContext = ((StorageAttachedIndex)index).getIndexContext(); + group.getIndexes().forEach(i -> { + StorageAttachedIndex index = (StorageAttachedIndex)i; - for (SSTableIndex sstableIndex : indexContext.getView()) + for (SSTableIndex sstableIndex : index.view()) { SSTableReader sstable = sstableIndex.getSSTable(); Descriptor descriptor = sstable.descriptor; AbstractBounds bounds = sstable.getBounds(); - dataset.row(ks, indexContext.getIndexName(), sstable.getFilename()) + dataset.row(ks, index.identifier().indexName, sstable.getFilename()) .column(TABLE_NAME, descriptor.cfname) - .column(COLUMN_NAME, indexContext.getColumnName()) + .column(COLUMN_NAME, index.termType().columnName()) .column(FORMAT_VERSION, sstableIndex.getVersion().toString()) .column(CELL_COUNT, sstableIndex.getRowCount()) .column(MIN_ROW_ID, sstableIndex.minSSTableRowId()) @@ -121,7 +118,7 @@ public class SSTableIndexesSystemView extends AbstractVirtualTable .column(PER_TABLE_DISK_SIZE, sstableIndex.getSSTableContext().diskUsage()) .column(PER_COLUMN_DISK_SIZE, sstableIndex.sizeOfPerColumnComponents()); } - } + }); } } } diff --git a/src/java/org/apache/cassandra/index/sai/virtual/SegmentsSystemView.java b/src/java/org/apache/cassandra/index/sai/virtual/SegmentsSystemView.java index 3829ef305e..d3206f1766 100644 --- a/src/java/org/apache/cassandra/index/sai/virtual/SegmentsSystemView.java +++ b/src/java/org/apache/cassandra/index/sai/virtual/SegmentsSystemView.java @@ -28,8 +28,6 @@ import org.apache.cassandra.db.virtual.AbstractVirtualTable; import org.apache.cassandra.db.virtual.SimpleDataSet; import org.apache.cassandra.db.virtual.VirtualTable; import org.apache.cassandra.dht.LocalPartitioner; -import org.apache.cassandra.index.Index; -import org.apache.cassandra.index.sai.IndexContext; import org.apache.cassandra.index.sai.StorageAttachedIndex; import org.apache.cassandra.index.sai.StorageAttachedIndexGroup; import org.apache.cassandra.index.sai.disk.SSTableIndex; @@ -89,8 +87,8 @@ public class SegmentsSystemView extends AbstractVirtualTable { SimpleDataSet dataset = new SimpleDataSet(metadata()); - forEachIndex(indexContext -> { - for (SSTableIndex sstableIndex : indexContext.getView()) + forEachIndex(index -> { + for (SSTableIndex sstableIndex : index.view()) { sstableIndex.populateSegmentView(dataset); } @@ -99,7 +97,7 @@ public class SegmentsSystemView extends AbstractVirtualTable return dataset; } - private void forEachIndex(Consumer process) + private void forEachIndex(Consumer process) { for (String ks : Schema.instance.getUserKeyspaces()) { @@ -112,12 +110,7 @@ public class SegmentsSystemView extends AbstractVirtualTable StorageAttachedIndexGroup group = StorageAttachedIndexGroup.getIndexGroup(cfs); if (group != null) - { - for (Index index : group.getIndexes()) - { - process.accept(((StorageAttachedIndex)index).getIndexContext()); - } - } + group.getIndexes().stream().map(index -> (StorageAttachedIndex) index).forEach(process); } } } diff --git a/test/distributed/org/apache/cassandra/distributed/test/sai/IndexStreamingFailureTest.java b/test/distributed/org/apache/cassandra/distributed/test/sai/IndexStreamingFailureTest.java index 00ca972826..533740ed0c 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/sai/IndexStreamingFailureTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/sai/IndexStreamingFailureTest.java @@ -21,33 +21,32 @@ package org.apache.cassandra.distributed.test.sai; import java.io.IOException; import java.util.concurrent.Callable; -import net.bytebuddy.ByteBuddy; -import net.bytebuddy.dynamic.loading.ClassLoadingStrategy; -import net.bytebuddy.implementation.MethodDelegation; -import net.bytebuddy.implementation.bind.annotation.SuperCall; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; +import net.bytebuddy.ByteBuddy; +import net.bytebuddy.dynamic.loading.ClassLoadingStrategy; +import net.bytebuddy.implementation.MethodDelegation; +import net.bytebuddy.implementation.bind.annotation.SuperCall; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.distributed.Cluster; import org.apache.cassandra.distributed.api.IInvokableInstance; import org.apache.cassandra.distributed.test.TestBaseImpl; -import org.apache.cassandra.index.sai.IndexContext; import org.apache.cassandra.index.sai.disk.format.IndexDescriptor; import org.apache.cassandra.index.sai.disk.v1.SAICodecUtils; import org.apache.cassandra.index.sai.disk.v1.segment.SegmentBuilder; import org.apache.cassandra.index.sai.disk.v1.segment.SegmentMetadata; +import org.apache.cassandra.index.sai.utils.IndexIdentifier; import org.apache.cassandra.utils.Throwables; import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.store.IndexInput; import static net.bytebuddy.matcher.ElementMatchers.named; -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.Assert.assertFalse; - import static org.apache.cassandra.distributed.api.Feature.GOSSIP; import static org.apache.cassandra.distributed.api.Feature.NETWORK; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.Assert.assertFalse; public class IndexStreamingFailureTest extends TestBaseImpl { @@ -164,7 +163,7 @@ public class IndexStreamingFailureTest extends TestBaseImpl } @SuppressWarnings("unused") - public static SegmentMetadata flush(IndexDescriptor indexDescriptor, IndexContext indexContext, @SuperCall Callable zuper) throws IOException + public static SegmentMetadata flush(IndexDescriptor indexDescriptor, IndexIdentifier indexIdentifier, @SuperCall Callable zuper) throws IOException { if (failFlush) throw new IOException(TEST_ERROR_MESSAGE); diff --git a/test/microbench/org/apache/cassandra/test/microbench/sai/KeyLookupBench.java b/test/microbench/org/apache/cassandra/test/microbench/sai/KeyLookupBench.java index 65bded3a45..78f700d83f 100644 --- a/test/microbench/org/apache/cassandra/test/microbench/sai/KeyLookupBench.java +++ b/test/microbench/org/apache/cassandra/test/microbench/sai/KeyLookupBench.java @@ -37,7 +37,6 @@ import org.apache.cassandra.index.sai.disk.format.IndexDescriptor; import org.apache.cassandra.index.sai.disk.v1.SSTableComponentsWriter; import org.apache.cassandra.index.sai.disk.v1.WidePrimaryKeyMap; import org.apache.cassandra.index.sai.utils.PrimaryKey; -import org.apache.cassandra.index.sai.utils.TypeUtil; import org.apache.cassandra.io.sstable.Descriptor; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.io.util.File; @@ -173,7 +172,7 @@ public class KeyLookupBench private static DecoratedKey makeKey(TableMetadata table, Object...partitionKeys) { ByteBuffer key; - if (TypeUtil.isComposite(table.partitionKeyType)) + if (table.partitionKeyType instanceof CompositeType) key = ((CompositeType)table.partitionKeyType).decompose(partitionKeys); else key = table.partitionKeyType.fromString((String)partitionKeys[0]); diff --git a/test/unit/org/apache/cassandra/index/sai/SAITester.java b/test/unit/org/apache/cassandra/index/sai/SAITester.java index e043d6602f..eead407567 100644 --- a/test/unit/org/apache/cassandra/index/sai/SAITester.java +++ b/test/unit/org/apache/cassandra/index/sai/SAITester.java @@ -31,8 +31,10 @@ import java.nio.file.StandardOpenOption; import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.Random; import java.util.Set; @@ -70,16 +72,18 @@ import org.apache.cassandra.db.Directories; import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.db.compaction.CompactionManager; import org.apache.cassandra.db.marshal.AbstractType; -import org.apache.cassandra.db.marshal.UTF8Type; +import org.apache.cassandra.db.marshal.Int32Type; import org.apache.cassandra.dht.Murmur3Partitioner; import org.apache.cassandra.index.Index; import org.apache.cassandra.index.sai.disk.SSTableIndex; import org.apache.cassandra.index.sai.disk.format.IndexComponent; import org.apache.cassandra.index.sai.disk.format.IndexDescriptor; +import org.apache.cassandra.index.sai.utils.IndexIdentifier; import org.apache.cassandra.index.sai.disk.format.OnDiskFormat; import org.apache.cassandra.index.sai.disk.format.Version; import org.apache.cassandra.index.sai.disk.v1.V1OnDiskFormat; import org.apache.cassandra.index.sai.disk.v1.segment.SegmentBuilder; +import org.apache.cassandra.index.sai.utils.IndexTermType; import org.apache.cassandra.index.sai.utils.PrimaryKey; import org.apache.cassandra.index.sai.utils.ResourceLeakDetector; import org.apache.cassandra.inject.Injection; @@ -90,9 +94,12 @@ import org.apache.cassandra.io.sstable.format.SSTableFormat; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.io.sstable.format.TOCComponent; import org.apache.cassandra.io.util.File; +import org.apache.cassandra.schema.CachingParams; import org.apache.cassandra.schema.ColumnMetadata; import org.apache.cassandra.schema.IndexMetadata; +import org.apache.cassandra.schema.MockSchema; import org.apache.cassandra.schema.Schema; +import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.service.StorageService; import org.apache.cassandra.service.snapshot.TableSnapshot; import org.apache.cassandra.utils.JVMStabilityInspector; @@ -250,16 +257,59 @@ public abstract class SAITester extends CQLTester public abstract void corrupt(File file) throws IOException; } - public static IndexContext createIndexContext(String name, AbstractType validator) + public static StorageAttachedIndex createMockIndex(ColumnMetadata column) { - return new IndexContext("test_ks", - "test_cf", - UTF8Type.instance, - Murmur3Partitioner.instance, - new ClusteringComparator(), - ColumnMetadata.regularColumn("sai", "internal", name, validator), - IndexTarget.Type.SIMPLE, - IndexMetadata.fromSchemaMetadata(name, IndexMetadata.Kind.CUSTOM, null)); + TableMetadata table = TableMetadata.builder(column.ksName, column.cfName) + .addPartitionKeyColumn("pk", Int32Type.instance) + .addRegularColumn(column.name, column.type) + .partitioner(Murmur3Partitioner.instance) + .caching(CachingParams.CACHE_NOTHING) + .build(); + + Map options = new HashMap<>(); + options.put(IndexTarget.CUSTOM_INDEX_OPTION_NAME, StorageAttachedIndex.class.getCanonicalName()); + options.put("target", column.name.toString()); + + IndexMetadata indexMetadata = IndexMetadata.fromSchemaMetadata(column.name.toString(), IndexMetadata.Kind.CUSTOM, options); + + ColumnFamilyStore cfs = MockSchema.newCFS(table); + + return new StorageAttachedIndex(cfs, indexMetadata); + } + + public static StorageAttachedIndex createMockIndex(String columnName, AbstractType cellType) + { + TableMetadata table = TableMetadata.builder("test", "test") + .addPartitionKeyColumn("pk", Int32Type.instance) + .addRegularColumn(columnName, cellType) + .partitioner(Murmur3Partitioner.instance) + .caching(CachingParams.CACHE_NOTHING) + .build(); + + Map options = new HashMap<>(); + options.put(IndexTarget.CUSTOM_INDEX_OPTION_NAME, StorageAttachedIndex.class.getCanonicalName()); + options.put("target", columnName); + + IndexMetadata indexMetadata = IndexMetadata.fromSchemaMetadata(columnName, IndexMetadata.Kind.CUSTOM, options); + + ColumnFamilyStore cfs = MockSchema.newCFS(table); + + return new StorageAttachedIndex(cfs, indexMetadata); + } + + public static IndexTermType createIndexTermType(AbstractType cellType) + { + return IndexTermType.create(ColumnMetadata.regularColumn("sai", "internal", "val", cellType), Collections.emptyList(), IndexTarget.Type.SIMPLE); + } + + public IndexIdentifier createIndexIdentifier(String indexName) + { + return createIndexIdentifier(keyspace(), currentTable(), indexName); + } + + public static IndexIdentifier createIndexIdentifier(String keyspaceName, String tableName, String indexName) + { + return new IndexIdentifier(keyspaceName, tableName, indexName); } protected StorageAttachedIndexGroup getCurrentIndexGroup() @@ -267,6 +317,11 @@ public abstract class SAITester extends CQLTester return StorageAttachedIndexGroup.getIndexGroup(getCurrentColumnFamilyStore()); } + protected void dropIndex(IndexIdentifier indexIdentifier) throws Throwable + { + dropIndex("DROP INDEX %s." + indexIdentifier.indexName); + } + protected void simulateNodeRestart() { simulateNodeRestart(true); @@ -294,12 +349,12 @@ public abstract class SAITester extends CQLTester } } - protected void corruptIndexComponent(IndexComponent indexComponent, IndexContext indexContext, CorruptionType corruptionType) throws Exception + protected void corruptIndexComponent(IndexComponent indexComponent, IndexIdentifier indexIdentifier, CorruptionType corruptionType) throws Exception { ColumnFamilyStore cfs = getCurrentColumnFamilyStore(); for (SSTableReader sstable : cfs.getLiveSSTables()) { - File file = IndexDescriptor.create(sstable).fileFor(indexComponent, indexContext); + File file = IndexDescriptor.create(sstable).fileFor(indexComponent, indexIdentifier); corruptionType.corrupt(file); } } @@ -316,7 +371,7 @@ public abstract class SAITester extends CQLTester waitForAssert(() -> assertTrue(indexNeedsFullRebuild(indexName))); } - protected boolean verifyChecksum(IndexContext indexContext) + protected boolean verifyChecksum(IndexTermType indexContext, IndexIdentifier indexIdentifier) { ColumnFamilyStore cfs = getCurrentColumnFamilyStore(); @@ -324,21 +379,22 @@ public abstract class SAITester extends CQLTester { IndexDescriptor indexDescriptor = IndexDescriptor.create(sstable); if (!indexDescriptor.validatePerSSTableComponents(IndexValidation.CHECKSUM) - || !indexDescriptor.validatePerIndexComponents(indexContext, IndexValidation.CHECKSUM)) + || !indexDescriptor.validatePerIndexComponents(indexContext, indexIdentifier, IndexValidation.CHECKSUM)) return false; } return true; } - protected boolean validateComponents(IndexContext indexContext) + protected boolean validateComponents(IndexTermType indexTermType, String indexName) { ColumnFamilyStore cfs = getCurrentColumnFamilyStore(); + IndexIdentifier indexIdentifier = createIndexIdentifier(cfs.getKeyspaceName(), cfs.getTableName(), indexName); for (SSTableReader sstable : cfs.getLiveSSTables()) { IndexDescriptor indexDescriptor = IndexDescriptor.create(sstable); if (!indexDescriptor.validatePerSSTableComponents(IndexValidation.HEADER_FOOTER) - || !indexDescriptor.validatePerIndexComponents(indexContext, IndexValidation.HEADER_FOOTER)) + || !indexDescriptor.validatePerIndexComponents(indexTermType, indexIdentifier, IndexValidation.HEADER_FOOTER)) return false; } return true; @@ -467,67 +523,45 @@ public abstract class SAITester extends CQLTester assertTrue(indexFiles().isEmpty()); } - protected void verifyIndexFiles(IndexContext numericIndexContext, IndexContext literalIndexContext, int numericFiles, int literalFiles) + protected void verifyIndexFiles(IndexTermType indexTermType, + IndexIdentifier indexIdentifier, + int indexFiles) { - verifyIndexFiles(numericIndexContext, - literalIndexContext, - Math.max(numericFiles, literalFiles), - numericFiles, - literalFiles, - numericFiles, - literalFiles); + verifyIndexFiles(indexTermType, indexIdentifier, indexFiles, indexFiles, indexFiles); } - protected void verifyIndexFiles(IndexContext numericIndexContext, - IndexContext literalIndexContext, + protected void verifyIndexFiles(IndexTermType indexTermType, + IndexIdentifier indexIdentifier, int perSSTableFiles, - int numericFiles, - int literalFiles, - int numericCompletionMarkers, - int literalCompletionMarkers) + int perColumnFiles, + int completionMarkers) { Set indexFiles = indexFiles(); for (IndexComponent indexComponent : Version.LATEST.onDiskFormat().perSSTableIndexComponents(false)) { - Set tableFiles = componentFiles(indexFiles, SSTableFormat.Components.Types.CUSTOM.createComponent(Version.LATEST.fileNameFormatter().format(indexComponent, null))); + Component component = SSTableFormat.Components.Types.CUSTOM.createComponent(Version.LATEST.fileNameFormatter().format(indexComponent, null)); + Set tableFiles = componentFiles(indexFiles, component); assertEquals(tableFiles.toString(), perSSTableFiles, tableFiles.size()); } - if (literalIndexContext != null) + for (IndexComponent indexComponent : Version.LATEST.onDiskFormat().perColumnIndexComponents(indexTermType)) { - for (IndexComponent indexComponent : Version.LATEST.onDiskFormat().perColumnIndexComponents(literalIndexContext)) - { - Set stringIndexFiles = componentFiles(indexFiles, - SSTableFormat.Components.Types.CUSTOM.createComponent(Version.LATEST.fileNameFormatter().format(indexComponent, - literalIndexContext))); - if (isBuildCompletionMarker(indexComponent)) - assertEquals(literalCompletionMarkers, stringIndexFiles.size()); - else - assertEquals(stringIndexFiles.toString(), literalFiles, stringIndexFiles.size()); - } - } - - if (numericIndexContext != null) - { - for (IndexComponent indexComponent : Version.LATEST.onDiskFormat().perColumnIndexComponents(numericIndexContext)) - { - Set numericIndexFiles = componentFiles(indexFiles, - SSTableFormat.Components.Types.CUSTOM.createComponent(Version.LATEST.fileNameFormatter().format(indexComponent, - numericIndexContext))); - if (isBuildCompletionMarker(indexComponent)) - assertEquals(numericCompletionMarkers, numericIndexFiles.size()); - else - assertEquals(numericIndexFiles.toString(), numericFiles, numericIndexFiles.size()); - } + String componentName = Version.LATEST.fileNameFormatter().format(indexComponent, indexIdentifier); + Component component = SSTableFormat.Components.Types.CUSTOM.createComponent(componentName); + Set stringIndexFiles = componentFiles(indexFiles, component); + if (isBuildCompletionMarker(indexComponent)) + assertEquals(completionMarkers, stringIndexFiles.size()); + else + assertEquals(stringIndexFiles.toString(), perColumnFiles, stringIndexFiles.size()); } } - protected void verifySSTableIndexes(String indexName, int count) + protected void verifySSTableIndexes(IndexIdentifier indexIdentifier, int count) { try { - verifySSTableIndexes(indexName, count, count); + verifySSTableIndexes(indexIdentifier, count, count); } catch (Exception e) { @@ -535,15 +569,15 @@ public abstract class SAITester extends CQLTester } } - protected void verifySSTableIndexes(String indexName, int sstableContextCount, int sstableIndexCount) + protected void verifySSTableIndexes(IndexIdentifier indexIdentifier, int sstableContextCount, int sstableIndexCount) { ColumnFamilyStore cfs = getCurrentColumnFamilyStore(); StorageAttachedIndexGroup indexGroup = getCurrentIndexGroup(); int contextCount = indexGroup.sstableContextManager().size(); assertEquals("Expected " + sstableContextCount +" SSTableContexts, but got " + contextCount, sstableContextCount, contextCount); - StorageAttachedIndex sai = (StorageAttachedIndex) cfs.indexManager.getIndexByName(indexName); - Collection sstableIndexes = sai == null ? Collections.emptyList() : sai.getIndexContext().getView().getIndexes(); + StorageAttachedIndex sai = (StorageAttachedIndex) cfs.indexManager.getIndexByName(indexIdentifier.indexName); + Collection sstableIndexes = sai == null ? Collections.emptyList() : sai.view().getIndexes(); assertEquals("Expected " + sstableIndexCount +" SSTableIndexes, but got " + sstableIndexes.toString(), sstableIndexCount, sstableIndexes.size()); } diff --git a/test/unit/org/apache/cassandra/index/sai/analyzer/NonTokenizingAnalyzerTest.java b/test/unit/org/apache/cassandra/index/sai/analyzer/NonTokenizingAnalyzerTest.java index fb73a98eae..9a7f8f9e42 100644 --- a/test/unit/org/apache/cassandra/index/sai/analyzer/NonTokenizingAnalyzerTest.java +++ b/test/unit/org/apache/cassandra/index/sai/analyzer/NonTokenizingAnalyzerTest.java @@ -23,6 +23,7 @@ import java.nio.ByteBuffer; import org.junit.Test; import org.apache.cassandra.db.marshal.UTF8Type; +import org.apache.cassandra.index.sai.SAITester; import org.apache.cassandra.utils.ByteBufferUtil; import static org.junit.Assert.assertEquals; @@ -68,7 +69,7 @@ public class NonTokenizingAnalyzerTest private String getAnalyzedString(String input, NonTokenizingOptions options) throws Exception { - NonTokenizingAnalyzer analyzer = new NonTokenizingAnalyzer(UTF8Type.instance, options); + NonTokenizingAnalyzer analyzer = new NonTokenizingAnalyzer(SAITester.createIndexTermType(UTF8Type.instance), options); analyzer.reset(ByteBuffer.wrap(input.getBytes())); return analyzer.hasNext() ? ByteBufferUtil.string(analyzer.next) : null; } diff --git a/test/unit/org/apache/cassandra/index/sai/cql/StorageAttachedIndexDDLTest.java b/test/unit/org/apache/cassandra/index/sai/cql/StorageAttachedIndexDDLTest.java index d86be4ce93..2ebf3edb29 100644 --- a/test/unit/org/apache/cassandra/index/sai/cql/StorageAttachedIndexDDLTest.java +++ b/test/unit/org/apache/cassandra/index/sai/cql/StorageAttachedIndexDDLTest.java @@ -54,20 +54,20 @@ import org.apache.cassandra.db.compaction.CompactionManager; import org.apache.cassandra.db.compaction.OperationType; import org.apache.cassandra.db.marshal.AbstractType; import org.apache.cassandra.db.marshal.Int32Type; -import org.apache.cassandra.db.marshal.ReversedType; import org.apache.cassandra.db.marshal.UTF8Type; import org.apache.cassandra.exceptions.InvalidRequestException; import org.apache.cassandra.index.Index; import org.apache.cassandra.index.SecondaryIndexManager; -import org.apache.cassandra.index.sai.IndexContext; import org.apache.cassandra.index.sai.SAITester; import org.apache.cassandra.index.sai.StorageAttachedIndex; import org.apache.cassandra.index.sai.StorageAttachedIndexBuilder; import org.apache.cassandra.index.sai.analyzer.NonTokenizingOptions; import org.apache.cassandra.index.sai.disk.format.IndexComponent; import org.apache.cassandra.index.sai.disk.format.Version; -import org.apache.cassandra.index.sai.disk.v1.segment.SegmentBuilder; import org.apache.cassandra.index.sai.disk.v1.bitpack.NumericValuesWriter; +import org.apache.cassandra.index.sai.disk.v1.segment.SegmentBuilder; +import org.apache.cassandra.index.sai.utils.IndexIdentifier; +import org.apache.cassandra.index.sai.utils.IndexTermType; import org.apache.cassandra.index.sai.view.View; import org.apache.cassandra.inject.ActionBuilder; import org.apache.cassandra.inject.Expression; @@ -75,7 +75,6 @@ import org.apache.cassandra.inject.Injection; import org.apache.cassandra.inject.Injections; import org.apache.cassandra.inject.InvokePointBuilder; import org.apache.cassandra.io.sstable.format.SSTableReader; -import org.apache.cassandra.schema.IndexMetadata; import org.apache.cassandra.schema.SchemaConstants; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.utils.FBUtilities; @@ -468,9 +467,9 @@ public class StorageAttachedIndexDDLTest extends SAITester SecondaryIndexManager sim = getCurrentColumnFamilyStore().indexManager; StorageAttachedIndex index = (StorageAttachedIndex) sim.getIndexByName(indexNameCk1); - IndexContext context = index.getIndexContext(); - assertTrue(context.isLiteral()); - assertTrue(context.getValidator() instanceof ReversedType); + IndexTermType indexTermType = index.termType(); + assertTrue(indexTermType.isLiteral()); + assertTrue(indexTermType.isReversed()); } @Test @@ -676,15 +675,18 @@ public class StorageAttachedIndexDDLTest extends SAITester { createTable(CREATE_TABLE_TEMPLATE); - String numericIndexName = createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1")); - String literalIndexName = createIndex(String.format(CREATE_INDEX_TEMPLATE, "v2")); - IndexContext numericIndexContext = createIndexContext(numericIndexName, Int32Type.instance); - IndexContext literalIndexContext = createIndexContext(literalIndexName, UTF8Type.instance); - verifyIndexFiles(numericIndexContext, literalIndexContext, 0, 0); + IndexIdentifier numericIndexIdentifier = createIndexIdentifier(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1"))); + IndexIdentifier literalIndexIdentifier = createIndexIdentifier(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v2"))); + IndexTermType numericIndexTermType = createIndexTermType(Int32Type.instance); + IndexTermType literalIndexTermType = createIndexTermType(UTF8Type.instance); + + verifyIndexFiles(numericIndexTermType, numericIndexIdentifier, 0); + verifyIndexFiles(literalIndexTermType, literalIndexIdentifier, 0); execute("INSERT INTO %s (id1, v1, v2) VALUES ('0', 0, '0')"); flush(); - verifyIndexFiles(numericIndexContext, literalIndexContext, 1, 1); + verifyIndexFiles(numericIndexTermType, numericIndexIdentifier, 1); + verifyIndexFiles(literalIndexTermType, literalIndexIdentifier, 1); ResultSet rows = executeNet("SELECT id1 FROM %s WHERE v1>=0"); assertEquals(1, rows.all().size()); rows = executeNet("SELECT id1 FROM %s WHERE v2='0'"); @@ -692,29 +694,34 @@ public class StorageAttachedIndexDDLTest extends SAITester execute("INSERT INTO %s (id1, v1, v2) VALUES ('1', 1, '0')"); flush(); - verifyIndexFiles(numericIndexContext, literalIndexContext, 2, 2); - verifySSTableIndexes(numericIndexName, 2, 2); - verifySSTableIndexes(literalIndexName, 2, 2); + verifyIndexFiles(numericIndexTermType, numericIndexIdentifier, 2); + verifyIndexFiles(literalIndexTermType, literalIndexIdentifier, 2); + + verifySSTableIndexes(numericIndexIdentifier, 2, 2); + verifySSTableIndexes(literalIndexIdentifier, 2, 2); rows = executeNet("SELECT id1 FROM %s WHERE v1>=0"); assertEquals(2, rows.all().size()); rows = executeNet("SELECT id1 FROM %s WHERE v2='0'"); assertEquals(2, rows.all().size()); - dropIndex("DROP INDEX %s." + numericIndexName); - verifyIndexFiles(numericIndexContext, literalIndexContext, 0, 2); - verifySSTableIndexes(numericIndexName, 2, 0); - verifySSTableIndexes(literalIndexName, 2, 2); + dropIndex(numericIndexIdentifier); + verifyIndexFiles(numericIndexTermType, numericIndexIdentifier, 2, 0, 0); + verifyIndexFiles(literalIndexTermType, literalIndexIdentifier, 2, 2, 2); + verifySSTableIndexes(numericIndexIdentifier, 2, 0); + verifySSTableIndexes(literalIndexIdentifier, 2, 2); rows = executeNet("SELECT id1 FROM %s WHERE v2='0'"); assertEquals(2, rows.all().size()); execute("INSERT INTO %s (id1, v1, v2) VALUES ('2', 2, '0')"); flush(); - verifyIndexFiles(numericIndexContext, literalIndexContext, 0, 3); + verifyIndexFiles(numericIndexTermType, numericIndexIdentifier, 3, 0, 0); + verifyIndexFiles(literalIndexTermType, literalIndexIdentifier, 3, 3, 3); rows = executeNet("SELECT id1 FROM %s WHERE v2='0'"); assertEquals(3, rows.all().size()); - dropIndex("DROP INDEX %s." + literalIndexName); - verifyIndexFiles(numericIndexContext, literalIndexContext, 0, 0); + dropIndex(literalIndexIdentifier); + verifyIndexFiles(numericIndexTermType, numericIndexIdentifier, 0); + verifyIndexFiles(literalIndexTermType, literalIndexIdentifier, 0); assertNull(getCurrentIndexGroup()); assertEquals("Segment memory limiter should revert to zero on drop.", 0L, getSegmentBufferUsedBytes()); @@ -735,10 +742,14 @@ public class StorageAttachedIndexDDLTest extends SAITester flush(); verifyNoIndexFiles(); - IndexContext numericIndexContext = createIndexContext(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1")), Int32Type.instance); - IndexContext literalIndexContext = createIndexContext(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v2")), UTF8Type.instance); - waitForTableIndexesQueryable(); - verifyIndexFiles(numericIndexContext, literalIndexContext, 2, 2); + IndexIdentifier numericIndexIdentifier = createIndexIdentifier(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1"))); + IndexIdentifier literalIndexIdentifier = createIndexIdentifier(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v2"))); + IndexTermType numericIndexTermType = createIndexTermType(Int32Type.instance); + IndexTermType literalIndexTermType = createIndexTermType(UTF8Type.instance); + + verifyIndexFiles(numericIndexTermType, numericIndexIdentifier, 2); + verifyIndexFiles(literalIndexTermType, literalIndexIdentifier, 2); + ResultSet rows = executeNet("SELECT id1 FROM %s WHERE v1>=0"); assertEquals(2, rows.all().size()); rows = executeNet("SELECT id1 FROM %s WHERE v2='0'"); @@ -762,15 +773,20 @@ public class StorageAttachedIndexDDLTest extends SAITester flush(); verifyNoIndexFiles(); - IndexContext numericIndexContext = createIndexContext(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1")), Int32Type.instance); - waitForTableIndexesQueryable(); - verifyIndexFiles(numericIndexContext, null, 2, 0); + IndexIdentifier numericIndexIdentifier = createIndexIdentifier(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1"))); + IndexTermType numericIndexTermType = createIndexTermType(Int32Type.instance); + + verifyIndexFiles(numericIndexTermType, numericIndexIdentifier, 2); + ResultSet rows = executeNet("SELECT id1 FROM %s WHERE v1>=0"); assertEquals(2, rows.all().size()); - IndexContext literalIndexContext = createIndexContext(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v2")), UTF8Type.instance); - waitForTableIndexesQueryable(); - verifyIndexFiles(numericIndexContext, literalIndexContext, 2, 2); + IndexIdentifier literalIndexIdentifier = createIndexIdentifier(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v2"))); + IndexTermType literalIndexTermType = createIndexTermType(UTF8Type.instance); + + verifyIndexFiles(numericIndexTermType, numericIndexIdentifier, 2); + verifyIndexFiles(literalIndexTermType, literalIndexIdentifier, 2); + rows = executeNet("SELECT id1 FROM %s WHERE v1>=0"); assertEquals(2, rows.all().size()); rows = executeNet("SELECT id1 FROM %s WHERE v2='0'"); @@ -786,13 +802,16 @@ public class StorageAttachedIndexDDLTest extends SAITester createTable(CREATE_TABLE_TEMPLATE); disableCompaction(KEYSPACE); - IndexContext numericIndexContext = createIndexContext(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1")), Int32Type.instance); - IndexContext literalIndexContext = createIndexContext(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v2")), UTF8Type.instance); + IndexIdentifier numericIndexIdentifier = createIndexIdentifier(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1"))); + IndexIdentifier literalIndexIdentifier = createIndexIdentifier(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v2"))); + IndexTermType numericIndexTermType = createIndexTermType(Int32Type.instance); + IndexTermType literalIndexTermType = createIndexTermType(UTF8Type.instance); verifyNoIndexFiles(); execute("INSERT INTO %s (id1, v1, v2) VALUES ('0', 0, '0');"); flush(); - verifyIndexFiles(numericIndexContext, literalIndexContext, 1, 1); + verifyIndexFiles(numericIndexTermType, numericIndexIdentifier, 1); + verifyIndexFiles(literalIndexTermType, literalIndexIdentifier, 1); ResultSet rows = executeNet("SELECT id1 FROM %s WHERE v1>=0"); assertEquals(1, rows.all().size()); rows = executeNet("SELECT id1 FROM %s WHERE v2='0'"); @@ -800,14 +819,16 @@ public class StorageAttachedIndexDDLTest extends SAITester execute("INSERT INTO %s (id1, v1, v2) VALUES ('1', 1, '0');"); flush(); - verifyIndexFiles(numericIndexContext, literalIndexContext, 2, 2); + verifyIndexFiles(numericIndexTermType, numericIndexIdentifier, 2); + verifyIndexFiles(literalIndexTermType, literalIndexIdentifier, 2); rows = executeNet("SELECT id1 FROM %s WHERE v1>=0"); assertEquals(2, rows.all().size()); rows = executeNet("SELECT id1 FROM %s WHERE v2='0'"); assertEquals(2, rows.all().size()); compact(); - waitForAssert(() -> verifyIndexFiles(numericIndexContext, literalIndexContext, 1, 1)); + waitForAssert(() -> verifyIndexFiles(numericIndexTermType, numericIndexIdentifier, 1)); + waitForAssert(() -> verifyIndexFiles(literalIndexTermType, literalIndexIdentifier, 1)); rows = executeNet("SELECT id1 FROM %s WHERE v1>=0"); assertEquals(2, rows.all().size()); @@ -834,10 +855,13 @@ public class StorageAttachedIndexDDLTest extends SAITester { createTable(CREATE_TABLE_TEMPLATE); + IndexIdentifier numericIndexIdentifier = null; + IndexIdentifier literalIndexIdentifier = null; + if (!concurrentTruncate) { - createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1")); - createIndex(String.format(CREATE_INDEX_TEMPLATE, "v2")); + numericIndexIdentifier = createIndexIdentifier(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1"))); + literalIndexIdentifier = createIndexIdentifier(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v2"))); } // create 100 rows, half in sstable and half in memtable @@ -851,8 +875,8 @@ public class StorageAttachedIndexDDLTest extends SAITester if (concurrentTruncate) { - createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1")); - createIndex(String.format(CREATE_INDEX_TEMPLATE, "v2")); + numericIndexIdentifier = createIndexIdentifier(createIndexAsync(String.format(CREATE_INDEX_TEMPLATE, "v1"))); + literalIndexIdentifier = createIndexIdentifier(createIndexAsync(String.format(CREATE_INDEX_TEMPLATE, "v2"))); truncate(true); waitForTableIndexesQueryable(); } @@ -864,8 +888,8 @@ public class StorageAttachedIndexDDLTest extends SAITester waitForAssert(this::verifyNoIndexFiles); // verify index-view-manager has been cleaned up - verifySSTableIndexes(IndexMetadata.generateDefaultIndexName(currentTable(), V1_COLUMN_IDENTIFIER), 0); - verifySSTableIndexes(IndexMetadata.generateDefaultIndexName(currentTable(), V2_COLUMN_IDENTIFIER), 0); + verifySSTableIndexes(numericIndexIdentifier, 0); + verifySSTableIndexes(literalIndexIdentifier, 0); assertEquals("Segment memory limiter should revert to zero after truncate.", 0L, getSegmentBufferUsedBytes()); assertEquals("There should be no segment builders in progress.", 0L, getColumnIndexBuildsInProgress()); @@ -876,8 +900,8 @@ public class StorageAttachedIndexDDLTest extends SAITester { // prepare schema and data createTable(CREATE_TABLE_TEMPLATE); - String numericIndexName = createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1")); - String stringIndexName = createIndex(String.format(CREATE_INDEX_TEMPLATE, "v2")); + IndexIdentifier numericIndexIdentifier = createIndexIdentifier(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1"))); + IndexIdentifier literalIndexIdentifier = createIndexIdentifier(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v2"))); execute("INSERT INTO %s (id1, v1, v2) VALUES ('0', 0, '0');"); execute("INSERT INTO %s (id1, v1, v2) VALUES ('1', 1, '0');"); @@ -885,36 +909,39 @@ public class StorageAttachedIndexDDLTest extends SAITester for (CorruptionType corruptionType : CorruptionType.values()) { - verifyRebuildCorruptedFiles(numericIndexName, stringIndexName, corruptionType, false); - verifyRebuildCorruptedFiles(numericIndexName, stringIndexName, corruptionType, true); + verifyRebuildCorruptedFiles(numericIndexIdentifier, literalIndexIdentifier, corruptionType, false); + verifyRebuildCorruptedFiles(numericIndexIdentifier, literalIndexIdentifier, corruptionType, true); } assertEquals("Segment memory limiter should revert to zero following rebuild.", 0L, getSegmentBufferUsedBytes()); assertEquals("There should be no segment builders in progress.", 0L, getColumnIndexBuildsInProgress()); } - private void verifyRebuildCorruptedFiles(String numericIndexName, - String stringIndexName, + private void verifyRebuildCorruptedFiles(IndexIdentifier numericIndexIdentifier, + IndexIdentifier literalIndexIdentifier, CorruptionType corruptionType, boolean rebuild) throws Throwable { - IndexContext numericIndexContext = createIndexContext(numericIndexName, Int32Type.instance); - IndexContext stringIndexContext = createIndexContext(stringIndexName, UTF8Type.instance); + IndexTermType numericIndexTermType = createIndexTermType(Int32Type.instance); + IndexTermType literalIndexTermType = createIndexTermType(UTF8Type.instance); for (IndexComponent component : Version.LATEST.onDiskFormat().perSSTableIndexComponents(false)) - verifyRebuildIndexComponent(numericIndexContext, stringIndexContext, component, null, corruptionType, true, true, rebuild); + verifyRebuildIndexComponent(numericIndexTermType, numericIndexIdentifier, literalIndexTermType, literalIndexIdentifier, component, null, null, corruptionType, true, true, rebuild); - for (IndexComponent component : Version.LATEST.onDiskFormat().perColumnIndexComponents(numericIndexContext)) - verifyRebuildIndexComponent(numericIndexContext, stringIndexContext, component, numericIndexContext, corruptionType, false, true, rebuild); + for (IndexComponent component : Version.LATEST.onDiskFormat().perColumnIndexComponents(numericIndexTermType)) + verifyRebuildIndexComponent(numericIndexTermType, numericIndexIdentifier, literalIndexTermType, literalIndexIdentifier, component, numericIndexTermType, numericIndexIdentifier, corruptionType, false, true, rebuild); - for (IndexComponent component : Version.LATEST.onDiskFormat().perColumnIndexComponents(stringIndexContext)) - verifyRebuildIndexComponent(numericIndexContext, stringIndexContext, component, stringIndexContext, corruptionType, true, false, rebuild); + for (IndexComponent component : Version.LATEST.onDiskFormat().perColumnIndexComponents(literalIndexTermType)) + verifyRebuildIndexComponent(numericIndexTermType, numericIndexIdentifier, literalIndexTermType, literalIndexIdentifier, component, literalIndexTermType, literalIndexIdentifier, corruptionType, true, false, rebuild); } - private void verifyRebuildIndexComponent(IndexContext numericIndexContext, - IndexContext stringIndexContext, + private void verifyRebuildIndexComponent(IndexTermType numericIndexTermType, + IndexIdentifier numericIndexIdentifier, + IndexTermType literalIndexTermType, + IndexIdentifier literalIndexIdentifier, IndexComponent component, - IndexContext corruptionContext, + IndexTermType corruptionIndexTermType, + IndexIdentifier corruptionIndexIdentifier, CorruptionType corruptionType, boolean failedStringIndex, boolean failedNumericIndex, @@ -938,11 +965,12 @@ public class StorageAttachedIndexDDLTest extends SAITester int rowCount = 2; // initial verification - verifySSTableIndexes(numericIndexContext.getIndexName(), 1); - verifySSTableIndexes(stringIndexContext.getIndexName(), 1); - verifyIndexFiles(numericIndexContext, stringIndexContext, 1, 1, 1, 1, 1); - assertTrue(verifyChecksum(numericIndexContext)); - assertTrue(verifyChecksum(numericIndexContext)); + verifySSTableIndexes(numericIndexIdentifier, 1); + verifySSTableIndexes(literalIndexIdentifier, 1); + verifyIndexFiles(numericIndexTermType, numericIndexIdentifier, 1); + verifyIndexFiles(literalIndexTermType, literalIndexIdentifier, 1); + assertTrue(verifyChecksum(numericIndexTermType, numericIndexIdentifier)); + assertTrue(verifyChecksum(literalIndexTermType, literalIndexIdentifier)); ResultSet rows = executeNet("SELECT id1 FROM %s WHERE v1>=0"); assertEquals(rowCount, rows.all().size()); @@ -950,8 +978,8 @@ public class StorageAttachedIndexDDLTest extends SAITester assertEquals(rowCount, rows.all().size()); // corrupt file - if (corruptionContext != null) - corruptIndexComponent(component, corruptionContext, corruptionType); + if (corruptionIndexTermType != null) + corruptIndexComponent(component, corruptionIndexIdentifier, corruptionType); else corruptIndexComponent(component, corruptionType); @@ -961,12 +989,12 @@ public class StorageAttachedIndexDDLTest extends SAITester boolean expectedLiteralState = !failedStringIndex || isBuildCompletionMarker(component); assertEquals("Checksum verification for " + component + " should be " + expectedNumericState + " but was " + !expectedNumericState, - expectedNumericState, verifyChecksum(numericIndexContext)); - assertEquals(expectedLiteralState, verifyChecksum(stringIndexContext)); + expectedNumericState, verifyChecksum(numericIndexTermType, numericIndexIdentifier)); + assertEquals(expectedLiteralState, verifyChecksum(literalIndexTermType, literalIndexIdentifier)); if (rebuild) { - rebuildIndexes(numericIndexContext.getIndexName(), stringIndexContext.getIndexName()); + rebuildIndexes(numericIndexIdentifier.indexName, literalIndexIdentifier.indexName); } else { @@ -974,8 +1002,8 @@ public class StorageAttachedIndexDDLTest extends SAITester reloadSSTableIndex(); // Verify the index cannot be read: - verifySSTableIndexes(numericIndexContext.getIndexName(), Version.LATEST.onDiskFormat().perSSTableIndexComponents(false).contains(component) ? 0 : 1, failedNumericIndex ? 0 : 1); - verifySSTableIndexes(stringIndexContext.getIndexName(), Version.LATEST.onDiskFormat().perSSTableIndexComponents(false).contains(component) ? 0 : 1, failedStringIndex ? 0 : 1); + verifySSTableIndexes(numericIndexIdentifier, Version.LATEST.onDiskFormat().perSSTableIndexComponents(false).contains(component) ? 0 : 1, failedNumericIndex ? 0 : 1); + verifySSTableIndexes(literalIndexIdentifier, Version.LATEST.onDiskFormat().perSSTableIndexComponents(false).contains(component) ? 0 : 1, failedStringIndex ? 0 : 1); try { @@ -1004,9 +1032,10 @@ public class StorageAttachedIndexDDLTest extends SAITester } // verify indexes are recovered - verifySSTableIndexes(numericIndexContext.getIndexName(), 1); - verifySSTableIndexes(numericIndexContext.getIndexName(), 1); - verifyIndexFiles(numericIndexContext, stringIndexContext, 1, 1, 1, 1, 1); + verifySSTableIndexes(numericIndexIdentifier, 1); + verifySSTableIndexes(numericIndexIdentifier, 1); + verifyIndexFiles(numericIndexTermType, numericIndexIdentifier, 1); + verifyIndexFiles(literalIndexTermType, literalIndexIdentifier, 1); rows = executeNet("SELECT id1 FROM %s WHERE v1>=0"); assertEquals(rowCount, rows.all().size()); @@ -1032,13 +1061,14 @@ public class StorageAttachedIndexDDLTest extends SAITester try { // Create a new index, which will actuate a build compaction and fail, but leave the node running... - IndexContext numericIndexContext = createIndexContext(createIndexAsync(String.format(CREATE_INDEX_TEMPLATE, "v1")), Int32Type.instance); + IndexIdentifier numericIndexIdentifier = createIndexIdentifier(createIndexAsync(String.format(CREATE_INDEX_TEMPLATE, "v1"))); + IndexTermType numericIndexTermType = createIndexTermType(Int32Type.instance); // two index builders running in different compaction threads because of parallelised index initial build waitForAssert(() -> assertEquals(2, indexBuildCounter.get())); waitForCompactionsFinished(); // Only token/offset files for the first SSTable in the compaction task should exist, while column-specific files are blown away: - verifyIndexFiles(numericIndexContext, null, 2, 0, 0, 0, 0); + verifyIndexFiles(numericIndexTermType, numericIndexIdentifier, 2, 0, 0); assertEquals("Segment memory limiter should revert to zero.", 0L, getSegmentBufferUsedBytes()); assertEquals("There should be no segment builders in progress.", 0L, getColumnIndexBuildsInProgress()); @@ -1090,8 +1120,10 @@ public class StorageAttachedIndexDDLTest extends SAITester createTable(CREATE_TABLE_TEMPLATE); disableCompaction(KEYSPACE); - IndexContext numericIndexContext = createIndexContext(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1")), Int32Type.instance); - IndexContext literalIndexContext = createIndexContext(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v2")), UTF8Type.instance); + IndexIdentifier numericIndexIdentifier = createIndexIdentifier(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1"))); + IndexIdentifier literalIndexIdentifier = createIndexIdentifier(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v2"))); + IndexTermType numericIndexTermType = createIndexTermType(Int32Type.instance); + IndexTermType literalIndexTermType = createIndexTermType(UTF8Type.instance); // flush empty index execute("INSERT INTO %s (id1) VALUES ('0');"); @@ -1100,7 +1132,8 @@ public class StorageAttachedIndexDDLTest extends SAITester execute("INSERT INTO %s (id1) VALUES ('1');"); flush(); - verifyIndexFiles(numericIndexContext, literalIndexContext, 2, 0, 0, 2, 2); + verifyIndexFiles(numericIndexTermType, numericIndexIdentifier, 2, 0, 2); + verifyIndexFiles(literalIndexTermType, literalIndexIdentifier, 2, 0, 2); ResultSet rows = executeNet("SELECT id1 FROM %s WHERE v1>=0"); assertEquals(0, rows.all().size()); @@ -1109,7 +1142,8 @@ public class StorageAttachedIndexDDLTest extends SAITester // compact empty index compact(); - waitForAssert(() -> verifyIndexFiles(numericIndexContext, literalIndexContext, 1, 0, 0, 1, 1)); + waitForAssert(() -> verifyIndexFiles(numericIndexTermType, numericIndexIdentifier, 1, 0, 1)); + waitForAssert(() -> verifyIndexFiles(literalIndexTermType, literalIndexIdentifier, 1, 0, 1)); rows = executeNet("SELECT id1 FROM %s WHERE v1>=0"); assertEquals(0, rows.all().size()); @@ -1170,15 +1204,15 @@ public class StorageAttachedIndexDDLTest extends SAITester createTable(CREATE_TABLE_TEMPLATE); disableCompaction(KEYSPACE); - - IndexContext numericIndexContext = createIndexContext(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1")), Int32Type.instance); - IndexContext literalIndexContext = createIndexContext(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v2")), UTF8Type.instance); - waitForTableIndexesQueryable(); - + IndexIdentifier numericIndexIdentifier = createIndexIdentifier(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1"))); + IndexIdentifier literalIndexIdentifier = createIndexIdentifier(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v2"))); + IndexTermType numericIndexTermType = createIndexTermType(Int32Type.instance); + IndexTermType literalIndexTermType = createIndexTermType(UTF8Type.instance); populateData.run(); - verifySSTableIndexes(IndexMetadata.generateDefaultIndexName(currentTable(), V1_COLUMN_IDENTIFIER), 2, 0); - verifySSTableIndexes(IndexMetadata.generateDefaultIndexName(currentTable(), V2_COLUMN_IDENTIFIER), 2, 0); - verifyIndexFiles(numericIndexContext, literalIndexContext, 2, 0, 0, 2, 2); + verifySSTableIndexes(numericIndexIdentifier, 2, 0); + verifySSTableIndexes(literalIndexIdentifier, 2, 0); + verifyIndexFiles(numericIndexTermType, numericIndexIdentifier, 2, 0, 2); + verifyIndexFiles(literalIndexTermType, literalIndexIdentifier, 2, 0, 2); ResultSet rows = executeNet("SELECT id1 FROM %s WHERE v1>=0"); assertEquals(0, rows.all().size()); @@ -1187,9 +1221,10 @@ public class StorageAttachedIndexDDLTest extends SAITester // compact empty index compact(); - verifySSTableIndexes(IndexMetadata.generateDefaultIndexName(currentTable(), V1_COLUMN_IDENTIFIER), 1, 0); - verifySSTableIndexes(IndexMetadata.generateDefaultIndexName(currentTable(), V2_COLUMN_IDENTIFIER), 1, 0); - waitForAssert(() -> verifyIndexFiles(numericIndexContext, literalIndexContext, 1, 0, 0, 1, 1)); + verifySSTableIndexes(numericIndexIdentifier, 1, 0); + verifySSTableIndexes(literalIndexIdentifier, 1, 0); + waitForAssert(() -> verifyIndexFiles(numericIndexTermType, numericIndexIdentifier, 1, 0, 1)); + waitForAssert(() -> verifyIndexFiles(literalIndexTermType, literalIndexIdentifier, 1, 0, 1)); rows = executeNet("SELECT id1 FROM %s WHERE v1>=0"); assertEquals(0, rows.all().size()); @@ -1218,10 +1253,10 @@ public class StorageAttachedIndexDDLTest extends SAITester .build(); Injections.inject(delayIndexBuilderCompletion); - String indexName = createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1")); + IndexIdentifier indexIdentifier = createIndexIdentifier(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1"))); waitForAssert(() -> assertEquals(1, delayIndexBuilderCompletion.getCount())); - dropIndex("DROP INDEX %s." + indexName); + dropIndex(indexIdentifier); // let blocked builders to continue delayIndexBuilderCompletion.countDown(); @@ -1230,12 +1265,12 @@ public class StorageAttachedIndexDDLTest extends SAITester delayIndexBuilderCompletion.disable(); assertNull(getCurrentIndexGroup()); - assertFalse("Expect index not built", SystemKeyspace.isIndexBuilt(KEYSPACE, indexName)); + assertFalse("Expect index not built", SystemKeyspace.isIndexBuilt(KEYSPACE, indexIdentifier.indexName)); // create index again, it should succeed - indexName = createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1")); + indexIdentifier = createIndexIdentifier(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1"))); waitForTableIndexesQueryable(); - verifySSTableIndexes(indexName, 1); + verifySSTableIndexes(indexIdentifier, 1); ResultSet rows = executeNet("SELECT id1 FROM %s WHERE v1>=0"); assertEquals(num, rows.all().size()); @@ -1263,7 +1298,9 @@ public class StorageAttachedIndexDDLTest extends SAITester Injections.inject(delayIndexBuilderCompletion); - IndexContext numericIndexContext = createIndexContext(createIndexAsync(String.format(CREATE_INDEX_TEMPLATE, "v1")), Int32Type.instance); + IndexIdentifier numericIndexIdentifier = createIndexIdentifier(createIndexAsync(String.format(CREATE_INDEX_TEMPLATE, "v1"))); + + IndexTermType numericIndexTermType = createIndexTermType(Int32Type.instance); waitForAssert(() -> assertTrue(getCompactionTasks() > 0), 1000, TimeUnit.MILLISECONDS); @@ -1287,16 +1324,16 @@ public class StorageAttachedIndexDDLTest extends SAITester delayIndexBuilderCompletion.disable(); // initial index builder should have stopped abruptly resulting in the index not being queryable - verifyInitialIndexFailed(numericIndexContext.getIndexName()); + verifyInitialIndexFailed(numericIndexIdentifier.indexName); Assertions.assertThat(getNotQueryableIndexes()).isNotEmpty(); ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(currentTable()); for (Index i : cfs.indexManager.listIndexes()) { StorageAttachedIndex index = (StorageAttachedIndex) i; - assertEquals(0, index.getIndexContext().getMemtableIndexManager().size()); + assertEquals(0, index.memtableIndexManager().size()); - View view = index.getIndexContext().getView(); + View view = index.view(); assertTrue("Expect index build stopped", view.getIndexes().isEmpty()); } @@ -1304,16 +1341,16 @@ public class StorageAttachedIndexDDLTest extends SAITester assertEquals("There should be no segment builders in progress.", 0L, getColumnIndexBuildsInProgress()); // rebuild index - ColumnFamilyStore.rebuildSecondaryIndex(KEYSPACE, currentTable(), numericIndexContext.getIndexName()); + ColumnFamilyStore.rebuildSecondaryIndex(KEYSPACE, currentTable(), numericIndexIdentifier.indexName); - verifyIndexFiles(numericIndexContext, null, sstable, 0); + verifyIndexFiles(numericIndexTermType, numericIndexIdentifier, sstable); ResultSet rows = executeNet("SELECT id1 FROM %s WHERE v1>=0"); assertEquals(num, rows.all().size()); assertEquals("Segment memory limiter should revert to zero following rebuild.", 0L, getSegmentBufferUsedBytes()); assertEquals("There should be no segment builders in progress.", 0L, getColumnIndexBuildsInProgress()); - assertTrue(verifyChecksum(numericIndexContext)); + assertTrue(verifyChecksum(numericIndexTermType, numericIndexIdentifier)); } @Test diff --git a/test/unit/org/apache/cassandra/index/sai/cql/UnindexedExpressionsTest.java b/test/unit/org/apache/cassandra/index/sai/cql/UnindexedExpressionsTest.java new file mode 100644 index 0000000000..4f9c60e593 --- /dev/null +++ b/test/unit/org/apache/cassandra/index/sai/cql/UnindexedExpressionsTest.java @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.index.sai.cql; + +import org.junit.Test; + +import org.apache.cassandra.cql3.restrictions.StatementRestrictions; +import org.apache.cassandra.index.sai.SAITester; + +import static org.junit.Assert.assertTrue; + +public class UnindexedExpressionsTest extends SAITester +{ + @Test + public void inOperatorIsNotHandledByIndexTest() throws Throwable + { + createTable("CREATE TABLE %s (k int PRIMARY KEY, v1 int, v2 int)"); + createIndex("CREATE INDEX ON %s(v1) USING 'sai'"); + createIndex("CREATE INDEX ON %s(v2) USING 'sai'"); + + execute("INSERT INTO %s (k, v1, v2) VALUES (1, 1, 1)"); + execute("INSERT INTO %s (k, v1, v2) VALUES (2, 2, 2)"); + execute("INSERT INTO %s (k, v1, v2) VALUES (3, 3, 1)"); + execute("INSERT INTO %s (k, v1, v2) VALUES (4, 4, 2)"); + + assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE, "SELECT * FROM %s WHERE v1 IN (1, 2)"); + + assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE, "SELECT * FROM %s WHERE v1 IN (1, 2) AND v2 = 1"); + + assertRowsIgnoringOrder(execute("SELECT * FROM %s WHERE v1 IN (1, 2) AND v2 = 1 ALLOW FILTERING"), row(1, 1, 1)); + + assertRowsIgnoringOrder(execute("SELECT * FROM %s WHERE v1 IN (1, 2) AND v2 = 2 ALLOW FILTERING"), row(2, 2, 2)); + + assertTrue(execute("SELECT * FROM %s WHERE v1 IN (1, 3) AND v2 = 2 ALLOW FILTERING").isEmpty()); + } + + @Test + public void unsupportedOperatorsAreHandledTest() throws Throwable + { + createTable("CREATE TABLE %s (pk int primary key, val1 int, val2 text)"); + createIndex("CREATE INDEX ON %s(val1) USING 'sai'"); + createIndex("CREATE INDEX ON %s(val2) USING 'sai'"); + + execute("INSERT INTO %s (pk, val1, val2) VALUES (1, 1, '11')"); + execute("INSERT INTO %s (pk, val1, val2) VALUES (2, 2, '22')"); + execute("INSERT INTO %s (pk, val1, val2) VALUES (3, 3, '33')"); + execute("INSERT INTO %s (pk, val1, val2) VALUES (4, 4, '44')"); + + // The LIKE operator is rejected because it needs to be handled by an index + assertInvalidMessage("LIKE restriction is only supported on properly indexed columns", + "SELECT pk FROM %s WHERE val1 = 1 AND val2 like '1%%'"); + + // The IS NOT operator is only valid on materialized views + assertInvalidMessage("Unsupported restriction:", "SELECT pk FROM %s WHERE val1 = 1 AND val2 is not null"); + + // The != operator is currently not supported at all + assertInvalidMessage("Unsupported \"!=\" relation:", "SELECT pk FROM %s WHERE val1 = 1 AND val2 != '22'"); + + assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE, "SELECT pk FROM %s WHERE val1 = 1 AND val2 < '22'"); + assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE, "SELECT pk FROM %s WHERE val1 = 1 AND val2 <= '11'"); + assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE, "SELECT pk FROM %s WHERE val1 = 1 AND val2 >= '11'"); + assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE, "SELECT pk FROM %s WHERE val1 = 1 AND val2 > '00'"); + assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE, "SELECT pk FROM %s WHERE val1 = 1 AND val2 in ('11', '22')"); + + assertRows(execute("SELECT pk FROM %s WHERE val1 >= 1 AND val2 < '22' ALLOW FILTERING"), row(1)); + assertRows(execute("SELECT pk FROM %s WHERE val1 >= 1 AND val2 <= '11' ALLOW FILTERING"), row(1)); + assertRows(execute("SELECT pk FROM %s WHERE val1 >= 1 AND val2 >= '11' AND val2 <= '22' ALLOW FILTERING"), row(1), row(2)); + assertRows(execute("SELECT pk FROM %s WHERE val1 >= 1 AND val2 > '00' AND val2 <= '11' ALLOW FILTERING"), row(1)); + assertRows(execute("SELECT pk FROM %s WHERE val1 >= 1 AND val2 in ('11', '22') ALLOW FILTERING"), row(1), row(2)); + } + + @Test + public void unindexedMapColumnTest() throws Throwable + { + createTable("CREATE TABLE %s (pk int primary key, val1 int, val2 map)"); + createIndex("CREATE INDEX ON %s(val1) USING 'sai'"); + + execute("INSERT INTO %s (pk, val1, val2) VALUES (1, 1, {1 : '1', 2 : '2', 3 : '3'})"); + execute("INSERT INTO %s (pk, val1, val2) VALUES (2, 2, {2 : '2', 3 : '3', 4 : '4'})"); + execute("INSERT INTO %s (pk, val1, val2) VALUES (3, 3, {3 : '3', 4 : '4', 5 : '5'})"); + execute("INSERT INTO %s (pk, val1, val2) VALUES (4, 4, {4 : '4', 5 : '5', 6 : '6'})"); + + assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE, "SELECT pk FROM %s WHERE val1 = 1 AND val2 CONTAINS KEY 1"); + assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE, "SELECT pk FROM %s WHERE val1 = 1 AND val2 CONTAINS '2'"); + + assertRows(execute("SELECT pk FROM %s WHERE val1 >= 1 AND val2 CONTAINS KEY 2 ALLOW FILTERING"), row(1), row(2)); + assertRows(execute("SELECT pk FROM %s WHERE val1 >= 1 AND val2 CONTAINS '2' ALLOW FILTERING"), row(1), row(2)); + assertRows(execute("SELECT pk FROM %s WHERE val1 >= 1 AND val2[2] = '2' ALLOW FILTERING"), row(1), row(2)); + } +} \ No newline at end of file diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/DataSet.java b/test/unit/org/apache/cassandra/index/sai/cql/types/DataSet.java index 7156b3cad2..6eeefb62e6 100644 --- a/test/unit/org/apache/cassandra/index/sai/cql/types/DataSet.java +++ b/test/unit/org/apache/cassandra/index/sai/cql/types/DataSet.java @@ -34,7 +34,7 @@ import java.util.concurrent.TimeUnit; import org.apache.cassandra.db.marshal.InetAddressType; import org.apache.cassandra.index.sai.SAITester; -import org.apache.cassandra.index.sai.utils.TypeUtil; +import org.apache.cassandra.index.sai.utils.IndexTermType; import org.apache.cassandra.serializers.SimpleDateSerializer; import org.apache.cassandra.serializers.TimeSerializer; import org.apache.cassandra.utils.TimeUUID; @@ -678,9 +678,9 @@ public abstract class DataSet extends SAITester while (list.contains(value)); values[index] = value; } - Arrays.sort(values, (o1, o2) -> TypeUtil.compare(TypeUtil.asIndexBytes(ByteBuffer.wrap(o1.getAddress()), InetAddressType.instance), - TypeUtil.asIndexBytes(ByteBuffer.wrap(o2.getAddress()), InetAddressType.instance), - InetAddressType.instance)); + IndexTermType indexTermType = createIndexTermType(InetAddressType.instance); + Arrays.sort(values, (o1, o2) -> indexTermType.compare(indexTermType.asIndexBytes(ByteBuffer.wrap(o1.getAddress())), + indexTermType.asIndexBytes(ByteBuffer.wrap(o2.getAddress())))); } @Override diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/NumericTypeSortingTest.java b/test/unit/org/apache/cassandra/index/sai/cql/types/NumericTypeSortingTest.java index a357be4aec..e981fe4e8b 100644 --- a/test/unit/org/apache/cassandra/index/sai/cql/types/NumericTypeSortingTest.java +++ b/test/unit/org/apache/cassandra/index/sai/cql/types/NumericTypeSortingTest.java @@ -27,8 +27,8 @@ import org.junit.Test; import org.apache.cassandra.db.marshal.DecimalType; import org.apache.cassandra.db.marshal.IntegerType; +import org.apache.cassandra.index.sai.utils.IndexTermType; import org.apache.cassandra.index.sai.utils.SAIRandomizedTester; -import org.apache.cassandra.index.sai.utils.TypeUtil; import static org.junit.Assert.assertTrue; @@ -48,6 +48,8 @@ public class NumericTypeSortingTest extends SAIRandomizedTester data[i] = randomNumber; } + IndexTermType indexTermType = createIndexTermType(DecimalType.instance); + Arrays.sort(data, BigDecimal::compareTo); for (int i = 1; i < data.length; i++) @@ -56,11 +58,11 @@ public class NumericTypeSortingTest extends SAIRandomizedTester BigDecimal i1 = data[i]; assertTrue(i0 + " <= " + i1, i0.compareTo(i1) <= 0); - ByteBuffer b0 = TypeUtil.asIndexBytes(DecimalType.instance.decompose(i0), DecimalType.instance); + ByteBuffer b0 = indexTermType.asIndexBytes(DecimalType.instance.decompose(i0)); - ByteBuffer b1 = TypeUtil.asIndexBytes(DecimalType.instance.decompose(i1), DecimalType.instance); + ByteBuffer b1 = indexTermType.asIndexBytes(DecimalType.instance.decompose(i1)); - assertTrue(i0 + " <= " + i1, TypeUtil.compare(b0, b1, DecimalType.instance) <= 0); + assertTrue(i0 + " <= " + i1, indexTermType.compare(b0, b1) <= 0); } } @@ -80,17 +82,19 @@ public class NumericTypeSortingTest extends SAIRandomizedTester Arrays.sort(data, BigInteger::compareTo); + IndexTermType indexTermType = createIndexTermType(IntegerType.instance); + for (int i = 1; i < data.length; i++) { BigInteger i0 = data[i - 1]; BigInteger i1 = data[i]; assertTrue(i0 + " <= " + i1, i0.compareTo(i1) <= 0); - ByteBuffer b0 = TypeUtil.asIndexBytes(IntegerType.instance.decompose(i0), IntegerType.instance); + ByteBuffer b0 = indexTermType.asIndexBytes(IntegerType.instance.decompose(i0)); - ByteBuffer b1 = TypeUtil.asIndexBytes(IntegerType.instance.decompose(i1), IntegerType.instance); + ByteBuffer b1 = indexTermType.asIndexBytes(IntegerType.instance.decompose(i1)); - assertTrue(i0 + " <= " + i1, TypeUtil.compare(b0, b1, IntegerType.instance) <= 0); + assertTrue(i0 + " <= " + i1, indexTermType.compare(b0, b1) <= 0); } } } diff --git a/test/unit/org/apache/cassandra/index/sai/disk/NodeStartupTest.java b/test/unit/org/apache/cassandra/index/sai/disk/NodeStartupTest.java index aab47eebd1..c87e013fdc 100644 --- a/test/unit/org/apache/cassandra/index/sai/disk/NodeStartupTest.java +++ b/test/unit/org/apache/cassandra/index/sai/disk/NodeStartupTest.java @@ -35,14 +35,15 @@ import org.junit.runners.Parameterized; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.marshal.Int32Type; import org.apache.cassandra.index.SecondaryIndexManager; -import org.apache.cassandra.index.sai.IndexContext; import org.apache.cassandra.index.sai.SAITester; import org.apache.cassandra.index.sai.StorageAttachedIndex; import org.apache.cassandra.index.sai.StorageAttachedIndexBuilder; import org.apache.cassandra.index.sai.disk.format.IndexComponent; import org.apache.cassandra.index.sai.disk.format.IndexDescriptor; +import org.apache.cassandra.index.sai.utils.IndexIdentifier; import org.apache.cassandra.index.sai.disk.format.Version; import org.apache.cassandra.index.sai.disk.v1.SSTableIndexWriter; +import org.apache.cassandra.index.sai.utils.IndexTermType; import org.apache.cassandra.inject.Injection; import org.apache.cassandra.inject.Injections; import org.apache.cassandra.inject.InvokePointBuilder; @@ -115,7 +116,8 @@ public class NodeStartupTest extends SAITester private static Throwable error = null; - private IndexContext indexContext = null; + private IndexIdentifier indexIdentifier = null; + private IndexTermType indexTermType = null; enum Populator { @@ -181,8 +183,8 @@ public class NodeStartupTest extends SAITester public void setup() throws Throwable { createTable("CREATE TABLE %s (id text PRIMARY KEY, v1 text)"); - String indexName = createIndex(String.format("CREATE CUSTOM INDEX ON %%s(v1) USING '%s'", StorageAttachedIndex.class.getName())); - indexContext = createIndexContext(indexName, Int32Type.instance); + indexIdentifier = createIndexIdentifier(createIndex(String.format("CREATE CUSTOM INDEX ON %%s(v1) USING '%s'", StorageAttachedIndex.class.getName()))); + indexTermType = createIndexTermType(Int32Type.instance); Injections.inject(ObjectArrays.concat(barriers, counters, Injection.class)); Stream.of(barriers).forEach(Injections.Barrier::reset); Stream.of(barriers).forEach(Injections.Barrier::disable); @@ -238,7 +240,7 @@ public class NodeStartupTest extends SAITester } @Test - public void startupOrderingTest() throws Throwable + public void startupOrderingTest() { populator.populate(this); @@ -326,7 +328,7 @@ public class NodeStartupTest extends SAITester private boolean isColumnIndexComplete() { ColumnFamilyStore cfs = Objects.requireNonNull(Schema.instance.getKeyspaceInstance(KEYSPACE)).getColumnFamilyStore(currentTable()); - return cfs.getLiveSSTables().stream().allMatch(sstable -> IndexDescriptor.create(sstable).isPerColumnIndexBuildComplete(indexContext)); + return cfs.getLiveSSTables().stream().allMatch(sstable -> IndexDescriptor.create(sstable).isPerColumnIndexBuildComplete(indexIdentifier)); } private void setState(IndexStateOnRestart state) @@ -337,19 +339,19 @@ public class NodeStartupTest extends SAITester break; case ALL_EMPTY: Version.LATEST.onDiskFormat().perSSTableIndexComponents(false).forEach(this::remove); - Version.LATEST.onDiskFormat().perColumnIndexComponents(indexContext).forEach(c -> remove(c, indexContext)); + Version.LATEST.onDiskFormat().perColumnIndexComponents(indexTermType).forEach(c -> remove(c, indexIdentifier)); break; case PER_SSTABLE_INCOMPLETE: remove(IndexComponent.GROUP_COMPLETION_MARKER); break; case PER_COLUMN_INCOMPLETE: - remove(IndexComponent.COLUMN_COMPLETION_MARKER, indexContext); + remove(IndexComponent.COLUMN_COMPLETION_MARKER, indexIdentifier); break; case PER_SSTABLE_CORRUPT: corrupt(); break; case PER_COLUMN_CORRUPT: - corrupt(indexContext); + corrupt(indexIdentifier); break; } } @@ -367,11 +369,11 @@ public class NodeStartupTest extends SAITester } } - private void remove(IndexComponent component, IndexContext indexContext) + private void remove(IndexComponent component, IndexIdentifier indexIdentifier) { try { - corruptIndexComponent(component, indexContext, CorruptionType.REMOVED); + corruptIndexComponent(component, indexIdentifier, CorruptionType.REMOVED); } catch (Exception e) { @@ -393,11 +395,11 @@ public class NodeStartupTest extends SAITester } } - private void corrupt(IndexContext indexContext) + private void corrupt(IndexIdentifier indexIdentifier) { try { - corruptIndexComponent(IndexComponent.META, indexContext, CorruptionType.TRUNCATED_HEADER); + corruptIndexComponent(IndexComponent.META, indexIdentifier, CorruptionType.TRUNCATED_HEADER); } catch (Exception e) { diff --git a/test/unit/org/apache/cassandra/index/sai/disk/format/IndexDescriptorTest.java b/test/unit/org/apache/cassandra/index/sai/disk/format/IndexDescriptorTest.java index 264e217e70..bacca543d5 100644 --- a/test/unit/org/apache/cassandra/index/sai/disk/format/IndexDescriptorTest.java +++ b/test/unit/org/apache/cassandra/index/sai/disk/format/IndexDescriptorTest.java @@ -30,10 +30,9 @@ import org.junit.Test; import org.junit.rules.TemporaryFolder; import org.apache.cassandra.config.DatabaseDescriptor; -import org.apache.cassandra.db.marshal.UTF8Type; import org.apache.cassandra.dht.Murmur3Partitioner; -import org.apache.cassandra.index.sai.IndexContext; import org.apache.cassandra.index.sai.SAITester; +import org.apache.cassandra.index.sai.utils.IndexIdentifier; import org.apache.cassandra.io.sstable.Descriptor; import org.apache.cassandra.io.util.File; @@ -48,7 +47,7 @@ public class IndexDescriptorTest @BeforeClass public static void initialise() { - DatabaseDescriptor.daemonInitialization(); + DatabaseDescriptor.toolInitialization(); } @Before @@ -81,10 +80,10 @@ public class IndexDescriptorTest createFileOnDisk("-SAI+aa+test_index+ColumnComplete.db"); IndexDescriptor indexDescriptor = IndexDescriptor.create(descriptor, Murmur3Partitioner.instance, SAITester.EMPTY_COMPARATOR); - IndexContext indexContext = SAITester.createIndexContext("test_index", UTF8Type.instance); + IndexIdentifier indexIdentifier = SAITester.createIndexIdentifier("test", "test", "test_index"); assertEquals(Version.AA, indexDescriptor.version); - assertTrue(indexDescriptor.hasComponent(IndexComponent.COLUMN_COMPLETION_MARKER, indexContext)); + assertTrue(indexDescriptor.hasComponent(IndexComponent.COLUMN_COMPLETION_MARKER, indexIdentifier)); } private void createFileOnDisk(String filename) throws Throwable diff --git a/test/unit/org/apache/cassandra/index/sai/disk/format/VersionTest.java b/test/unit/org/apache/cassandra/index/sai/disk/format/VersionTest.java index d2cd1fc14a..cd9232dce9 100644 --- a/test/unit/org/apache/cassandra/index/sai/disk/format/VersionTest.java +++ b/test/unit/org/apache/cassandra/index/sai/disk/format/VersionTest.java @@ -30,7 +30,7 @@ public class VersionTest @BeforeClass public static void initialise() { - DatabaseDescriptor.daemonInitialization(); + DatabaseDescriptor.toolInitialization(); } @Test diff --git a/test/unit/org/apache/cassandra/index/sai/disk/v1/BalancedTreeIndexSearcherTest.java b/test/unit/org/apache/cassandra/index/sai/disk/v1/BalancedTreeIndexSearcherTest.java index 67aa1a7178..0e1fa29b94 100644 --- a/test/unit/org/apache/cassandra/index/sai/disk/v1/BalancedTreeIndexSearcherTest.java +++ b/test/unit/org/apache/cassandra/index/sai/disk/v1/BalancedTreeIndexSearcherTest.java @@ -28,6 +28,7 @@ import com.google.common.collect.Iterators; import com.google.common.collect.Lists; import org.junit.Test; +import org.apache.cassandra.cql3.Operator; import org.apache.cassandra.db.marshal.DecimalType; import org.apache.cassandra.db.marshal.Int32Type; import org.apache.cassandra.db.marshal.IntegerType; @@ -111,7 +112,7 @@ public class BalancedTreeIndexSearcherTest extends SAIRandomizedTester IndexSegmentSearcher indexSearcher = BlockBalancedTreeIndexBuilder.buildDecimalSearcher(newIndexDescriptor(), BigDecimal.ZERO, BigDecimal.valueOf(10L)); testRangeQueries(indexSearcher, DecimalType.instance, DecimalType.instance, BigDecimal::valueOf, - getLongsOnInterval(21L, 70L)); + getLongsOnInterval(20L, 70L)); } private List getLongsOnInterval(long lowerInclusive, long upperInclusive) @@ -148,11 +149,9 @@ public class BalancedTreeIndexSearcherTest extends SAIRandomizedTester final NumberType rawType, final NumberType encodedType, final Function rawValueProducer) throws Exception { - try (KeyRangeIterator results = indexSearcher.search(new Expression(SAITester.createIndexContext("meh", rawType)) - {{ - operator = IndexOperator.EQ; - lower = upper = new Bound(rawType.decompose(rawValueProducer.apply(EQ_TEST_LOWER_BOUND_INCLUSIVE)), encodedType, true); - }}, null, mock(QueryContext.class))) + try (KeyRangeIterator results = indexSearcher.search(Expression.create(SAITester.createIndexTermType(rawType)) + .add(Operator.EQ, rawType.decompose(rawValueProducer.apply(EQ_TEST_LOWER_BOUND_INCLUSIVE))) + , null, mock(QueryContext.class))) { assertEquals(results.getMinimum(), results.getCurrent()); assertTrue(results.hasNext()); @@ -160,11 +159,9 @@ public class BalancedTreeIndexSearcherTest extends SAIRandomizedTester assertEquals(0L, results.next().token().getLongValue()); } - try (KeyRangeIterator results = indexSearcher.search(new Expression(SAITester.createIndexContext("meh", rawType)) - {{ - operator = IndexOperator.EQ; - lower = upper = new Bound(rawType.decompose(rawValueProducer.apply(EQ_TEST_UPPER_BOUND_EXCLUSIVE)), encodedType, true); - }}, null, mock(QueryContext.class))) + try (KeyRangeIterator results = indexSearcher.search(Expression.create(SAITester.createIndexTermType(rawType)) + .add(Operator.EQ, rawType.decompose(rawValueProducer.apply(EQ_TEST_UPPER_BOUND_EXCLUSIVE))), + null, mock(QueryContext.class))) { assertFalse(results.hasNext()); indexSearcher.close(); @@ -175,7 +172,7 @@ public class BalancedTreeIndexSearcherTest extends SAIRandomizedTester final NumberType rawType, final NumberType encodedType, final Function rawValueProducer) throws Exception { - List expectedTokenList = getLongsOnInterval(3L, 7L); + List expectedTokenList = getLongsOnInterval(2L, 7L); testRangeQueries(indexSearcher, rawType, encodedType, rawValueProducer, expectedTokenList); } @@ -184,13 +181,10 @@ public class BalancedTreeIndexSearcherTest extends SAIRandomizedTester final NumberType rawType, final NumberType encodedType, final Function rawValueProducer, List expectedTokenList) throws Exception { - try (KeyRangeIterator results = indexSearcher.search(new Expression(SAITester.createIndexContext("meh", rawType)) - {{ - operator = IndexOperator.RANGE; - - lower = new Bound(rawType.decompose(rawValueProducer.apply((short)2)), encodedType, false); - upper = new Bound(rawType.decompose(rawValueProducer.apply((short)7)), encodedType, true); - }}, null, mock(QueryContext.class))) + try (KeyRangeIterator results = indexSearcher.search(Expression.create(SAITester.createIndexTermType(rawType)) + .add(Operator.GTE, rawType.decompose(rawValueProducer.apply((short)2))) + .add(Operator.LTE, rawType.decompose(rawValueProducer.apply((short)7))), + null, mock(QueryContext.class))) { assertEquals(results.getMinimum(), results.getCurrent()); assertTrue(results.hasNext()); @@ -199,19 +193,19 @@ public class BalancedTreeIndexSearcherTest extends SAIRandomizedTester assertEquals(expectedTokenList, actualTokenList); } - try (KeyRangeIterator results = indexSearcher.search(new Expression(SAITester.createIndexContext("meh", rawType)) + try (KeyRangeIterator results = indexSearcher.search(new Expression.IndexedExpression(SAITester.createMockIndex("meh", rawType)) {{ operator = IndexOperator.RANGE; - lower = new Bound(rawType.decompose(rawValueProducer.apply(RANGE_TEST_UPPER_BOUND_EXCLUSIVE)), encodedType, true); + lower = new Bound(rawType.decompose(rawValueProducer.apply(RANGE_TEST_UPPER_BOUND_EXCLUSIVE)), getIndexTermType(), true); }}, null, mock(QueryContext.class))) { assertFalse(results.hasNext()); } - try (KeyRangeIterator results = indexSearcher.search(new Expression(SAITester.createIndexContext("meh", rawType)) + try (KeyRangeIterator results = indexSearcher.search(new Expression.IndexedExpression(SAITester.createMockIndex("meh", rawType)) {{ operator = IndexOperator.RANGE; - upper = new Bound(rawType.decompose(rawValueProducer.apply(RANGE_TEST_LOWER_BOUND_INCLUSIVE)), encodedType, false); + upper = new Bound(rawType.decompose(rawValueProducer.apply(RANGE_TEST_LOWER_BOUND_INCLUSIVE)), getIndexTermType(), false); }}, null, mock(QueryContext.class))) { assertFalse(results.hasNext()); diff --git a/test/unit/org/apache/cassandra/index/sai/disk/v1/InvertedIndexSearcherTest.java b/test/unit/org/apache/cassandra/index/sai/disk/v1/InvertedIndexSearcherTest.java index ad6ef744de..ccffe76b21 100644 --- a/test/unit/org/apache/cassandra/index/sai/disk/v1/InvertedIndexSearcherTest.java +++ b/test/unit/org/apache/cassandra/index/sai/disk/v1/InvertedIndexSearcherTest.java @@ -31,9 +31,9 @@ import org.apache.cassandra.db.ClusteringComparator; import org.apache.cassandra.db.marshal.UTF8Type; import org.apache.cassandra.dht.Murmur3Partitioner; import org.apache.cassandra.dht.Token; -import org.apache.cassandra.index.sai.IndexContext; import org.apache.cassandra.index.sai.QueryContext; import org.apache.cassandra.index.sai.SAITester; +import org.apache.cassandra.index.sai.StorageAttachedIndex; import org.apache.cassandra.index.sai.iterators.KeyRangeIterator; import org.apache.cassandra.index.sai.memory.MemtableTermsIterator; import org.apache.cassandra.index.sai.disk.PrimaryKeyMap; @@ -100,22 +100,18 @@ public class InvertedIndexSearcherTest extends SAIRandomizedTester @Test public void testEqQueriesAgainstStringIndex() throws Exception - { - doTestEqQueriesAgainstStringIndex(); - } - - private void doTestEqQueriesAgainstStringIndex() throws Exception { QueryContext context = mock(QueryContext.class); + final StorageAttachedIndex index = createMockIndex(newIndex(), UTF8Type.instance); + final int numTerms = getRandom().nextIntBetween(64, 512), numPostings = getRandom().nextIntBetween(256, 1024); final List> termsEnum = buildTermsEnum(numTerms, numPostings); - try (IndexSegmentSearcher searcher = buildIndexAndOpenSearcher(numTerms, numPostings, termsEnum)) + try (IndexSegmentSearcher searcher = buildIndexAndOpenSearcher(index, numTerms, numPostings, termsEnum)) { for (int t = 0; t < numTerms; ++t) { - try (KeyRangeIterator results = searcher.search(new Expression(SAITester.createIndexContext("meh", UTF8Type.instance)) - .add(Operator.EQ, wrap(termsEnum.get(t).left)), null, context)) + try (KeyRangeIterator results = searcher.search(Expression.create(index).add(Operator.EQ, wrap(termsEnum.get(t).left)), null, context)) { assertEquals(results.getMinimum(), results.getCurrent()); assertTrue(results.hasNext()); @@ -130,8 +126,7 @@ public class InvertedIndexSearcherTest extends SAIRandomizedTester assertFalse(results.hasNext()); } - try (KeyRangeIterator results = searcher.search(new Expression(SAITester.createIndexContext("meh", UTF8Type.instance)) - .add(Operator.EQ, wrap(termsEnum.get(t).left)), null, context)) + try (KeyRangeIterator results = searcher.search(Expression.create(index).add(Operator.EQ, wrap(termsEnum.get(t).left)), null, context)) { assertEquals(results.getMinimum(), results.getCurrent()); assertTrue(results.hasNext()); @@ -153,13 +148,11 @@ public class InvertedIndexSearcherTest extends SAIRandomizedTester // try searching for terms that weren't indexed final String tooLongTerm = randomSimpleString(10, 12); - KeyRangeIterator results = searcher.search(new Expression(SAITester.createIndexContext("meh", UTF8Type.instance)) - .add(Operator.EQ, UTF8Type.instance.decompose(tooLongTerm)), null, context); + KeyRangeIterator results = searcher.search(Expression.create(index).add(Operator.EQ, UTF8Type.instance.decompose(tooLongTerm)), null, context); assertFalse(results.hasNext()); final String tooShortTerm = randomSimpleString(1, 2); - results = searcher.search(new Expression(SAITester.createIndexContext("meh", UTF8Type.instance)) - .add(Operator.EQ, UTF8Type.instance.decompose(tooShortTerm)), null, context); + results = searcher.search(Expression.create(index).add(Operator.EQ, UTF8Type.instance.decompose(tooShortTerm)), null, context); assertFalse(results.hasNext()); } } @@ -168,14 +161,14 @@ public class InvertedIndexSearcherTest extends SAIRandomizedTester public void testUnsupportedOperator() throws Exception { QueryContext context = mock(QueryContext.class); + final StorageAttachedIndex index = createMockIndex(newIndex(), UTF8Type.instance); final int numTerms = getRandom().nextIntBetween(5, 15), numPostings = getRandom().nextIntBetween(5, 20); final List> termsEnum = buildTermsEnum(numTerms, numPostings); - try (IndexSegmentSearcher searcher = buildIndexAndOpenSearcher(numTerms, numPostings, termsEnum)) + try (IndexSegmentSearcher searcher = buildIndexAndOpenSearcher(index, numTerms, numPostings, termsEnum)) { - searcher.search(new Expression(SAITester.createIndexContext("meh", UTF8Type.instance)) - .add(Operator.GT, UTF8Type.instance.decompose("a")), null, context); + searcher.search(Expression.create(index).add(Operator.GT, UTF8Type.instance.decompose("a")), null, context); fail("Expect IllegalArgumentException thrown, but didn't"); } @@ -185,15 +178,16 @@ public class InvertedIndexSearcherTest extends SAIRandomizedTester } } - private IndexSegmentSearcher buildIndexAndOpenSearcher(int terms, int postings, List> termsEnum) throws IOException + private IndexSegmentSearcher buildIndexAndOpenSearcher(StorageAttachedIndex index, + int terms, + int postings, + List> termsEnum) throws IOException { final int size = terms * postings; final IndexDescriptor indexDescriptor = newIndexDescriptor(); - final String index = newIndex(); - final IndexContext indexContext = SAITester.createIndexContext(index, UTF8Type.instance); SegmentMetadata.ComponentMetadataMap indexMetas; - try (LiteralIndexWriter writer = new LiteralIndexWriter(indexDescriptor, indexContext)) + try (LiteralIndexWriter writer = new LiteralIndexWriter(indexDescriptor, index.identifier())) { indexMetas = writer.writeCompleteSegment(new MemtableTermsIterator(null, null, termsEnum.iterator())); } @@ -208,12 +202,12 @@ public class InvertedIndexSearcherTest extends SAIRandomizedTester wrap(termsEnum.get(terms - 1).left), indexMetas); - try (PerColumnIndexFiles indexFiles = new PerColumnIndexFiles(indexDescriptor, indexContext)) + try (PerColumnIndexFiles indexFiles = new PerColumnIndexFiles(indexDescriptor, index.termType(), index.identifier())) { final IndexSegmentSearcher searcher = IndexSegmentSearcher.open(TEST_PRIMARY_KEY_MAP_FACTORY, indexFiles, segmentMetadata, - SAITester.createIndexContext(index, UTF8Type.instance)); + index); assertThat(searcher, is(instanceOf(LiteralIndexSegmentSearcher.class))); return searcher; } diff --git a/test/unit/org/apache/cassandra/index/sai/disk/v1/MetadataTest.java b/test/unit/org/apache/cassandra/index/sai/disk/v1/MetadataTest.java index f2aa82e5b2..db8eb28b7b 100644 --- a/test/unit/org/apache/cassandra/index/sai/disk/v1/MetadataTest.java +++ b/test/unit/org/apache/cassandra/index/sai/disk/v1/MetadataTest.java @@ -29,11 +29,9 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; -import org.apache.cassandra.db.marshal.UTF8Type; -import org.apache.cassandra.index.sai.IndexContext; -import org.apache.cassandra.index.sai.SAITester; import org.apache.cassandra.index.sai.disk.format.IndexComponent; import org.apache.cassandra.index.sai.disk.format.IndexDescriptor; +import org.apache.cassandra.index.sai.utils.IndexIdentifier; import org.apache.cassandra.index.sai.disk.io.IndexOutputWriter; import org.apache.cassandra.index.sai.utils.SAIRandomizedTester; import org.apache.cassandra.io.util.File; @@ -52,21 +50,20 @@ public class MetadataTest extends SAIRandomizedTester public final ExpectedException expectedException = ExpectedException.none(); private IndexDescriptor indexDescriptor; - private IndexContext indexContext; + private IndexIdentifier indexIdentifier; @Before public void setup() throws Throwable { indexDescriptor = newIndexDescriptor(); - String index = newIndex(); - indexContext = SAITester.createIndexContext(index, UTF8Type.instance); + indexIdentifier = createIndexIdentifier("test", "test", newIndex()); } @Test public void shouldReadWrittenMetadata() throws Exception { final Map data = new HashMap<>(); - try (MetadataWriter writer = new MetadataWriter(indexDescriptor.openPerIndexOutput(IndexComponent.META, indexContext))) + try (MetadataWriter writer = new MetadataWriter(indexDescriptor.openPerIndexOutput(IndexComponent.META, indexIdentifier))) { int num = nextInt(1, 50); for (int x = 0; x < num; x++) @@ -82,7 +79,7 @@ public class MetadataTest extends SAIRandomizedTester } } } - MetadataSource reader = MetadataSource.loadColumnMetadata(indexDescriptor, indexContext); + MetadataSource reader = MetadataSource.loadColumnMetadata(indexDescriptor, indexIdentifier); for (Map.Entry entry : data.entrySet()) { @@ -98,7 +95,7 @@ public class MetadataTest extends SAIRandomizedTester @Test public void shouldFailWhenFileHasNoHeader() throws IOException { - try (IndexOutputWriter out = indexDescriptor.openPerIndexOutput(IndexComponent.META, indexContext)) + try (IndexOutputWriter out = indexDescriptor.openPerIndexOutput(IndexComponent.META, indexIdentifier)) { final byte[] bytes = nextBytes(13, 29); out.writeBytes(bytes, bytes.length); @@ -106,7 +103,7 @@ public class MetadataTest extends SAIRandomizedTester expectedException.expect(CorruptIndexException.class); expectedException.expectMessage("codec header mismatch"); - MetadataSource.loadColumnMetadata(indexDescriptor, indexContext); + MetadataSource.loadColumnMetadata(indexDescriptor, indexIdentifier); } @Test @@ -130,7 +127,7 @@ public class MetadataTest extends SAIRandomizedTester expectedException.expect(CorruptIndexException.class); expectedException.expectMessage("misplaced codec footer (file truncated?)"); - MetadataSource.loadColumnMetadata(indexDescriptor, indexContext); + MetadataSource.loadColumnMetadata(indexDescriptor, indexIdentifier); } } @@ -166,13 +163,13 @@ public class MetadataTest extends SAIRandomizedTester expectedException.expect(CorruptIndexException.class); expectedException.expectMessage("checksum failed"); - MetadataSource.loadColumnMetadata(indexDescriptor, indexContext); + MetadataSource.loadColumnMetadata(indexDescriptor, indexIdentifier); } } private IndexOutputWriter writeRandomBytes() throws IOException { - final IndexOutputWriter output = indexDescriptor.openPerIndexOutput(IndexComponent.META, indexContext); + final IndexOutputWriter output = indexDescriptor.openPerIndexOutput(IndexComponent.META, indexIdentifier); try (MetadataWriter writer = new MetadataWriter(output)) { byte[] bytes = nextBytes(11, 1024); diff --git a/test/unit/org/apache/cassandra/index/sai/disk/v1/SegmentFlushTest.java b/test/unit/org/apache/cassandra/index/sai/disk/v1/SegmentFlushTest.java index cbd74d8bef..df125f7b39 100644 --- a/test/unit/org/apache/cassandra/index/sai/disk/v1/SegmentFlushTest.java +++ b/test/unit/org/apache/cassandra/index/sai/disk/v1/SegmentFlushTest.java @@ -31,19 +31,18 @@ import org.junit.BeforeClass; import org.junit.Test; import org.apache.cassandra.config.DatabaseDescriptor; -import org.apache.cassandra.cql3.statements.schema.IndexTarget; import org.apache.cassandra.db.Clustering; -import org.apache.cassandra.db.ClusteringComparator; import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.db.marshal.UTF8Type; import org.apache.cassandra.db.rows.BTreeRow; import org.apache.cassandra.db.rows.BufferCell; import org.apache.cassandra.db.rows.Row; import org.apache.cassandra.dht.Murmur3Partitioner; -import org.apache.cassandra.index.sai.IndexContext; import org.apache.cassandra.index.sai.SAITester; +import org.apache.cassandra.index.sai.StorageAttachedIndex; import org.apache.cassandra.index.sai.disk.format.IndexComponent; import org.apache.cassandra.index.sai.disk.format.IndexDescriptor; +import org.apache.cassandra.index.sai.utils.IndexIdentifier; import org.apache.cassandra.index.sai.disk.v1.segment.SegmentBuilder; import org.apache.cassandra.index.sai.disk.v1.segment.SegmentMetadata; import org.apache.cassandra.index.sai.postings.PostingList; @@ -54,7 +53,6 @@ import org.apache.cassandra.io.sstable.SequenceBasedSSTableId; import org.apache.cassandra.io.util.File; import org.apache.cassandra.io.util.FileHandle; import org.apache.cassandra.schema.ColumnMetadata; -import org.apache.cassandra.schema.IndexMetadata; import org.apache.cassandra.service.StorageService; import org.apache.cassandra.utils.bytecomparable.ByteComparable; @@ -76,7 +74,7 @@ public class SegmentFlushTest @BeforeClass public static void init() { - DatabaseDescriptor.daemonInitialization(); + DatabaseDescriptor.toolInitialization(); DatabaseDescriptor.setPartitionerUnsafe(Murmur3Partitioner.instance); StorageService.instance.setPartitionerUnsafe(Murmur3Partitioner.instance); } @@ -114,18 +112,10 @@ public class SegmentFlushTest SAITester.EMPTY_COMPARATOR); ColumnMetadata column = ColumnMetadata.regularColumn("sai", "internal", "column", UTF8Type.instance); - IndexMetadata config = IndexMetadata.fromSchemaMetadata("index_name", IndexMetadata.Kind.CUSTOM, null); - IndexContext indexContext = new IndexContext("ks", - "cf", - UTF8Type.instance, - Murmur3Partitioner.instance, - new ClusteringComparator(), - column, - IndexTarget.Type.SIMPLE, - config); + StorageAttachedIndex index = SAITester.createMockIndex(column); - SSTableIndexWriter writer = new SSTableIndexWriter(indexDescriptor, indexContext, V1OnDiskFormat.SEGMENT_BUILD_MEMORY_LIMITER, () -> true); + SSTableIndexWriter writer = new SSTableIndexWriter(indexDescriptor, index, V1OnDiskFormat.SEGMENT_BUILD_MEMORY_LIMITER, () -> true); List keys = Arrays.asList(dk("1"), dk("2")); Collections.sort(keys); @@ -143,7 +133,7 @@ public class SegmentFlushTest writer.complete(Stopwatch.createStarted()); - MetadataSource source = MetadataSource.loadColumnMetadata(indexDescriptor, indexContext); + MetadataSource source = MetadataSource.loadColumnMetadata(indexDescriptor, index.identifier()); List segmentMetadatas = SegmentMetadata.load(source, indexDescriptor.primaryKeyFactory); assertEquals(segments, segmentMetadatas.size()); @@ -159,7 +149,7 @@ public class SegmentFlushTest maxTerm = segments == 1 ? term2 : term1; numRows = segments == 1 ? 2 : 1; verifySegmentMetadata(segmentMetadata); - verifyStringIndex(indexDescriptor, indexContext, segmentMetadata); + verifyStringIndex(indexDescriptor, index.identifier(), segmentMetadata); if (segments > 1) { @@ -174,7 +164,7 @@ public class SegmentFlushTest segmentMetadata = segmentMetadatas.get(1); verifySegmentMetadata(segmentMetadata); - verifyStringIndex(indexDescriptor, indexContext, segmentMetadata); + verifyStringIndex(indexDescriptor, index.identifier(), segmentMetadata); } } @@ -188,10 +178,10 @@ public class SegmentFlushTest assertEquals(numRows, segmentMetadata.numRows); } - private void verifyStringIndex(IndexDescriptor indexDescriptor, IndexContext indexContext, SegmentMetadata segmentMetadata) throws IOException + private void verifyStringIndex(IndexDescriptor indexDescriptor, IndexIdentifier indexIdentifier, SegmentMetadata segmentMetadata) throws IOException { - FileHandle termsData = indexDescriptor.createPerIndexFileHandle(IndexComponent.TERMS_DATA, indexContext, null); - FileHandle postingLists = indexDescriptor.createPerIndexFileHandle(IndexComponent.POSTING_LISTS, indexContext, null); + FileHandle termsData = indexDescriptor.createPerIndexFileHandle(IndexComponent.TERMS_DATA, indexIdentifier, null); + FileHandle postingLists = indexDescriptor.createPerIndexFileHandle(IndexComponent.POSTING_LISTS, indexIdentifier, null); try (TermsIterator iterator = new TermsScanner(termsData, postingLists, segmentMetadata.componentMetadatas.get(IndexComponent.TERMS_DATA).root)) { diff --git a/test/unit/org/apache/cassandra/index/sai/disk/v1/SegmentTest.java b/test/unit/org/apache/cassandra/index/sai/disk/v1/SegmentTest.java index 70bf70e238..6a0869fbfd 100644 --- a/test/unit/org/apache/cassandra/index/sai/disk/v1/SegmentTest.java +++ b/test/unit/org/apache/cassandra/index/sai/disk/v1/SegmentTest.java @@ -46,7 +46,7 @@ public class SegmentTest @BeforeClass public static void init() { - DatabaseDescriptor.daemonInitialization(); + DatabaseDescriptor.toolInitialization(); DatabaseDescriptor.setPartitionerUnsafe(Murmur3Partitioner.instance); partitioner = DatabaseDescriptor.getPartitioner(); min = partitioner.getMinimumToken(); diff --git a/test/unit/org/apache/cassandra/index/sai/disk/v1/TermsReaderTest.java b/test/unit/org/apache/cassandra/index/sai/disk/v1/TermsReaderTest.java index 72451b09e6..b4111425ae 100644 --- a/test/unit/org/apache/cassandra/index/sai/disk/v1/TermsReaderTest.java +++ b/test/unit/org/apache/cassandra/index/sai/disk/v1/TermsReaderTest.java @@ -23,20 +23,18 @@ import java.util.List; import org.junit.Test; import com.carrotsearch.hppc.LongArrayList; -import org.apache.cassandra.db.marshal.UTF8Type; -import org.apache.cassandra.index.sai.IndexContext; import org.apache.cassandra.index.sai.QueryContext; -import org.apache.cassandra.index.sai.SAITester; -import org.apache.cassandra.index.sai.memory.MemtableTermsIterator; -import org.apache.cassandra.index.sai.postings.PostingList; -import org.apache.cassandra.index.sai.utils.TermsIterator; import org.apache.cassandra.index.sai.disk.format.IndexComponent; import org.apache.cassandra.index.sai.disk.format.IndexDescriptor; +import org.apache.cassandra.index.sai.utils.IndexIdentifier; import org.apache.cassandra.index.sai.disk.v1.segment.LiteralIndexSegmentTermsReader; import org.apache.cassandra.index.sai.disk.v1.segment.SegmentMetadata; import org.apache.cassandra.index.sai.disk.v1.trie.LiteralIndexWriter; +import org.apache.cassandra.index.sai.memory.MemtableTermsIterator; import org.apache.cassandra.index.sai.metrics.QueryEventListener; +import org.apache.cassandra.index.sai.postings.PostingList; import org.apache.cassandra.index.sai.utils.SAIRandomizedTester; +import org.apache.cassandra.index.sai.utils.TermsIterator; import org.apache.cassandra.io.util.FileHandle; import org.apache.cassandra.utils.Pair; import org.apache.cassandra.utils.bytecomparable.ByteComparable; @@ -72,18 +70,17 @@ public class TermsReaderTest extends SAIRandomizedTester { final int terms = 70, postings = 2; final IndexDescriptor indexDescriptor = newIndexDescriptor(); - final String index = newIndex(); - final IndexContext indexContext = SAITester.createIndexContext(index, UTF8Type.instance); + final IndexIdentifier indexIdentifier = createIndexIdentifier("test", "test", newIndex()); final List> termsEnum = buildTermsEnum(terms, postings); SegmentMetadata.ComponentMetadataMap indexMetas; - try (LiteralIndexWriter writer = new LiteralIndexWriter(indexDescriptor, indexContext)) + try (LiteralIndexWriter writer = new LiteralIndexWriter(indexDescriptor, indexIdentifier)) { indexMetas = writer.writeCompleteSegment(new MemtableTermsIterator(null, null, termsEnum.iterator())); } - FileHandle termsData = indexDescriptor.createPerIndexFileHandle(IndexComponent.TERMS_DATA, indexContext, null); - FileHandle postingLists = indexDescriptor.createPerIndexFileHandle(IndexComponent.POSTING_LISTS, indexContext, null); + FileHandle termsData = indexDescriptor.createPerIndexFileHandle(IndexComponent.TERMS_DATA, indexIdentifier, null); + FileHandle postingLists = indexDescriptor.createPerIndexFileHandle(IndexComponent.POSTING_LISTS, indexIdentifier, null); try (TermsIterator iterator = new TermsScanner(termsData, postingLists, indexMetas.get(IndexComponent.TERMS_DATA).root)) { @@ -99,22 +96,21 @@ public class TermsReaderTest extends SAIRandomizedTester private void testTermQueries(int numTerms, int numPostings) throws IOException { final IndexDescriptor indexDescriptor = newIndexDescriptor(); - final String index = newIndex(); - final IndexContext indexContext = SAITester.createIndexContext(index, UTF8Type.instance); + final IndexIdentifier indexIdentifier = createIndexIdentifier("test", "test", newIndex()); final List> termsEnum = buildTermsEnum(numTerms, numPostings); SegmentMetadata.ComponentMetadataMap indexMetas; - try (LiteralIndexWriter writer = new LiteralIndexWriter(indexDescriptor, indexContext)) + try (LiteralIndexWriter writer = new LiteralIndexWriter(indexDescriptor, indexIdentifier)) { indexMetas = writer.writeCompleteSegment(new MemtableTermsIterator(null, null, termsEnum.iterator())); } - FileHandle termsData = indexDescriptor.createPerIndexFileHandle(IndexComponent.TERMS_DATA, indexContext, null); - FileHandle postingLists = indexDescriptor.createPerIndexFileHandle(IndexComponent.POSTING_LISTS, indexContext, null); + FileHandle termsData = indexDescriptor.createPerIndexFileHandle(IndexComponent.TERMS_DATA, indexIdentifier, null); + FileHandle postingLists = indexDescriptor.createPerIndexFileHandle(IndexComponent.POSTING_LISTS, indexIdentifier, null); long termsFooterPointer = Long.parseLong(indexMetas.get(IndexComponent.TERMS_DATA).attributes.get(SAICodecUtils.FOOTER_POINTER)); - try (LiteralIndexSegmentTermsReader reader = new LiteralIndexSegmentTermsReader(indexContext, + try (LiteralIndexSegmentTermsReader reader = new LiteralIndexSegmentTermsReader(indexIdentifier, termsData, postingLists, indexMetas.get(IndexComponent.TERMS_DATA).root, diff --git a/test/unit/org/apache/cassandra/index/sai/disk/v1/bbtree/BlockBalancedTreeIndexBuilder.java b/test/unit/org/apache/cassandra/index/sai/disk/v1/bbtree/BlockBalancedTreeIndexBuilder.java index d34a56bd00..a7b629f20b 100644 --- a/test/unit/org/apache/cassandra/index/sai/disk/v1/bbtree/BlockBalancedTreeIndexBuilder.java +++ b/test/unit/org/apache/cassandra/index/sai/disk/v1/bbtree/BlockBalancedTreeIndexBuilder.java @@ -40,18 +40,18 @@ import org.apache.cassandra.db.marshal.ShortType; import org.apache.cassandra.db.marshal.UTF8Type; import org.apache.cassandra.dht.Murmur3Partitioner; import org.apache.cassandra.dht.Token; -import org.apache.cassandra.index.sai.IndexContext; import org.apache.cassandra.index.sai.SAITester; +import org.apache.cassandra.index.sai.StorageAttachedIndex; import org.apache.cassandra.index.sai.disk.PrimaryKeyMap; import org.apache.cassandra.index.sai.disk.format.IndexDescriptor; -import org.apache.cassandra.index.sai.disk.v1.segment.NumericIndexSegmentSearcher; import org.apache.cassandra.index.sai.disk.v1.PerColumnIndexFiles; import org.apache.cassandra.index.sai.disk.v1.segment.IndexSegmentSearcher; +import org.apache.cassandra.index.sai.disk.v1.segment.NumericIndexSegmentSearcher; import org.apache.cassandra.index.sai.disk.v1.segment.SegmentMetadata; import org.apache.cassandra.index.sai.memory.MemtableTermsIterator; +import org.apache.cassandra.index.sai.utils.IndexTermType; import org.apache.cassandra.index.sai.utils.PrimaryKey; import org.apache.cassandra.index.sai.utils.TermsIterator; -import org.apache.cassandra.index.sai.utils.TypeUtil; import org.apache.cassandra.utils.AbstractGuavaIterator; import org.apache.cassandra.utils.Pair; import org.apache.cassandra.utils.bytecomparable.ByteComparable; @@ -98,38 +98,36 @@ public class BlockBalancedTreeIndexBuilder private static final BigDecimal ONE_TENTH = BigDecimal.valueOf(1, 1); private final IndexDescriptor indexDescriptor; - private final AbstractType type; private final AbstractGuavaIterator> terms; private final int size; private final int minSegmentRowId; private final int maxSegmentRowId; public BlockBalancedTreeIndexBuilder(IndexDescriptor indexDescriptor, - AbstractType type, AbstractGuavaIterator> terms, int size, int minSegmentRowId, int maxSegmentRowId) { this.indexDescriptor = indexDescriptor; - this.type = type; this.terms = terms; this.size = size; this.minSegmentRowId = minSegmentRowId; this.maxSegmentRowId = maxSegmentRowId; } - NumericIndexSegmentSearcher flushAndOpen() throws IOException + NumericIndexSegmentSearcher flushAndOpen(AbstractType type) throws IOException { final TermsIterator termEnum = new MemtableTermsIterator(null, null, terms); final SegmentMetadata metadata; - IndexContext columnContext = SAITester.createIndexContext("test", Int32Type.instance); + StorageAttachedIndex index = SAITester.createMockIndex("test", type); + NumericIndexWriter writer = new NumericIndexWriter(indexDescriptor, - columnContext, - TypeUtil.fixedSizeOf(type), + index.identifier(), + index.termType().fixedSizeOf(), maxSegmentRowId); - final SegmentMetadata.ComponentMetadataMap indexMetas = writer.writeCompleteSegment(BlockBalancedTreeIterator.fromTermsIterator(termEnum, type)); + final SegmentMetadata.ComponentMetadataMap indexMetas = writer.writeCompleteSegment(BlockBalancedTreeIterator.fromTermsIterator(termEnum, index.termType())); metadata = new SegmentMetadata(0, size, minSegmentRowId, @@ -141,9 +139,9 @@ public class BlockBalancedTreeIndexBuilder UTF8Type.instance.fromString("d"), indexMetas); - try (PerColumnIndexFiles indexFiles = new PerColumnIndexFiles(indexDescriptor, SAITester.createIndexContext("test", Int32Type.instance))) + try (PerColumnIndexFiles indexFiles = new PerColumnIndexFiles(indexDescriptor, index.termType(), index.identifier())) { - IndexSegmentSearcher searcher = IndexSegmentSearcher.open(TEST_PRIMARY_KEY_MAP_FACTORY, indexFiles, metadata, columnContext); + IndexSegmentSearcher searcher = IndexSegmentSearcher.open(TEST_PRIMARY_KEY_MAP_FACTORY, indexFiles, metadata, index); assertThat(searcher, is(instanceOf(NumericIndexSegmentSearcher.class))); return (NumericIndexSegmentSearcher) searcher; } @@ -161,12 +159,11 @@ public class BlockBalancedTreeIndexBuilder final int size = endTermExclusive - startTermInclusive; Assert.assertTrue(size > 0); BlockBalancedTreeIndexBuilder indexBuilder = new BlockBalancedTreeIndexBuilder(indexDescriptor, - Int32Type.instance, singleOrd(int32Range(startTermInclusive, endTermExclusive), Int32Type.instance, startTermInclusive, size), size, startTermInclusive, endTermExclusive); - return indexBuilder.flushAndOpen(); + return indexBuilder.flushAndOpen(Int32Type.instance); } public static IndexSegmentSearcher buildDecimalSearcher(IndexDescriptor indexDescriptor, BigDecimal startTermInclusive, BigDecimal endTermExclusive) @@ -176,12 +173,11 @@ public class BlockBalancedTreeIndexBuilder int size = bigDifference.intValueExact() * 10; Assert.assertTrue(size > 0); BlockBalancedTreeIndexBuilder indexBuilder = new BlockBalancedTreeIndexBuilder(indexDescriptor, - DecimalType.instance, singleOrd(decimalRange(startTermInclusive, endTermExclusive), DecimalType.instance, startTermInclusive.intValueExact() * 10, size), size, startTermInclusive.intValueExact() * 10, endTermExclusive.intValueExact() * 10); - return indexBuilder.flushAndOpen(); + return indexBuilder.flushAndOpen(DecimalType.instance); } public static IndexSegmentSearcher buildBigIntegerSearcher(IndexDescriptor indexDescriptor, BigInteger startTermInclusive, BigInteger endTermExclusive) @@ -191,12 +187,11 @@ public class BlockBalancedTreeIndexBuilder int size = bigDifference.intValueExact(); Assert.assertTrue(size > 0); BlockBalancedTreeIndexBuilder indexBuilder = new BlockBalancedTreeIndexBuilder(indexDescriptor, - IntegerType.instance, singleOrd(bigIntegerRange(startTermInclusive, endTermExclusive), IntegerType.instance, startTermInclusive.intValueExact(), size), size, startTermInclusive.intValueExact(), endTermExclusive.intValueExact()); - return indexBuilder.flushAndOpen(); + return indexBuilder.flushAndOpen(IntegerType.instance); } /** @@ -211,12 +206,11 @@ public class BlockBalancedTreeIndexBuilder final long size = endTermExclusive - startTermInclusive; Assert.assertTrue(size > 0); BlockBalancedTreeIndexBuilder indexBuilder = new BlockBalancedTreeIndexBuilder(indexDescriptor, - LongType.instance, singleOrd(longRange(startTermInclusive, endTermExclusive), LongType.instance, Math.toIntExact(startTermInclusive), Math.toIntExact(size)), Math.toIntExact(size), Math.toIntExact(startTermInclusive), Math.toIntExact(endTermExclusive)); - return indexBuilder.flushAndOpen(); + return indexBuilder.flushAndOpen(LongType.instance); } /** @@ -231,12 +225,11 @@ public class BlockBalancedTreeIndexBuilder final int size = endTermExclusive - startTermInclusive; Assert.assertTrue(size > 0); BlockBalancedTreeIndexBuilder indexBuilder = new BlockBalancedTreeIndexBuilder(indexDescriptor, - ShortType.instance, singleOrd(shortRange(startTermInclusive, endTermExclusive), ShortType.instance, startTermInclusive, size), size, startTermInclusive, endTermExclusive); - return indexBuilder.flushAndOpen(); + return indexBuilder.flushAndOpen(ShortType.instance); } /** @@ -245,6 +238,7 @@ public class BlockBalancedTreeIndexBuilder */ public static AbstractGuavaIterator> singleOrd(Iterator terms, AbstractType type, int segmentRowIdOffset, int size) { + IndexTermType indexTermType = SAITester.createIndexTermType(type); return new AbstractGuavaIterator>() { private long currentTerm = 0; @@ -262,7 +256,7 @@ public class BlockBalancedTreeIndexBuilder postings.add(currentSegmentRowId++); assertTrue(terms.hasNext()); - final ByteSource encoded = TypeUtil.asComparableBytes(terms.next(), type, ByteComparable.Version.OSS50); + final ByteSource encoded = indexTermType.asComparableBytes(terms.next(), ByteComparable.Version.OSS50); return Pair.create(v -> encoded, postings); } }; @@ -305,9 +299,10 @@ public class BlockBalancedTreeIndexBuilder return result; } }; + IndexTermType indexTermType = SAITester.createIndexTermType(DecimalType.instance); return Stream.generate(generator) .limit(n) - .map(bd -> TypeUtil.asIndexBytes(DecimalType.instance.decompose(bd), DecimalType.instance)) + .map(bd -> indexTermType.asIndexBytes(DecimalType.instance.decompose(bd))) .collect(Collectors.toList()) .iterator(); } @@ -325,9 +320,10 @@ public class BlockBalancedTreeIndexBuilder return result; } }; + IndexTermType indexTermType = SAITester.createIndexTermType(IntegerType.instance); return Stream.generate(generator) .limit(n) - .map(bd -> TypeUtil.asIndexBytes(IntegerType.instance.decompose(bd), IntegerType.instance)) + .map(bd -> indexTermType.asIndexBytes(IntegerType.instance.decompose(bd))) .collect(Collectors.toList()) .iterator(); } diff --git a/test/unit/org/apache/cassandra/index/sai/disk/v1/bbtree/BlockBalancedTreePostingsWriterTest.java b/test/unit/org/apache/cassandra/index/sai/disk/v1/bbtree/BlockBalancedTreePostingsWriterTest.java index 509027663b..ead3971967 100644 --- a/test/unit/org/apache/cassandra/index/sai/disk/v1/bbtree/BlockBalancedTreePostingsWriterTest.java +++ b/test/unit/org/apache/cassandra/index/sai/disk/v1/bbtree/BlockBalancedTreePostingsWriterTest.java @@ -26,12 +26,11 @@ import org.junit.Before; import org.junit.Test; import org.agrona.collections.IntArrayList; -import org.apache.cassandra.db.marshal.Int32Type; -import org.apache.cassandra.index.sai.IndexContext; import org.apache.cassandra.index.sai.SAITester; import org.apache.cassandra.index.sai.disk.ArrayPostingList; import org.apache.cassandra.index.sai.disk.format.IndexComponent; import org.apache.cassandra.index.sai.disk.format.IndexDescriptor; +import org.apache.cassandra.index.sai.utils.IndexIdentifier; import org.apache.cassandra.index.sai.disk.io.IndexOutputWriter; import org.apache.cassandra.index.sai.disk.v1.postings.PostingsReader; import org.apache.cassandra.index.sai.metrics.QueryEventListener; @@ -48,13 +47,13 @@ import static org.mockito.Mockito.mock; public class BlockBalancedTreePostingsWriterTest extends SAIRandomizedTester { private IndexDescriptor indexDescriptor; - private IndexContext indexContext; + private IndexIdentifier indexIdentifier; @Before public void setup() throws Throwable { indexDescriptor = newIndexDescriptor(); - indexContext = SAITester.createIndexContext(newIndex(), Int32Type.instance); + indexIdentifier = SAITester.createIndexIdentifier("test", "test", newIndex()); } @Test @@ -73,12 +72,12 @@ public class BlockBalancedTreePostingsWriterTest extends SAIRandomizedTester writer.onLeaf(112, 4, pathToRoot(1, 3, 7, 14, 28)); long fp; - try (IndexOutputWriter output = indexDescriptor.openPerIndexOutput(IndexComponent.POSTING_LISTS, indexContext)) + try (IndexOutputWriter output = indexDescriptor.openPerIndexOutput(IndexComponent.POSTING_LISTS, indexIdentifier)) { - fp = writer.finish(output, leaves, indexContext); + fp = writer.finish(output, leaves, indexIdentifier); } - BlockBalancedTreePostingsIndex postingsIndex = new BlockBalancedTreePostingsIndex(indexDescriptor.createPerIndexFileHandle(IndexComponent.POSTING_LISTS, indexContext, null), fp); + BlockBalancedTreePostingsIndex postingsIndex = new BlockBalancedTreePostingsIndex(indexDescriptor.createPerIndexFileHandle(IndexComponent.POSTING_LISTS, indexIdentifier, null), fp); assertEquals(10, postingsIndex.size()); // Internal postings... @@ -120,13 +119,13 @@ public class BlockBalancedTreePostingsWriterTest extends SAIRandomizedTester writer.onLeaf(16, 1, pathToRoot(1, 2, 4, 8)); long fp; - try (IndexOutputWriter output = indexDescriptor.openPerIndexOutput(IndexComponent.POSTING_LISTS, indexContext)) + try (IndexOutputWriter output = indexDescriptor.openPerIndexOutput(IndexComponent.POSTING_LISTS, indexIdentifier)) { - fp = writer.finish(output, leaves, indexContext); + fp = writer.finish(output, leaves, indexIdentifier); } // There is only a single posting list...the leaf posting list. - BlockBalancedTreePostingsIndex postingsIndex = new BlockBalancedTreePostingsIndex(indexDescriptor.createPerIndexFileHandle(IndexComponent.POSTING_LISTS, indexContext, null), fp); + BlockBalancedTreePostingsIndex postingsIndex = new BlockBalancedTreePostingsIndex(indexDescriptor.createPerIndexFileHandle(IndexComponent.POSTING_LISTS, indexIdentifier, null), fp); assertEquals(1, postingsIndex.size()); } @@ -142,19 +141,19 @@ public class BlockBalancedTreePostingsWriterTest extends SAIRandomizedTester writer.onLeaf(16, 1, pathToRoot(1, 2, 4, 8)); long fp; - try (IndexOutputWriter output = indexDescriptor.openPerIndexOutput(IndexComponent.POSTING_LISTS, indexContext)) + try (IndexOutputWriter output = indexDescriptor.openPerIndexOutput(IndexComponent.POSTING_LISTS, indexIdentifier)) { - fp = writer.finish(output, leaves, indexContext); + fp = writer.finish(output, leaves, indexIdentifier); } // There is only a single posting list...the leaf posting list. - BlockBalancedTreePostingsIndex postingsIndex = new BlockBalancedTreePostingsIndex(indexDescriptor.createPerIndexFileHandle(IndexComponent.POSTING_LISTS, indexContext, null), fp); + BlockBalancedTreePostingsIndex postingsIndex = new BlockBalancedTreePostingsIndex(indexDescriptor.createPerIndexFileHandle(IndexComponent.POSTING_LISTS, indexIdentifier, null), fp); assertEquals(1, postingsIndex.size()); } private void assertPostingReaderEquals(BlockBalancedTreePostingsIndex postingsIndex, int nodeID, long... postings) throws IOException { - assertPostingReaderEquals(indexDescriptor.openPerIndexInput(IndexComponent.POSTING_LISTS, indexContext), + assertPostingReaderEquals(indexDescriptor.openPerIndexInput(IndexComponent.POSTING_LISTS, indexIdentifier), postingsIndex.getPostingsFilePointer(nodeID), new ArrayPostingList(postings)); } diff --git a/test/unit/org/apache/cassandra/index/sai/disk/v1/bbtree/BlockBalancedTreeQueriesTest.java b/test/unit/org/apache/cassandra/index/sai/disk/v1/bbtree/BlockBalancedTreeQueriesTest.java index a055236481..9fc940c7bf 100644 --- a/test/unit/org/apache/cassandra/index/sai/disk/v1/bbtree/BlockBalancedTreeQueriesTest.java +++ b/test/unit/org/apache/cassandra/index/sai/disk/v1/bbtree/BlockBalancedTreeQueriesTest.java @@ -21,7 +21,6 @@ import org.junit.Test; import org.apache.cassandra.cql3.Operator; import org.apache.cassandra.db.marshal.Int32Type; -import org.apache.cassandra.index.sai.SAITester; import org.apache.cassandra.index.sai.plan.Expression; import org.apache.cassandra.index.sai.utils.SAIRandomizedTester; import org.apache.cassandra.utils.bytecomparable.ByteComparable; @@ -40,7 +39,7 @@ public class BlockBalancedTreeQueriesTest extends SAIRandomizedTester @Test public void testMatchesAll() { - Expression expression = new Expression(SAITester.createIndexContext("meh", Int32Type.instance)); + Expression expression = Expression.create(createMockIndex("meh", Int32Type.instance)); BlockBalancedTreeReader.IntersectVisitor query = BlockBalancedTreeQueries.balancedTreeQueryFrom(expression, 4); for (int visit = 0; visit < between(100, 1000); visit++) @@ -213,7 +212,7 @@ public class BlockBalancedTreeQueriesTest extends SAIRandomizedTester private Expression buildExpression(Operator op, int value) { - Expression expression = new Expression(SAITester.createIndexContext("meh", Int32Type.instance)); + Expression expression = Expression.create(createMockIndex("meh", Int32Type.instance)); expression.add(op, Int32Type.instance.decompose(value)); return expression; } diff --git a/test/unit/org/apache/cassandra/index/sai/disk/v1/bbtree/BlockBalancedTreeRamBufferTest.java b/test/unit/org/apache/cassandra/index/sai/disk/v1/bbtree/BlockBalancedTreeRamBufferTest.java index da5faa9d3d..efc5394f9d 100644 --- a/test/unit/org/apache/cassandra/index/sai/disk/v1/bbtree/BlockBalancedTreeRamBufferTest.java +++ b/test/unit/org/apache/cassandra/index/sai/disk/v1/bbtree/BlockBalancedTreeRamBufferTest.java @@ -27,8 +27,8 @@ import org.junit.Test; import org.apache.cassandra.db.marshal.Int32Type; import org.apache.cassandra.index.sai.postings.PostingList; +import org.apache.cassandra.index.sai.utils.IndexTermType; import org.apache.cassandra.index.sai.utils.SAIRandomizedTester; -import org.apache.cassandra.index.sai.utils.TypeUtil; import org.apache.cassandra.utils.Pair; import org.apache.cassandra.utils.bytecomparable.ByteComparable; import org.apache.cassandra.utils.bytecomparable.ByteSource; @@ -51,9 +51,11 @@ public class BlockBalancedTreeRamBufferTest extends SAIRandomizedTester BlockBalancedTreeRamBuffer buffer = new BlockBalancedTreeRamBuffer(Integer.BYTES); + IndexTermType indexTermType = createIndexTermType(Int32Type.instance); + byte[] scratch = new byte[Integer.BYTES]; values.forEach(v -> { - TypeUtil.toComparableBytes(Int32Type.instance.decompose(v), Int32Type.instance, scratch); + indexTermType.toComparableBytes(Int32Type.instance.decompose(v), scratch); buffer.add(0, scratch); }); diff --git a/test/unit/org/apache/cassandra/index/sai/disk/v1/bbtree/BlockBalancedTreeReaderTest.java b/test/unit/org/apache/cassandra/index/sai/disk/v1/bbtree/BlockBalancedTreeReaderTest.java index 028257a725..4daba51446 100644 --- a/test/unit/org/apache/cassandra/index/sai/disk/v1/bbtree/BlockBalancedTreeReaderTest.java +++ b/test/unit/org/apache/cassandra/index/sai/disk/v1/bbtree/BlockBalancedTreeReaderTest.java @@ -29,9 +29,9 @@ import org.junit.Test; import org.apache.cassandra.cql3.Operator; import org.apache.cassandra.db.marshal.Int32Type; -import org.apache.cassandra.index.sai.IndexContext; import org.apache.cassandra.index.sai.QueryContext; import org.apache.cassandra.index.sai.SAITester; +import org.apache.cassandra.index.sai.StorageAttachedIndex; import org.apache.cassandra.index.sai.disk.format.IndexComponent; import org.apache.cassandra.index.sai.disk.format.IndexDescriptor; import org.apache.cassandra.index.sai.disk.v1.segment.SegmentMetadata; @@ -90,13 +90,13 @@ public class BlockBalancedTreeReaderTest extends SAIRandomizedTester }; private IndexDescriptor indexDescriptor; - private IndexContext indexContext; + private StorageAttachedIndex index; @Before public void setup() throws Throwable { indexDescriptor = newIndexDescriptor(); - indexContext = SAITester.createIndexContext(newIndex(), Int32Type.instance); + index = SAITester.createMockIndex(newIndex(), Int32Type.instance); } @Test @@ -260,7 +260,7 @@ public class BlockBalancedTreeReaderTest extends SAIRandomizedTester @SuppressWarnings("SameParameterValue") private void assertRange(BlockBalancedTreeReader reader, long lowerBound, long upperBound) { - Expression expression = new Expression(indexContext); + Expression expression = Expression.create(index); expression.add(Operator.GT, Int32Type.instance.decompose(444)); expression.add(Operator.LT, Int32Type.instance.decompose(555)); @@ -323,7 +323,7 @@ public class BlockBalancedTreeReaderTest extends SAIRandomizedTester { setBDKPostingsWriterSizing(8, 2); final NumericIndexWriter writer = new NumericIndexWriter(indexDescriptor, - indexContext, + index.identifier(), maxPointsPerLeaf, Integer.BYTES, Math.toIntExact(buffer.numRows())); @@ -334,9 +334,9 @@ public class BlockBalancedTreeReaderTest extends SAIRandomizedTester final long postingsPosition = metadata.get(IndexComponent.POSTING_LISTS).root; assertThat(postingsPosition, is(greaterThan(0L))); - FileHandle treeHandle = indexDescriptor.createPerIndexFileHandle(IndexComponent.BALANCED_TREE, indexContext, null); - FileHandle treePostingsHandle = indexDescriptor.createPerIndexFileHandle(IndexComponent.POSTING_LISTS, indexContext, null); - return new BlockBalancedTreeReader(indexContext, + FileHandle treeHandle = indexDescriptor.createPerIndexFileHandle(IndexComponent.BALANCED_TREE, index.identifier()); + FileHandle treePostingsHandle = indexDescriptor.createPerIndexFileHandle(IndexComponent.POSTING_LISTS, index.identifier()); + return new BlockBalancedTreeReader(index.identifier(), treeHandle, treePosition, treePostingsHandle, diff --git a/test/unit/org/apache/cassandra/index/sai/disk/v1/bbtree/NumericIndexWriterTest.java b/test/unit/org/apache/cassandra/index/sai/disk/v1/bbtree/NumericIndexWriterTest.java index eba8ed6ad3..be639ed430 100644 --- a/test/unit/org/apache/cassandra/index/sai/disk/v1/bbtree/NumericIndexWriterTest.java +++ b/test/unit/org/apache/cassandra/index/sai/disk/v1/bbtree/NumericIndexWriterTest.java @@ -24,18 +24,18 @@ import org.junit.Test; import com.carrotsearch.hppc.LongArrayList; import org.apache.cassandra.db.marshal.Int32Type; -import org.apache.cassandra.index.sai.IndexContext; import org.apache.cassandra.index.sai.QueryContext; import org.apache.cassandra.index.sai.SAITester; import org.apache.cassandra.index.sai.disk.format.IndexComponent; import org.apache.cassandra.index.sai.disk.format.IndexDescriptor; +import org.apache.cassandra.index.sai.utils.IndexIdentifier; import org.apache.cassandra.index.sai.disk.v1.segment.SegmentMetadata; import org.apache.cassandra.index.sai.memory.MemtableTermsIterator; import org.apache.cassandra.index.sai.metrics.QueryEventListener; import org.apache.cassandra.index.sai.postings.PostingList; +import org.apache.cassandra.index.sai.utils.IndexTermType; import org.apache.cassandra.index.sai.utils.SAIRandomizedTester; import org.apache.cassandra.index.sai.utils.TermsIterator; -import org.apache.cassandra.index.sai.utils.TypeUtil; import org.apache.cassandra.io.util.FileHandle; import org.apache.cassandra.utils.AbstractGuavaIterator; import org.apache.cassandra.utils.Pair; @@ -52,13 +52,15 @@ import static org.mockito.Mockito.when; public class NumericIndexWriterTest extends SAIRandomizedTester { private IndexDescriptor indexDescriptor; - private IndexContext indexContext; + private IndexTermType indexTermType; + private IndexIdentifier indexIdentifier; @Before public void setup() throws Throwable { indexDescriptor = newIndexDescriptor(); - indexContext = SAITester.createIndexContext(newIndex(), Int32Type.instance); + indexTermType = SAITester.createIndexTermType(Int32Type.instance); + indexIdentifier = SAITester.createIndexIdentifier("test", "test", newIndex()); } @Test @@ -79,15 +81,15 @@ public class NumericIndexWriterTest extends SAIRandomizedTester SegmentMetadata.ComponentMetadataMap indexMetas; NumericIndexWriter writer = new NumericIndexWriter(indexDescriptor, - indexContext, + indexIdentifier, Integer.BYTES, rowCount); indexMetas = writer.writeCompleteSegment(ramBuffer.iterator()); - final FileHandle treeHandle = indexDescriptor.createPerIndexFileHandle(IndexComponent.BALANCED_TREE, indexContext, null); - final FileHandle treePostingsHandle = indexDescriptor.createPerIndexFileHandle(IndexComponent.POSTING_LISTS, indexContext, null); + final FileHandle treeHandle = indexDescriptor.createPerIndexFileHandle(IndexComponent.BALANCED_TREE, indexIdentifier, null); + final FileHandle treePostingsHandle = indexDescriptor.createPerIndexFileHandle(IndexComponent.POSTING_LISTS, indexIdentifier, null); - try (BlockBalancedTreeReader reader = new BlockBalancedTreeReader(indexContext, + try (BlockBalancedTreeReader reader = new BlockBalancedTreeReader(indexIdentifier, treeHandle, indexMetas.get(IndexComponent.BALANCED_TREE).root, treePostingsHandle, @@ -125,15 +127,15 @@ public class NumericIndexWriterTest extends SAIRandomizedTester SegmentMetadata.ComponentMetadataMap indexMetas; NumericIndexWriter writer = new NumericIndexWriter(indexDescriptor, - indexContext, - TypeUtil.fixedSizeOf(Int32Type.instance), + indexIdentifier, + indexTermType.fixedSizeOf(), maxSegmentRowId); - indexMetas = writer.writeCompleteSegment(BlockBalancedTreeIterator.fromTermsIterator(termEnum, Int32Type.instance)); + indexMetas = writer.writeCompleteSegment(BlockBalancedTreeIterator.fromTermsIterator(termEnum, indexTermType)); - final FileHandle treeHandle = indexDescriptor.createPerIndexFileHandle(IndexComponent.BALANCED_TREE, indexContext, null); - final FileHandle treePostingsHandle = indexDescriptor.createPerIndexFileHandle(IndexComponent.POSTING_LISTS, indexContext, null); + final FileHandle treeHandle = indexDescriptor.createPerIndexFileHandle(IndexComponent.BALANCED_TREE, indexIdentifier, null); + final FileHandle treePostingsHandle = indexDescriptor.createPerIndexFileHandle(IndexComponent.POSTING_LISTS, indexIdentifier, null); - try (BlockBalancedTreeReader reader = new BlockBalancedTreeReader(indexContext, + try (BlockBalancedTreeReader reader = new BlockBalancedTreeReader(indexIdentifier, treeHandle, indexMetas.get(IndexComponent.BALANCED_TREE).root, treePostingsHandle, diff --git a/test/unit/org/apache/cassandra/index/sai/disk/v1/postings/PostingsTest.java b/test/unit/org/apache/cassandra/index/sai/disk/v1/postings/PostingsTest.java index 7397e47682..c437ffbdeb 100644 --- a/test/unit/org/apache/cassandra/index/sai/disk/v1/postings/PostingsTest.java +++ b/test/unit/org/apache/cassandra/index/sai/disk/v1/postings/PostingsTest.java @@ -26,9 +26,8 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; -import org.apache.cassandra.db.marshal.UTF8Type; -import org.apache.cassandra.index.sai.IndexContext; import org.apache.cassandra.index.sai.SAITester; +import org.apache.cassandra.index.sai.utils.IndexIdentifier; import org.apache.cassandra.index.sai.postings.PostingList; import org.apache.cassandra.index.sai.disk.format.IndexComponent; import org.apache.cassandra.index.sai.disk.format.IndexDescriptor; @@ -47,14 +46,16 @@ public class PostingsTest extends SAIRandomizedTester public final ExpectedException expectedException = ExpectedException.none(); private IndexDescriptor indexDescriptor; - private IndexContext indexContext; + private IndexIdentifier indexIdentifier; @Before public void setup() throws Throwable { indexDescriptor = newIndexDescriptor(); String index = newIndex(); - indexContext = SAITester.createIndexContext(index, UTF8Type.instance); + indexIdentifier = SAITester.createIndexIdentifier(indexDescriptor.sstableDescriptor.ksname, + indexDescriptor.sstableDescriptor.cfname, + index); } @@ -65,13 +66,13 @@ public class PostingsTest extends SAIRandomizedTester final ArrayPostingList expectedPostingList = new ArrayPostingList(10, 20, 30, 40, 50, 60); long postingPointer; - try (PostingsWriter writer = new PostingsWriter(indexDescriptor, indexContext, blockSize)) + try (PostingsWriter writer = new PostingsWriter(indexDescriptor, indexIdentifier, blockSize)) { postingPointer = writer.write(expectedPostingList); writer.complete(); } - IndexInput input = indexDescriptor.openPerIndexInput(IndexComponent.POSTING_LISTS, indexContext); + IndexInput input = indexDescriptor.openPerIndexInput(IndexComponent.POSTING_LISTS, indexIdentifier); SAICodecUtils.validate(input); input.seek(postingPointer); @@ -96,7 +97,7 @@ public class PostingsTest extends SAIRandomizedTester reader.close(); assertEquals(reader.size(), listener.decodes); - input = indexDescriptor.openPerIndexInput(IndexComponent.POSTING_LISTS, indexContext); + input = indexDescriptor.openPerIndexInput(IndexComponent.POSTING_LISTS, indexIdentifier); listener = new CountingPostingListEventListener(); reader = new PostingsReader(input, postingPointer, listener); @@ -118,7 +119,7 @@ public class PostingsTest extends SAIRandomizedTester final ArrayPostingList[] expected = new ArrayPostingList[numPostingLists]; final long[] postingPointers = new long[numPostingLists]; - try (PostingsWriter writer = new PostingsWriter(indexDescriptor, indexContext, blockSize)) + try (PostingsWriter writer = new PostingsWriter(indexDescriptor, indexIdentifier, blockSize)) { for (int i = 0; i < numPostingLists; ++i) { @@ -130,14 +131,14 @@ public class PostingsTest extends SAIRandomizedTester writer.complete(); } - try (IndexInput input = indexDescriptor.openPerIndexInput(IndexComponent.POSTING_LISTS, indexContext)) + try (IndexInput input = indexDescriptor.openPerIndexInput(IndexComponent.POSTING_LISTS, indexIdentifier)) { SAICodecUtils.validate(input); } for (int i = 0; i < numPostingLists; ++i) { - IndexInput input = indexDescriptor.openPerIndexInput(IndexComponent.POSTING_LISTS, indexContext); + IndexInput input = indexDescriptor.openPerIndexInput(IndexComponent.POSTING_LISTS, indexIdentifier); input.seek(postingPointers[i]); ArrayPostingList expectedPostingList = expected[i]; PostingsReader.BlocksSummary summary = assertBlockSummary(blockSize, expectedPostingList, input); @@ -156,7 +157,7 @@ public class PostingsTest extends SAIRandomizedTester // test random advances through the posting list listener = new CountingPostingListEventListener(); - input = indexDescriptor.openPerIndexInput(IndexComponent.POSTING_LISTS, indexContext); + input = indexDescriptor.openPerIndexInput(IndexComponent.POSTING_LISTS, indexIdentifier); try (PostingsReader reader = new PostingsReader(input, postingPointers[i], listener)) { expectedPostingList.reset(); @@ -171,7 +172,7 @@ public class PostingsTest extends SAIRandomizedTester // test skipping to the last block listener = new CountingPostingListEventListener(); - input = indexDescriptor.openPerIndexInput(IndexComponent.POSTING_LISTS, indexContext); + input = indexDescriptor.openPerIndexInput(IndexComponent.POSTING_LISTS, indexIdentifier); try (PostingsReader reader = new PostingsReader(input, postingPointers[i], listener)) { long tokenToAdvance = -1; @@ -200,13 +201,13 @@ public class PostingsTest extends SAIRandomizedTester final ArrayPostingList expectedPostingList = new ArrayPostingList(0, 1, 1, 3, 3, 5, 5, 7, 7, 7, 7, 7, 7, 9, 9, 10, 11, 12); long postingPointer; - try (PostingsWriter writer = new PostingsWriter(indexDescriptor, indexContext, blockSize)) + try (PostingsWriter writer = new PostingsWriter(indexDescriptor, indexIdentifier, blockSize)) { postingPointer = writer.write(expectedPostingList); writer.complete(); } - IndexInput input = indexDescriptor.openPerIndexInput(IndexComponent.POSTING_LISTS, indexContext); + IndexInput input = indexDescriptor.openPerIndexInput(IndexComponent.POSTING_LISTS, indexIdentifier); CountingPostingListEventListener listener = new CountingPostingListEventListener(); try (PostingsReader reader = new PostingsReader(input, postingPointer, listener)) { @@ -223,13 +224,13 @@ public class PostingsTest extends SAIRandomizedTester final ArrayPostingList expected = new ArrayPostingList(postings); long fp; - try (PostingsWriter writer = new PostingsWriter(indexDescriptor, indexContext, blockSize)) + try (PostingsWriter writer = new PostingsWriter(indexDescriptor, indexIdentifier, blockSize)) { fp = writer.write(expected); writer.complete(); } - try (IndexInput input = indexDescriptor.openPerIndexInput(IndexComponent.POSTING_LISTS, indexContext)) + try (IndexInput input = indexDescriptor.openPerIndexInput(IndexComponent.POSTING_LISTS, indexIdentifier)) { SAICodecUtils.validate(input); input.seek(fp); @@ -264,13 +265,13 @@ public class PostingsTest extends SAIRandomizedTester final ArrayPostingList expected = new ArrayPostingList(postings); long fp; - try (PostingsWriter writer = new PostingsWriter(indexDescriptor, indexContext, blockSize)) + try (PostingsWriter writer = new PostingsWriter(indexDescriptor, indexIdentifier, blockSize)) { fp = writer.write(expected); writer.complete(); } - try (IndexInput input = indexDescriptor.openPerIndexInput(IndexComponent.POSTING_LISTS, indexContext)) + try (IndexInput input = indexDescriptor.openPerIndexInput(IndexComponent.POSTING_LISTS, indexIdentifier)) { SAICodecUtils.validate(input); input.seek(fp); @@ -291,7 +292,7 @@ public class PostingsTest extends SAIRandomizedTester @SuppressWarnings("all") public void testNullPostingList() throws IOException { - try (PostingsWriter writer = new PostingsWriter(indexDescriptor, indexContext)) + try (PostingsWriter writer = new PostingsWriter(indexDescriptor, indexIdentifier)) { expectedException.expect(IllegalArgumentException.class); writer.write(null); @@ -302,7 +303,7 @@ public class PostingsTest extends SAIRandomizedTester @Test public void testEmptyPostingList() throws IOException { - try (PostingsWriter writer = new PostingsWriter(indexDescriptor, indexContext)) + try (PostingsWriter writer = new PostingsWriter(indexDescriptor, indexIdentifier)) { expectedException.expect(IllegalArgumentException.class); writer.write(new ArrayPostingList()); @@ -312,7 +313,7 @@ public class PostingsTest extends SAIRandomizedTester @Test public void testNonAscendingPostingList() throws IOException { - try (PostingsWriter writer = new PostingsWriter(indexDescriptor, indexContext)) + try (PostingsWriter writer = new PostingsWriter(indexDescriptor, indexIdentifier)) { expectedException.expect(IllegalArgumentException.class); writer.write(new ArrayPostingList(1, 0)); @@ -350,7 +351,7 @@ public class PostingsTest extends SAIRandomizedTester private PostingsReader openReader(long fp, QueryEventListener.PostingListEventListener listener) throws IOException { - IndexInput input = indexDescriptor.openPerIndexInput(IndexComponent.POSTING_LISTS, indexContext); + IndexInput input = indexDescriptor.openPerIndexInput(IndexComponent.POSTING_LISTS, indexIdentifier); input.seek(fp); return new PostingsReader(input, fp, listener); } diff --git a/test/unit/org/apache/cassandra/index/sai/disk/v1/trie/TrieTermsDictionaryTest.java b/test/unit/org/apache/cassandra/index/sai/disk/v1/trie/TrieTermsDictionaryTest.java index af11a390ba..8569fa9a90 100644 --- a/test/unit/org/apache/cassandra/index/sai/disk/v1/trie/TrieTermsDictionaryTest.java +++ b/test/unit/org/apache/cassandra/index/sai/disk/v1/trie/TrieTermsDictionaryTest.java @@ -28,11 +28,9 @@ import org.apache.commons.lang3.mutable.MutableLong; import org.junit.Before; import org.junit.Test; -import org.apache.cassandra.db.marshal.UTF8Type; -import org.apache.cassandra.index.sai.IndexContext; -import org.apache.cassandra.index.sai.SAITester; import org.apache.cassandra.index.sai.disk.format.IndexComponent; import org.apache.cassandra.index.sai.disk.format.IndexDescriptor; +import org.apache.cassandra.index.sai.utils.IndexIdentifier; import org.apache.cassandra.index.sai.utils.SAIRandomizedTester; import org.apache.cassandra.io.util.FileHandle; import org.apache.cassandra.utils.ByteBufferUtil; @@ -48,14 +46,13 @@ import static org.junit.Assert.assertTrue; public class TrieTermsDictionaryTest extends SAIRandomizedTester { private IndexDescriptor indexDescriptor; - private IndexContext indexContext; + private IndexIdentifier indexIdentifier; @Before public void setup() throws Throwable { indexDescriptor = newIndexDescriptor(); - String index = newIndex(); - indexContext = SAITester.createIndexContext(index, UTF8Type.instance); + indexIdentifier = createIndexIdentifier("test", "test", newIndex()); } @Test @@ -67,7 +64,7 @@ public class TrieTermsDictionaryTest extends SAIRandomizedTester private void doTestExactMatch() throws Exception { long fp; - try (TrieTermsDictionaryWriter writer = new TrieTermsDictionaryWriter(indexDescriptor, indexContext)) + try (TrieTermsDictionaryWriter writer = new TrieTermsDictionaryWriter(indexDescriptor, indexIdentifier)) { writer.add(asByteComparable("ab"), 0); writer.add(asByteComparable("abb"), 1); @@ -77,7 +74,7 @@ public class TrieTermsDictionaryTest extends SAIRandomizedTester fp = writer.complete(new MutableLong()); } - try (FileHandle input = indexDescriptor.createPerIndexFileHandle(IndexComponent.TERMS_DATA, indexContext, null); + try (FileHandle input = indexDescriptor.createPerIndexFileHandle(IndexComponent.TERMS_DATA, indexIdentifier); TrieTermsDictionaryReader reader = new TrieTermsDictionaryReader(input.instantiateRebufferer(null), fp)) { assertEquals(TrieTermsDictionaryReader.NOT_FOUND, reader.exactMatch(asByteComparable("a"))); @@ -95,7 +92,7 @@ public class TrieTermsDictionaryTest extends SAIRandomizedTester final List byteComparables = generateSortedByteComparables(); long fp; - try (TrieTermsDictionaryWriter writer = new TrieTermsDictionaryWriter(indexDescriptor, indexContext)) + try (TrieTermsDictionaryWriter writer = new TrieTermsDictionaryWriter(indexDescriptor, indexIdentifier)) { for (int i = 0; i < byteComparables.size(); ++i) { @@ -104,7 +101,7 @@ public class TrieTermsDictionaryTest extends SAIRandomizedTester fp = writer.complete(new MutableLong()); } - try (FileHandle input = indexDescriptor.createPerIndexFileHandle(IndexComponent.TERMS_DATA, indexContext, null); + try (FileHandle input = indexDescriptor.createPerIndexFileHandle(IndexComponent.TERMS_DATA, indexIdentifier); TrieTermsIterator iterator = new TrieTermsIterator(input.instantiateRebufferer(null), fp)) { final Iterator expected = byteComparables.iterator(); @@ -127,7 +124,7 @@ public class TrieTermsDictionaryTest extends SAIRandomizedTester final List byteComparables = generateSortedByteComparables(); long fp; - try (TrieTermsDictionaryWriter writer = new TrieTermsDictionaryWriter(indexDescriptor, indexContext)) + try (TrieTermsDictionaryWriter writer = new TrieTermsDictionaryWriter(indexDescriptor, indexIdentifier)) { for (int i = 0; i < byteComparables.size(); ++i) { @@ -136,7 +133,7 @@ public class TrieTermsDictionaryTest extends SAIRandomizedTester fp = writer.complete(new MutableLong()); } - try (FileHandle input = indexDescriptor.createPerIndexFileHandle(IndexComponent.TERMS_DATA, indexContext, null); + try (FileHandle input = indexDescriptor.createPerIndexFileHandle(IndexComponent.TERMS_DATA, indexIdentifier); TrieTermsDictionaryReader reader = new TrieTermsDictionaryReader(input.instantiateRebufferer(null), fp)) { final ByteComparable expectedMaxTerm = byteComparables.get(byteComparables.size() - 1); diff --git a/test/unit/org/apache/cassandra/index/sai/functional/CompactionTest.java b/test/unit/org/apache/cassandra/index/sai/functional/CompactionTest.java index aff3c5e862..5b38876de3 100644 --- a/test/unit/org/apache/cassandra/index/sai/functional/CompactionTest.java +++ b/test/unit/org/apache/cassandra/index/sai/functional/CompactionTest.java @@ -39,9 +39,10 @@ import org.apache.cassandra.db.lifecycle.LifecycleTransaction; import org.apache.cassandra.db.marshal.Int32Type; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; -import org.apache.cassandra.index.sai.IndexContext; import org.apache.cassandra.index.sai.SAITester; +import org.apache.cassandra.index.sai.utils.IndexIdentifier; import org.apache.cassandra.index.sai.disk.v1.SSTableIndexWriter; +import org.apache.cassandra.index.sai.utils.IndexTermType; import org.apache.cassandra.inject.ActionBuilder; import org.apache.cassandra.inject.Expression; import org.apache.cassandra.inject.Injection; @@ -72,7 +73,8 @@ public class CompactionTest extends SAITester public void testAntiCompaction() throws Throwable { createTable(CREATE_TABLE_TEMPLATE); - IndexContext numericIndexContext = createIndexContext(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1")), Int32Type.instance); + IndexIdentifier numericIndexIdentifier = createIndexIdentifier(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1"))); + IndexTermType numericIndexTermType = createIndexTermType(Int32Type.instance); verifyNoIndexFiles(); // create 100 rows in 1 sstable @@ -83,8 +85,8 @@ public class CompactionTest extends SAITester // verify 1 sstable index assertNumRows(num, "SELECT * FROM %%s WHERE v1 >= 0"); - verifyIndexFiles(numericIndexContext, null, 1, 0); - verifySSTableIndexes(numericIndexContext.getIndexName(), 1); + verifyIndexFiles(numericIndexTermType, numericIndexIdentifier, 1); + verifySSTableIndexes(numericIndexIdentifier, 1); // split sstable into repaired and unrepaired ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(currentTable()); @@ -110,8 +112,8 @@ public class CompactionTest extends SAITester // verify 2 sstable indexes assertNumRows(num, "SELECT * FROM %%s WHERE v1 >= 0"); - waitForAssert(() -> verifyIndexFiles(numericIndexContext, null, 2, 0)); - verifySSTableIndexes(numericIndexContext.getIndexName(), 2); + waitForAssert(() -> verifyIndexFiles(numericIndexTermType, numericIndexIdentifier, 2)); + verifySSTableIndexes(numericIndexIdentifier, 2); // index components are included after anti-compaction verifyIndexComponentsIncludedInSSTable(); @@ -121,8 +123,8 @@ public class CompactionTest extends SAITester public void testConcurrentQueryWithCompaction() { createTable(CREATE_TABLE_TEMPLATE); - String v1IndexName = createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1")); - String v2IndexName = createIndex(String.format(CREATE_INDEX_TEMPLATE, "v2")); + IndexIdentifier numericIndexIdentifier = createIndexIdentifier(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1"))); + IndexIdentifier literalIndexIdentifier = createIndexIdentifier(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v2"))); waitForTableIndexesQueryable(); int num = 10; @@ -149,16 +151,16 @@ public class CompactionTest extends SAITester compactionTest.start(); - verifySSTableIndexes(v1IndexName, num); - verifySSTableIndexes(v2IndexName, num); + verifySSTableIndexes(numericIndexIdentifier, num); + verifySSTableIndexes(literalIndexIdentifier, num); } @Test public void testAbortCompactionWithEarlyOpenSSTables() throws Throwable { createTable(CREATE_TABLE_TEMPLATE); - String v1IndexName = createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1")); - String v2IndexName = createIndex(String.format(CREATE_INDEX_TEMPLATE, "v2")); + IndexIdentifier numericIndexIdentifier = createIndexIdentifier(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1"))); + IndexIdentifier literalIndexIdentifier = createIndexIdentifier(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v2"))); int sstables = 2; int num = 10; @@ -207,8 +209,8 @@ public class CompactionTest extends SAITester // verify indexes are working assertNumRows(num, "SELECT id1 FROM %%s WHERE v1=0"); assertNumRows(num, "SELECT id1 FROM %%s WHERE v2='0'"); - verifySSTableIndexes(v1IndexName, sstables); - verifySSTableIndexes(v2IndexName, sstables); + verifySSTableIndexes(numericIndexIdentifier, sstables); + verifySSTableIndexes(literalIndexIdentifier, sstables); } @Test @@ -277,8 +279,8 @@ public class CompactionTest extends SAITester assertNumRows(num, "SELECT id1 FROM %%s WHERE v1>=0"); assertNumRows(num, "SELECT id1 FROM %%s WHERE v2='0'"); - verifySSTableIndexes(IndexMetadata.generateDefaultIndexName(currentTable(), V1_COLUMN_IDENTIFIER), sstables); - verifySSTableIndexes(IndexMetadata.generateDefaultIndexName(currentTable(), V2_COLUMN_IDENTIFIER), sstables); + verifySSTableIndexes(createIndexIdentifier(IndexMetadata.generateDefaultIndexName(currentTable(), V1_COLUMN_IDENTIFIER)), sstables); + verifySSTableIndexes(createIndexIdentifier(IndexMetadata.generateDefaultIndexName(currentTable(), V2_COLUMN_IDENTIFIER)), sstables); } @Test diff --git a/test/unit/org/apache/cassandra/index/sai/functional/FailureTest.java b/test/unit/org/apache/cassandra/index/sai/functional/FailureTest.java index 16b3164901..6907006e08 100644 --- a/test/unit/org/apache/cassandra/index/sai/functional/FailureTest.java +++ b/test/unit/org/apache/cassandra/index/sai/functional/FailureTest.java @@ -24,9 +24,10 @@ import org.junit.Test; import org.apache.cassandra.db.marshal.Int32Type; import org.apache.cassandra.index.IndexNotAvailableException; -import org.apache.cassandra.index.sai.IndexContext; import org.apache.cassandra.index.sai.SAITester; import org.apache.cassandra.index.sai.SSTableContext; +import org.apache.cassandra.index.sai.utils.IndexIdentifier; +import org.apache.cassandra.index.sai.utils.IndexTermType; import org.apache.cassandra.inject.Injection; import org.apache.cassandra.inject.Injections; import org.assertj.core.api.Assertions; @@ -39,7 +40,8 @@ public class FailureTest extends SAITester public void shouldMakeIndexNonQueryableOnSSTableContextFailureDuringFlush() throws Throwable { createTable(CREATE_TABLE_TEMPLATE); - IndexContext numericIndexContext = createIndexContext(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1")), Int32Type.instance); + IndexIdentifier indexIdentifier = createIndexIdentifier(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1"))); + IndexTermType indexTermType = createIndexTermType(Int32Type.instance); execute("INSERT INTO %s (id1, v1) VALUES ('1', 1)"); execute("INSERT INTO %s (id1, v1) VALUES ('2', 2)"); @@ -47,8 +49,8 @@ public class FailureTest extends SAITester assertEquals(1, execute("SELECT id1 FROM %s WHERE v1 > 1").size()); - verifyIndexFiles(numericIndexContext, null, 1, 1, 0, 1, 0); - verifySSTableIndexes(numericIndexContext.getIndexName(), 1, 1); + verifyIndexFiles(indexTermType, indexIdentifier, 1, 1, 1); + verifySSTableIndexes(indexIdentifier, 1, 1); execute("INSERT INTO %s (id1, v1) VALUES ('3', 3)"); @@ -66,8 +68,8 @@ public class FailureTest extends SAITester // Now verify that a restart actually repairs the index... simulateNodeRestart(); - verifyIndexFiles(numericIndexContext, null, 2, 0); - verifySSTableIndexes(numericIndexContext.getIndexName(), 2, 2); + verifyIndexFiles(indexTermType, indexIdentifier, 2, 2, 2); + verifySSTableIndexes(indexIdentifier, 2, 2); assertEquals(2, execute("SELECT id1 FROM %s WHERE v1 > 1").size()); } @@ -76,7 +78,8 @@ public class FailureTest extends SAITester public void shouldMakeIndexNonQueryableOnSSTableContextFailureDuringCompaction() throws Throwable { createTable(CREATE_TABLE_TEMPLATE); - IndexContext numericIndexContext = createIndexContext(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1")), Int32Type.instance); + IndexIdentifier indexIdentifier = createIndexIdentifier(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1"))); + IndexTermType indexTermType = createIndexTermType(Int32Type.instance); execute("INSERT INTO %s (id1, v1) VALUES ('1', 1)"); flush(); @@ -86,8 +89,8 @@ public class FailureTest extends SAITester assertEquals(1, execute("SELECT id1 FROM %s WHERE v1 > 1").size()); - verifyIndexFiles(numericIndexContext, null, 2, 2, 0, 2, 0); - verifySSTableIndexes(numericIndexContext.getIndexName(), 2, 2); + verifyIndexFiles(indexTermType, indexIdentifier, 2, 2, 2); + verifySSTableIndexes(indexIdentifier, 2, 2); Injection ssTableContextCreationFailure = newFailureOnEntry("context_failure_on_compaction", SSTableContext.class, "", RuntimeException.class); Injections.inject(ssTableContextCreationFailure); @@ -114,13 +117,13 @@ public class FailureTest extends SAITester Injection ssTableContextCreationFailure = newFailureOnEntry("context_failure_on_creation", SSTableContext.class, "", RuntimeException.class); Injections.inject(ssTableContextCreationFailure); - String v2IndexName = createIndexAsync(String.format(CREATE_INDEX_TEMPLATE, "v2")); + IndexIdentifier indexIdentifier = createIndexIdentifier(createIndexAsync(String.format(CREATE_INDEX_TEMPLATE, "v2"))); // Verify that the initial index build fails... - verifyInitialIndexFailed(v2IndexName); + verifyInitialIndexFailed(indexIdentifier.indexName); verifyNoIndexFiles(); - verifySSTableIndexes(v2IndexName, 0); + verifySSTableIndexes(indexIdentifier, 0); // ...and then verify that, while the node is still operational, the index is not. Assertions.assertThatThrownBy(() -> execute("SELECT * FROM %s WHERE v2 = '1'")) diff --git a/test/unit/org/apache/cassandra/index/sai/functional/FlushingTest.java b/test/unit/org/apache/cassandra/index/sai/functional/FlushingTest.java index 79a126d09c..fa480b5c8d 100644 --- a/test/unit/org/apache/cassandra/index/sai/functional/FlushingTest.java +++ b/test/unit/org/apache/cassandra/index/sai/functional/FlushingTest.java @@ -24,16 +24,17 @@ import org.junit.Test; import com.datastax.driver.core.ResultSet; import org.apache.cassandra.db.marshal.Int32Type; -import org.apache.cassandra.index.sai.IndexContext; import org.apache.cassandra.index.sai.SAITester; +import org.apache.cassandra.index.sai.utils.IndexIdentifier; import org.apache.cassandra.index.sai.disk.v1.bbtree.NumericIndexWriter; +import org.apache.cassandra.index.sai.utils.IndexTermType; import static org.junit.Assert.assertEquals; public class FlushingTest extends SAITester { @Test - public void testFlushingLargeStaleMemtableIndex() throws Throwable + public void testFlushingLargeStaleMemtableIndex() { createTable(CREATE_TABLE_TEMPLATE); createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1")); @@ -51,10 +52,11 @@ public class FlushingTest extends SAITester } @Test - public void testFlushingOverwriteDelete() throws Throwable + public void testFlushingOverwriteDelete() { createTable(CREATE_TABLE_TEMPLATE); - IndexContext numericIndexContext = createIndexContext(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1")), Int32Type.instance); + IndexIdentifier indexIdentifier = createIndexIdentifier(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1"))); + IndexTermType indexTermType = createIndexTermType(Int32Type.instance); int sstables = 3; for (int j = 0; j < sstables; j++) @@ -66,14 +68,14 @@ public class FlushingTest extends SAITester ResultSet rows = executeNet("SELECT id1 FROM %s WHERE v1 >= 0"); assertEquals(0, rows.all().size()); - verifyIndexFiles(numericIndexContext, null, sstables, 0, 0, sstables, 0); - verifySSTableIndexes(numericIndexContext.getIndexName(), sstables, 0); + verifyIndexFiles(indexTermType, indexIdentifier, sstables, 0, sstables); + verifySSTableIndexes(indexIdentifier, sstables, 0); compact(); - waitForAssert(() -> verifyIndexFiles(numericIndexContext, null, 1, 0, 0, 1, 0)); + waitForAssert(() -> verifyIndexFiles(indexTermType, indexIdentifier, 1, 0, 1)); rows = executeNet("SELECT id1 FROM %s WHERE v1 >= 0"); assertEquals(0, rows.all().size()); - verifySSTableIndexes(numericIndexContext.getIndexName(), 1, 0); + verifySSTableIndexes(indexIdentifier, 1, 0); } } diff --git a/test/unit/org/apache/cassandra/index/sai/functional/GroupComponentsTest.java b/test/unit/org/apache/cassandra/index/sai/functional/GroupComponentsTest.java index d245340d8d..0747f8eecf 100644 --- a/test/unit/org/apache/cassandra/index/sai/functional/GroupComponentsTest.java +++ b/test/unit/org/apache/cassandra/index/sai/functional/GroupComponentsTest.java @@ -28,11 +28,11 @@ import org.junit.Test; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.marshal.UTF8Type; -import org.apache.cassandra.index.sai.IndexContext; import org.apache.cassandra.index.sai.SAITester; import org.apache.cassandra.index.sai.StorageAttachedIndex; import org.apache.cassandra.index.sai.StorageAttachedIndexGroup; import org.apache.cassandra.index.sai.disk.format.Version; +import org.apache.cassandra.index.sai.utils.IndexTermType; import org.apache.cassandra.io.sstable.Component; import org.apache.cassandra.io.sstable.format.SSTableReader; @@ -61,7 +61,7 @@ public class GroupComponentsTest extends SAITester // index files are released but not removed cfs.invalidate(true, false); - Assert.assertTrue(index.getIndexContext().getView().getIndexes().isEmpty()); + Assert.assertTrue(index.view().getIndexes().isEmpty()); for (Component component : components) Assert.assertTrue(sstable.descriptor.fileFor(component).exists()); } @@ -91,7 +91,10 @@ public class GroupComponentsTest extends SAITester public void getLiveComponentsForPopulatedIndex() { createTable("CREATE TABLE %s (pk int primary key, value text)"); - IndexContext indexContext = createIndexContext(createIndex("CREATE INDEX ON %s(value) USING 'sai'"), UTF8Type.instance); + + createIndex("CREATE INDEX ON %s(value) USING 'sai'"); + IndexTermType indexTermType = createIndexTermType(UTF8Type.instance); + execute("INSERT INTO %s (pk, value) VALUES (1, '1')"); flush(); @@ -106,7 +109,7 @@ public class GroupComponentsTest extends SAITester Set components = StorageAttachedIndexGroup.getLiveComponents(sstables.iterator().next(), getIndexesFromGroup(group)); assertEquals(Version.LATEST.onDiskFormat().perSSTableIndexComponents(false).size() + - Version.LATEST.onDiskFormat().perColumnIndexComponents(indexContext).size(), + Version.LATEST.onDiskFormat().perColumnIndexComponents(indexTermType).size(), components.size()); } diff --git a/test/unit/org/apache/cassandra/index/sai/functional/NodeRestartTest.java b/test/unit/org/apache/cassandra/index/sai/functional/NodeRestartTest.java index 8ed9b6b7d9..1dcacec8b9 100644 --- a/test/unit/org/apache/cassandra/index/sai/functional/NodeRestartTest.java +++ b/test/unit/org/apache/cassandra/index/sai/functional/NodeRestartTest.java @@ -25,9 +25,10 @@ import org.junit.Test; import org.apache.cassandra.db.marshal.Int32Type; import org.apache.cassandra.db.marshal.UTF8Type; -import org.apache.cassandra.index.sai.IndexContext; import org.apache.cassandra.index.sai.SAITester; import org.apache.cassandra.index.sai.StorageAttachedIndex; +import org.apache.cassandra.index.sai.utils.IndexIdentifier; +import org.apache.cassandra.index.sai.utils.IndexTermType; import org.apache.cassandra.inject.Injection; import org.apache.cassandra.inject.Injections; import org.apache.cassandra.inject.InvokePointBuilder; @@ -112,17 +113,20 @@ public class NodeRestartTest extends SAITester execute("INSERT INTO %s (id1, v1, v2) VALUES ('0', 0, '0');"); flush(); - IndexContext numericIndexContext = createIndexContext(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1")), Int32Type.instance); - IndexContext literalIndexContext = createIndexContext(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v2")), UTF8Type.instance); - waitForTableIndexesQueryable(); - verifyIndexFiles(numericIndexContext,literalIndexContext, 1, 1); + IndexIdentifier numericIndexIdentifier = createIndexIdentifier(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1"))); + IndexIdentifier literalIndexIdentifier = createIndexIdentifier(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v2"))); + IndexTermType numericIndexTermType = createIndexTermType(Int32Type.instance); + IndexTermType literalIndexTermType = createIndexTermType(UTF8Type.instance); + verifyIndexFiles(numericIndexTermType, numericIndexIdentifier, 1); + verifyIndexFiles(literalIndexTermType, literalIndexIdentifier, 1); assertNumRows(1, "SELECT * FROM %%s WHERE v1 >= 0"); assertNumRows(1, "SELECT * FROM %%s WHERE v2 = '0'"); assertValidationCount(0, 0); simulateNodeRestart(); - verifyIndexFiles(numericIndexContext, literalIndexContext, 1, 1); + verifyIndexFiles(numericIndexTermType, numericIndexIdentifier, 1); + verifyIndexFiles(literalIndexTermType, literalIndexIdentifier, 1); assertNumRows(1, "SELECT * FROM %%s WHERE v1 >= 0"); assertNumRows(1, "SELECT * FROM %%s WHERE v2 = '0'"); @@ -174,9 +178,11 @@ public class NodeRestartTest extends SAITester execute("INSERT INTO %s (id1, v1, v2) VALUES ('0', 0, '0')"); flush(); - IndexContext numericIndexContext = createIndexContext(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1")), Int32Type.instance); + IndexIdentifier numericIndexIdentifier = createIndexIdentifier(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1"))); + IndexTermType numericIndexTermType = createIndexTermType(Int32Type.instance); + waitForTableIndexesQueryable(); - verifyIndexFiles(numericIndexContext, null, 1, 0); + verifyIndexFiles(numericIndexTermType, numericIndexIdentifier, 1); assertNumRows(1, "SELECT * FROM %%s WHERE v1 >= 0"); assertValidationCount(0, 0); } diff --git a/test/unit/org/apache/cassandra/index/sai/functional/SnapshotTest.java b/test/unit/org/apache/cassandra/index/sai/functional/SnapshotTest.java index cd20955d4a..fd38bc6006 100644 --- a/test/unit/org/apache/cassandra/index/sai/functional/SnapshotTest.java +++ b/test/unit/org/apache/cassandra/index/sai/functional/SnapshotTest.java @@ -22,8 +22,9 @@ import org.junit.Before; import org.junit.Test; import org.apache.cassandra.db.marshal.Int32Type; -import org.apache.cassandra.index.sai.IndexContext; import org.apache.cassandra.index.sai.SAITester; +import org.apache.cassandra.index.sai.utils.IndexIdentifier; +import org.apache.cassandra.index.sai.utils.IndexTermType; import org.apache.cassandra.inject.Injections; import static org.junit.Assert.assertEquals; @@ -51,10 +52,11 @@ public class SnapshotTest extends SAITester // Insert some initial data and create the index over it execute("INSERT INTO %s (id1, v1) VALUES ('0', 0);"); - IndexContext numericIndexContext = createIndexContext(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1")), Int32Type.instance); + IndexIdentifier indexIdentifier = createIndexIdentifier(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1"))); + IndexTermType indexTermType = createIndexTermType(Int32Type.instance); waitForTableIndexesQueryable(); flush(); - verifyIndexFiles(numericIndexContext, null, 1, 1, 0, 1, 0); + verifyIndexFiles(indexTermType, indexIdentifier, 1, 1, 1); // Note: This test will fail here if it is run on its own because the per-index validation // is run if the node is starting up but validatation isn't done once the node is started assertValidationCount(0, 0); @@ -63,7 +65,7 @@ public class SnapshotTest extends SAITester // Add some data into a second sstable execute("INSERT INTO %s (id1, v1) VALUES ('1', 0);"); flush(); - verifyIndexFiles(numericIndexContext, null, 2, 2, 0, 2, 0); + verifyIndexFiles(indexTermType, indexIdentifier, 2, 2, 2); assertValidationCount(0, 0); // Take a snapshot recording the index files last modified date @@ -79,7 +81,7 @@ public class SnapshotTest extends SAITester // Add some data into a third sstable, out of the scope of our snapshot execute("INSERT INTO %s (id1, v1) VALUES ('2', 0);"); flush(); - verifyIndexFiles(numericIndexContext, null, 3, 3, 0, 3, 0); + verifyIndexFiles(indexTermType, indexIdentifier, 3, 3, 3); assertNumRows(3, "SELECT * FROM %%s WHERE v1 >= 0"); assertValidationCount(0, 0); @@ -91,7 +93,7 @@ public class SnapshotTest extends SAITester // Restore the snapshot, only the two first sstables should be restored restoreSnapshot(snapshot); - verifyIndexFiles(numericIndexContext, null, 2, 2, 0, 2, 0); + verifyIndexFiles(indexTermType, indexIdentifier, 2, 2, 2); assertEquals(snapshotLastModified, indexFilesLastModified()); assertNumRows(2, "SELECT * FROM %%s WHERE v1 >= 0"); assertValidationCount(2, 2); // newly loaded @@ -100,8 +102,8 @@ public class SnapshotTest extends SAITester verifyIndexComponentsIncludedInSSTable(); // Rebuild the index to verify that the index files are overridden - rebuildIndexes(numericIndexContext.getIndexName()); - verifyIndexFiles(numericIndexContext, null, 2, 0); + rebuildIndexes(indexIdentifier.indexName); + verifyIndexFiles(indexTermType, indexIdentifier, 2); assertNotEquals(snapshotLastModified, indexFilesLastModified()); assertNumRows(2, "SELECT * FROM %%s WHERE v1 >= 0"); assertValidationCount(2, 2); // compaction should not validate @@ -128,9 +130,10 @@ public class SnapshotTest extends SAITester verifyIndexComponentsNotIncludedInSSTable(); // create index - IndexContext numericIndexContext = createIndexContext(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1")), Int32Type.instance); + IndexIdentifier indexIdentifier = createIndexIdentifier(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1"))); + IndexTermType indexTermType = createIndexTermType(Int32Type.instance); waitForTableIndexesQueryable(); - verifyIndexFiles(numericIndexContext, null, 2, 0); + verifyIndexFiles(indexTermType, indexIdentifier, 2); assertValidationCount(0, 0); // index components are included after initial build @@ -154,7 +157,7 @@ public class SnapshotTest extends SAITester // Restore the snapshot restoreSnapshot(snapshot); - verifyIndexFiles(numericIndexContext, null, 2, 0); + verifyIndexFiles(indexTermType, indexIdentifier, 2); assertEquals(snapshotLastModified, indexFilesLastModified()); assertNumRows(2, "SELECT * FROM %%s WHERE v1 >= 0"); assertValidationCount(2, 2); // newly loaded diff --git a/test/unit/org/apache/cassandra/index/sai/memory/TrieMemoryIndexTest.java b/test/unit/org/apache/cassandra/index/sai/memory/TrieMemoryIndexTest.java index 4979d87a6c..0ac2a2998b 100644 --- a/test/unit/org/apache/cassandra/index/sai/memory/TrieMemoryIndexTest.java +++ b/test/unit/org/apache/cassandra/index/sai/memory/TrieMemoryIndexTest.java @@ -34,6 +34,7 @@ import org.junit.Test; import org.apache.cassandra.cql3.Operator; import org.apache.cassandra.cql3.statements.schema.IndexTarget; import org.apache.cassandra.db.Clustering; +import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.db.PartitionPosition; import org.apache.cassandra.db.marshal.AbstractType; @@ -45,17 +46,14 @@ import org.apache.cassandra.dht.ExcludingBounds; import org.apache.cassandra.dht.IncludingExcludingBounds; import org.apache.cassandra.dht.Murmur3Partitioner; import org.apache.cassandra.dht.Range; -import org.apache.cassandra.index.TargetParser; -import org.apache.cassandra.index.sai.IndexContext; import org.apache.cassandra.index.sai.StorageAttachedIndex; -import org.apache.cassandra.index.sai.plan.Expression; import org.apache.cassandra.index.sai.iterators.KeyRangeIterator; +import org.apache.cassandra.index.sai.plan.Expression; import org.apache.cassandra.index.sai.utils.PrimaryKeys; import org.apache.cassandra.index.sai.utils.SAIRandomizedTester; -import org.apache.cassandra.index.sai.utils.TypeUtil; import org.apache.cassandra.schema.CachingParams; -import org.apache.cassandra.schema.ColumnMetadata; import org.apache.cassandra.schema.IndexMetadata; +import org.apache.cassandra.schema.MockSchema; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.Pair; @@ -74,7 +72,7 @@ public class TrieMemoryIndexTest extends SAIRandomizedTester private static final DecoratedKey key = Murmur3Partitioner.instance.decorateKey(ByteBufferUtil.bytes("key")); - private IndexContext indexContext; + private StorageAttachedIndex index; @Test public void heapGrowsAsDataIsAddedTest() @@ -169,7 +167,7 @@ public class TrieMemoryIndexTest extends SAIRandomizedTester private Expression generateRandomExpression() { - Expression expression = new Expression(indexContext); + Expression expression = Expression.create(index); int equality = getRandom().nextIntBetween(0, 100); int lower = getRandom().nextIntBetween(0, 75); @@ -216,7 +214,7 @@ public class TrieMemoryIndexTest extends SAIRandomizedTester assertEquals(1, pair.right.size()); final int rowId = i; - final ByteComparable expectedByteComparable = TypeUtil.isLiteral(type) + final ByteComparable expectedByteComparable = index.index.termType().isLiteral() ? ByteComparable.fixedLength(decompose.apply(rowId)) : version -> type.asComparableBytes(decompose.apply(rowId), version); final ByteComparable actualByteComparable = pair.left; @@ -241,15 +239,10 @@ public class TrieMemoryIndexTest extends SAIRandomizedTester options.put("target", REG_COL); IndexMetadata indexMetadata = IndexMetadata.fromSchemaMetadata("col_index", IndexMetadata.Kind.CUSTOM, options); - Pair target = TargetParser.parse(table, indexMetadata); - indexContext = new IndexContext(table.keyspace, - table.name, - table.partitionKeyType, - table.partitioner, - table.comparator, - target.left, - target.right, - indexMetadata); - return new TrieMemoryIndex(indexContext); + + ColumnFamilyStore cfs = MockSchema.newCFS(table); + + index = new StorageAttachedIndex(cfs, indexMetadata); + return new TrieMemoryIndex(index); } } diff --git a/test/unit/org/apache/cassandra/index/sai/memory/VectorMemoryIndexTest.java b/test/unit/org/apache/cassandra/index/sai/memory/VectorMemoryIndexTest.java index 33bbcedd76..5ed06b0a69 100644 --- a/test/unit/org/apache/cassandra/index/sai/memory/VectorMemoryIndexTest.java +++ b/test/unit/org/apache/cassandra/index/sai/memory/VectorMemoryIndexTest.java @@ -56,9 +56,9 @@ import org.apache.cassandra.dht.ExcludingBounds; import org.apache.cassandra.dht.IPartitioner; import org.apache.cassandra.dht.IncludingExcludingBounds; import org.apache.cassandra.dht.Range; -import org.apache.cassandra.index.sai.IndexContext; import org.apache.cassandra.index.sai.QueryContext; import org.apache.cassandra.index.sai.SAITester; +import org.apache.cassandra.index.sai.StorageAttachedIndex; import org.apache.cassandra.index.sai.iterators.KeyRangeIterator; import org.apache.cassandra.index.sai.plan.Expression; import org.apache.cassandra.index.sai.utils.PrimaryKey; @@ -66,7 +66,6 @@ import org.apache.cassandra.index.sai.utils.RangeUtil; import org.apache.cassandra.inject.Injections; import org.apache.cassandra.inject.InvokePointBuilder; import org.apache.cassandra.locator.TokenMetadata; -import org.apache.cassandra.schema.MockSchema; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.service.StorageService; import org.apache.cassandra.utils.FBUtilities; @@ -85,7 +84,7 @@ public class VectorMemoryIndexTest extends SAITester .build(); private ColumnFamilyStore cfs; - private IndexContext indexContext; + private StorageAttachedIndex index; private VectorMemoryIndex memtableIndex; private IPartitioner partitioner; private Map keyMap; @@ -104,14 +103,10 @@ public class VectorMemoryIndexTest extends SAITester TokenMetadata metadata = StorageService.instance.getTokenMetadata(); metadata.updateNormalTokens(BootStrapper.getRandomTokens(metadata, 10), FBUtilities.getBroadcastAddressAndPort()); - TableMetadata tableMetadata = TableMetadata.builder("ks", "tb") - .addPartitionKeyColumn("pk", Int32Type.instance) - .addRegularColumn("val", Int32Type.instance) - .build(); - cfs = MockSchema.newCFS(tableMetadata); - partitioner = cfs.getPartitioner(); dimensionCount = getRandom().nextIntBetween(2, 2048); - indexContext = SAITester.createIndexContext("index", VectorType.getInstance(FloatType.instance, dimensionCount)); + index = SAITester.createMockIndex("index", VectorType.getInstance(FloatType.instance, dimensionCount)); + cfs = index.baseCfs(); + partitioner = cfs.getPartitioner(); indexSearchCounter.reset(); keyMap = new TreeMap<>(); rowMap = new HashMap<>(); @@ -122,7 +117,7 @@ public class VectorMemoryIndexTest extends SAITester @Test public void randomQueryTest() throws Exception { - memtableIndex = new VectorMemoryIndex(indexContext); + memtableIndex = new VectorMemoryIndex(index); for (int row = 0; row < getRandom().nextIntBetween(1000, 5000); row++) { @@ -186,7 +181,7 @@ public class VectorMemoryIndexTest extends SAITester private Expression generateRandomExpression() { - Expression expression = new Expression(indexContext); + Expression expression = Expression.create(index); expression.add(Operator.ANN, randomVector()); return expression; } diff --git a/test/unit/org/apache/cassandra/index/sai/metrics/IndexGroupMetricsTest.java b/test/unit/org/apache/cassandra/index/sai/metrics/IndexGroupMetricsTest.java index 26a7ab6714..355243a7d0 100644 --- a/test/unit/org/apache/cassandra/index/sai/metrics/IndexGroupMetricsTest.java +++ b/test/unit/org/apache/cassandra/index/sai/metrics/IndexGroupMetricsTest.java @@ -20,8 +20,6 @@ package org.apache.cassandra.index.sai.metrics; import org.junit.Before; import org.junit.Test; -import org.apache.cassandra.db.marshal.UTF8Type; -import org.apache.cassandra.index.sai.IndexContext; import org.apache.cassandra.index.sai.disk.format.Version; import static org.junit.Assert.assertEquals; @@ -46,7 +44,6 @@ public class IndexGroupMetricsTest extends AbstractMetricsTest // create first index createTable(CREATE_TABLE_TEMPLATE); String v1IndexName = createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1")); - IndexContext v1IndexContext = createIndexContext(v1IndexName, UTF8Type.instance); // no open files assertEquals(0, getOpenIndexFiles()); @@ -62,7 +59,7 @@ public class IndexGroupMetricsTest extends AbstractMetricsTest // with 10 sstable int indexopenFileCountWithOnlyNumeric = getOpenIndexFiles(); assertEquals(sstables * (Version.LATEST.onDiskFormat().openFilesPerSSTableIndex(false) + - Version.LATEST.onDiskFormat().openFilesPerColumnIndex(v1IndexContext)), + Version.LATEST.onDiskFormat().openFilesPerColumnIndex()), indexopenFileCountWithOnlyNumeric); long diskUsageWithOnlyNumeric = getDiskUsage(); @@ -72,7 +69,7 @@ public class IndexGroupMetricsTest extends AbstractMetricsTest compact(); assertEquals(Version.LATEST.onDiskFormat().openFilesPerSSTableIndex(false) + - Version.LATEST.onDiskFormat().openFilesPerColumnIndex(v1IndexContext), + Version.LATEST.onDiskFormat().openFilesPerColumnIndex(), getOpenIndexFiles()); // drop last index, no open index files diff --git a/test/unit/org/apache/cassandra/index/sai/plan/ExpressionTest.java b/test/unit/org/apache/cassandra/index/sai/plan/ExpressionTest.java index f34f5325f1..3d32a405dd 100644 --- a/test/unit/org/apache/cassandra/index/sai/plan/ExpressionTest.java +++ b/test/unit/org/apache/cassandra/index/sai/plan/ExpressionTest.java @@ -23,6 +23,8 @@ import java.nio.ByteBuffer; import org.junit.Test; import org.apache.cassandra.db.marshal.UTF8Type; +import org.apache.cassandra.index.sai.SAITester; +import org.apache.cassandra.index.sai.utils.IndexTermType; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotEquals; @@ -33,10 +35,11 @@ public class ExpressionTest @Test public void testBoundHashCode() { + IndexTermType indexTermType = SAITester.createIndexTermType(UTF8Type.instance); ByteBuffer buf1 = UTF8Type.instance.decompose("blah"); - Expression.Bound b1 = new Expression.Bound(buf1, UTF8Type.instance, true); + Expression.Bound b1 = new Expression.Bound(buf1, indexTermType, true); ByteBuffer buf2 = UTF8Type.instance.decompose("blah"); - Expression.Bound b2 = new Expression.Bound(buf2, UTF8Type.instance, true); + Expression.Bound b2 = new Expression.Bound(buf2, indexTermType, true); assertEquals(b1, b2); assertEquals(b1.hashCode(), b2.hashCode()); } @@ -44,10 +47,11 @@ public class ExpressionTest @Test public void testNotMatchingBoundHashCode() { + IndexTermType indexTermType = SAITester.createIndexTermType(UTF8Type.instance); ByteBuffer buf1 = UTF8Type.instance.decompose("blah"); - Expression.Bound b1 = new Expression.Bound(buf1, UTF8Type.instance, true); + Expression.Bound b1 = new Expression.Bound(buf1, indexTermType, true); ByteBuffer buf2 = UTF8Type.instance.decompose("blah2"); - Expression.Bound b2 = new Expression.Bound(buf2, UTF8Type.instance, true); + Expression.Bound b2 = new Expression.Bound(buf2, indexTermType, true); assertNotEquals(b1, b2); assertNotEquals(b1.hashCode(), b2.hashCode()); } diff --git a/test/unit/org/apache/cassandra/index/sai/plan/OperationTest.java b/test/unit/org/apache/cassandra/index/sai/plan/OperationTest.java index e4da6c9fe3..9e6b0982ac 100644 --- a/test/unit/org/apache/cassandra/index/sai/plan/OperationTest.java +++ b/test/unit/org/apache/cassandra/index/sai/plan/OperationTest.java @@ -29,7 +29,6 @@ import java.util.concurrent.TimeUnit; import com.google.common.base.Preconditions; import com.google.common.collect.Multimap; import com.google.common.collect.Sets; -import org.junit.Assert; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; @@ -62,7 +61,6 @@ import org.apache.cassandra.db.rows.Unfiltered; import org.apache.cassandra.dht.Murmur3Partitioner; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.index.sai.QueryContext; -import org.apache.cassandra.index.sai.SAITester; import org.apache.cassandra.index.sai.StorageAttachedIndex; import org.apache.cassandra.schema.ColumnMetadata; import org.apache.cassandra.schema.IndexMetadata; @@ -73,6 +71,9 @@ import org.apache.cassandra.utils.FBUtilities; import static org.apache.cassandra.config.CassandraRelevantProperties.CASSANDRA_CONFIG; import static org.apache.cassandra.db.marshal.Int32Type.instance; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; public class OperationTest { @@ -138,17 +139,15 @@ public class OperationTest final ColumnMetadata age = getColumn(UTF8Type.instance.decompose("age")); // age > 1 AND age < 7 - Map expressions = convert(Operation.buildIndexExpressions(controller, Operation.BooleanOperator.AND, + Map expressions = convert(Operation.buildIndexExpressions(controller, Arrays.asList(new SimpleExpression(age, Operator.GT, Int32Type.instance.decompose(1)), - new SimpleExpression(age, Operator.LT, Int32Type.instance.decompose(7))))); + new SimpleExpression(age, Operator.LT, Int32Type.instance.decompose(7))))); - Assert.assertEquals(1, expressions.size()); - Assert.assertEquals(new Expression(SAITester.createIndexContext("age", Int32Type.instance)) - {{ - operator = IndexOperator.RANGE; - lower = new Bound(Int32Type.instance.decompose(1), Int32Type.instance, false); - upper = new Bound(Int32Type.instance.decompose(7), Int32Type.instance, false); - }}, expressions.get(Expression.IndexOperator.RANGE)); + assertEquals(1, expressions.size()); + + Expression rangeExpression = expressions.get(Expression.IndexOperator.RANGE); + + assertExpression(rangeExpression, Expression.IndexOperator.RANGE, Int32Type.instance.decompose(1), false, Int32Type.instance.decompose(7), false); } @Test @@ -164,15 +163,15 @@ public class OperationTest Unfiltered row = buildRow(buildCell(age, instance.decompose(6), System.currentTimeMillis())); Row staticRow = buildRow(Clustering.STATIC_CLUSTERING); - Assert.assertFalse(filterTree.isSatisfiedBy(key, row, staticRow)); + assertFalse(filterTree.isSatisfiedBy(key, row, staticRow)); row = buildRow(buildCell(age, instance.decompose(5), System.currentTimeMillis())); - Assert.assertTrue(filterTree.isSatisfiedBy(key, row, staticRow)); + assertTrue(filterTree.isSatisfiedBy(key, row, staticRow)); row = buildRow(buildCell(age, instance.decompose(6), System.currentTimeMillis())); - Assert.assertFalse(filterTree.isSatisfiedBy(key, row, staticRow)); + assertFalse(filterTree.isSatisfiedBy(key, row, staticRow)); // range with exclusions - age > 1 AND age <= 10 node = new Operation.AndNode(); @@ -186,7 +185,7 @@ public class OperationTest row = buildRow(buildCell(age, instance.decompose(i), System.currentTimeMillis())); boolean result = filterTree.isSatisfiedBy(key, row, staticRow); - Assert.assertTrue(exclusions.contains(i) != result); + assertTrue(exclusions.contains(i) != result); } // now let's test aggregated AND commands @@ -202,7 +201,7 @@ public class OperationTest row = buildRow(buildCell(age, instance.decompose(i), System.currentTimeMillis())); boolean result = filterTree.isSatisfiedBy(key, row, staticRow); - Assert.assertTrue(result); + assertTrue(result); } // multiple analyzed expressions in the Operation timestamp >= 10 AND age = 5 @@ -215,17 +214,17 @@ public class OperationTest row = buildRow(buildCell(age, instance.decompose(6), System.currentTimeMillis()), buildCell(timestamp, LongType.instance.decompose(11L), System.currentTimeMillis())); - Assert.assertFalse(filterTree.isSatisfiedBy(key, row, staticRow)); + assertFalse(filterTree.isSatisfiedBy(key, row, staticRow)); row = buildRow(buildCell(age, instance.decompose(5), System.currentTimeMillis()), buildCell(timestamp, LongType.instance.decompose(22L), System.currentTimeMillis())); - Assert.assertTrue(filterTree.isSatisfiedBy(key, row, staticRow)); + assertTrue(filterTree.isSatisfiedBy(key, row, staticRow)); row = buildRow(buildCell(age, instance.decompose(5), System.currentTimeMillis()), buildCell(timestamp, LongType.instance.decompose(9L), System.currentTimeMillis())); - Assert.assertFalse(filterTree.isSatisfiedBy(key, row, staticRow)); + assertFalse(filterTree.isSatisfiedBy(key, row, staticRow)); } @@ -237,39 +236,33 @@ public class OperationTest // first_name = 'a' AND height > 5 Map expressions; - expressions = convert(Operation.buildIndexExpressions(controller, Operation.BooleanOperator.AND, + expressions = convert(Operation.buildIndexExpressions(controller, Arrays.asList(new SimpleExpression(firstName, Operator.EQ, UTF8Type.instance.decompose("a")), new SimpleExpression(height, Operator.GT, Int32Type.instance.decompose(5))))); - Assert.assertEquals(2, expressions.size()); + assertEquals(2, expressions.size()); - expressions = convert(Operation.buildIndexExpressions(controller, Operation.BooleanOperator.AND, + expressions = convert(Operation.buildIndexExpressions(controller, Arrays.asList(new SimpleExpression(firstName, Operator.EQ, UTF8Type.instance.decompose("a")), new SimpleExpression(height, Operator.GT, Int32Type.instance.decompose(0)), new SimpleExpression(height, Operator.EQ, Int32Type.instance.decompose(5))))); - Assert.assertEquals(2, expressions.size()); + assertEquals(2, expressions.size()); - Assert.assertEquals(new Expression(SAITester.createIndexContext("height", Int32Type.instance)) - {{ - operator = IndexOperator.RANGE; - lower = new Bound(Int32Type.instance.decompose(0), Int32Type.instance, false); - upper = new Bound(Int32Type.instance.decompose(5), Int32Type.instance, true); - }}, expressions.get(Expression.IndexOperator.RANGE)); + Expression rangeExpression = expressions.get(Expression.IndexOperator.RANGE); - expressions = convert(Operation.buildIndexExpressions(controller, Operation.BooleanOperator.AND, + assertExpression(rangeExpression, Expression.IndexOperator.RANGE, Int32Type.instance.decompose(0), false, Int32Type.instance.decompose(5), true); + + expressions = convert(Operation.buildIndexExpressions(controller, Arrays.asList(new SimpleExpression(firstName, Operator.EQ, UTF8Type.instance.decompose("a")), - new SimpleExpression(height, Operator.GTE, Int32Type.instance.decompose(0)), - new SimpleExpression(height, Operator.LT, Int32Type.instance.decompose(10))))); + new SimpleExpression(height, Operator.GTE, Int32Type.instance.decompose(0)), + new SimpleExpression(height, Operator.LT, Int32Type.instance.decompose(10))))); - Assert.assertEquals(2, expressions.size()); + assertEquals(2, expressions.size()); - Assert.assertEquals(new Expression(SAITester.createIndexContext("height", Int32Type.instance)) - {{ - operator = IndexOperator.RANGE; - lower = new Bound(Int32Type.instance.decompose(0), Int32Type.instance, true); - upper = new Bound(Int32Type.instance.decompose(10), Int32Type.instance, false); - }}, expressions.get(Expression.IndexOperator.RANGE)); + rangeExpression = expressions.get(Expression.IndexOperator.RANGE); + + assertExpression(rangeExpression, Expression.IndexOperator.RANGE, Int32Type.instance.decompose(0), true, Int32Type.instance.decompose(10), false); } @Test @@ -290,46 +283,46 @@ public class OperationTest node.add(new Operation.ExpressionNode(new SimpleExpression(age, Operator.EQ, Int32Type.instance.decompose(27)))); node.add(new Operation.ExpressionNode(new SimpleExpression(height, Operator.EQ, Int32Type.instance.decompose(182)))); - Assert.assertTrue(node.buildFilter(controllerClustering).isSatisfiedBy(key, row, staticRow)); + assertTrue(node.buildFilter(controllerClustering).isSatisfiedBy(key, row, staticRow)); node = new Operation.AndNode(); node.add(new Operation.ExpressionNode(new SimpleExpression(age, Operator.EQ, Int32Type.instance.decompose(28)))); node.add(new Operation.ExpressionNode(new SimpleExpression(height, Operator.EQ, Int32Type.instance.decompose(182)))); - Assert.assertFalse(node.buildFilter(controllerClustering).isSatisfiedBy(key, row, staticRow)); + assertFalse(node.buildFilter(controllerClustering).isSatisfiedBy(key, row, staticRow)); node = new Operation.AndNode(); node.add(new Operation.ExpressionNode(new SimpleExpression(location, Operator.EQ, UTF8Type.instance.decompose("US")))); node.add(new Operation.ExpressionNode(new SimpleExpression(age, Operator.GTE, Int32Type.instance.decompose(27)))); - Assert.assertTrue(node.buildFilter(controllerClustering).isSatisfiedBy(key, row, staticRow)); + assertTrue(node.buildFilter(controllerClustering).isSatisfiedBy(key, row, staticRow)); node = new Operation.AndNode(); node.add(new Operation.ExpressionNode(new SimpleExpression(location, Operator.EQ, UTF8Type.instance.decompose("BY")))); node.add(new Operation.ExpressionNode(new SimpleExpression(age, Operator.GTE, Int32Type.instance.decompose(28)))); - Assert.assertFalse(node.buildFilter(controllerClustering).isSatisfiedBy(key, row, staticRow)); + assertFalse(node.buildFilter(controllerClustering).isSatisfiedBy(key, row, staticRow)); node = new Operation.AndNode(); node.add(new Operation.ExpressionNode(new SimpleExpression(location, Operator.EQ, UTF8Type.instance.decompose("US")))); node.add(new Operation.ExpressionNode(new SimpleExpression(age, Operator.LTE, Int32Type.instance.decompose(27)))); node.add(new Operation.ExpressionNode(new SimpleExpression(height, Operator.GTE, Int32Type.instance.decompose(182)))); - Assert.assertTrue(node.buildFilter(controllerClustering).isSatisfiedBy(key, row, staticRow)); + assertTrue(node.buildFilter(controllerClustering).isSatisfiedBy(key, row, staticRow)); node = new Operation.AndNode(); node.add(new Operation.ExpressionNode(new SimpleExpression(location, Operator.EQ, UTF8Type.instance.decompose("US")))); node.add(new Operation.ExpressionNode(new SimpleExpression(height, Operator.GTE, Int32Type.instance.decompose(182)))); node.add(new Operation.ExpressionNode(new SimpleExpression(score, Operator.EQ, DoubleType.instance.decompose(1.0d)))); - Assert.assertTrue(node.buildFilter(controllerClustering).isSatisfiedBy(key, row, staticRow)); + assertTrue(node.buildFilter(controllerClustering).isSatisfiedBy(key, row, staticRow)); node = new Operation.AndNode(); node.add(new Operation.ExpressionNode(new SimpleExpression(height, Operator.GTE, Int32Type.instance.decompose(182)))); node.add(new Operation.ExpressionNode(new SimpleExpression(score, Operator.EQ, DoubleType.instance.decompose(1.0d)))); - Assert.assertTrue(node.buildFilter(controllerClustering).isSatisfiedBy(key, row, staticRow)); + assertTrue(node.buildFilter(controllerClustering).isSatisfiedBy(key, row, staticRow)); } private Map convert(Multimap expressions) @@ -337,9 +330,9 @@ public class OperationTest Map converted = new EnumMap<>(Expression.IndexOperator.class); for (Expression expression : expressions.values()) { - Expression column = converted.get(expression.getOp()); + Expression column = converted.get(expression.getIndexOperator()); assert column == null; // sanity check - converted.put(expression.getOp(), expression); + converted.put(expression.getIndexOperator(), expression); } return converted; @@ -362,14 +355,14 @@ public class OperationTest node.add(new Operation.ExpressionNode(new SimpleExpression(sensorType, Operator.EQ, UTF8Type.instance.decompose("TEMPERATURE")))); node.add(new Operation.ExpressionNode(new SimpleExpression(value, Operator.EQ, DoubleType.instance.decompose(24.56)))); - Assert.assertTrue(node.buildFilter(controllerStatic).isSatisfiedBy(key, row, staticRow)); + assertTrue(node.buildFilter(controllerStatic).isSatisfiedBy(key, row, staticRow)); // sensor_type ='TEMPERATURE' AND value = 30 node = new Operation.AndNode(); node.add(new Operation.ExpressionNode(new SimpleExpression(sensorType, Operator.EQ, UTF8Type.instance.decompose("TEMPERATURE")))); node.add(new Operation.ExpressionNode(new SimpleExpression(value, Operator.EQ, DoubleType.instance.decompose(30.00)))); - Assert.assertFalse(node.buildFilter(controllerStatic).isSatisfiedBy(key, row, staticRow)); + assertFalse(node.buildFilter(controllerStatic).isSatisfiedBy(key, row, staticRow)); } public static TableMetadata.Builder skinnySAITableMetadata(String keyspace, String table) @@ -437,6 +430,16 @@ public class OperationTest return builder.indexes(indexes.build()); } + private void assertExpression(Expression expression, Expression.IndexOperator indexOperator, ByteBuffer lower, + boolean lowerInclusive, ByteBuffer upper, boolean upperInclusive) + { + assertEquals(indexOperator, expression.getIndexOperator()); + assertEquals(lower, expression.lower().value.raw); + assertEquals(lowerInclusive, expression.lower().inclusive); + assertEquals(upper, expression.upper().value.raw); + assertEquals(upperInclusive, expression.upper().inclusive); + } + private static void addIndex(Indexes.Builder indexes, String table, String column) { String indexName = table + '_' + column; @@ -447,11 +450,13 @@ public class OperationTest }})); } - private static DecoratedKey buildKey(Object... key) { + private static DecoratedKey buildKey(Object... key) + { return buildKey(BACKEND, key); } - private static DecoratedKey buildKey(ColumnFamilyStore cfs, Object... key) { + private static DecoratedKey buildKey(ColumnFamilyStore cfs, Object... key) + { AbstractType type = cfs.metadata().partitionKeyType; ByteBuffer decomposed; if(type instanceof CompositeType) diff --git a/test/unit/org/apache/cassandra/index/sai/utils/AbstractPrimaryKeyTester.java b/test/unit/org/apache/cassandra/index/sai/utils/AbstractPrimaryKeyTester.java index be887bc64c..8ccd63436d 100644 --- a/test/unit/org/apache/cassandra/index/sai/utils/AbstractPrimaryKeyTester.java +++ b/test/unit/org/apache/cassandra/index/sai/utils/AbstractPrimaryKeyTester.java @@ -146,7 +146,7 @@ public class AbstractPrimaryKeyTester extends SAIRandomizedTester protected DecoratedKey makeKey(TableMetadata table, Object...partitionKeys) { ByteBuffer key; - if (TypeUtil.isComposite(table.partitionKeyType)) + if (table.partitionKeyType instanceof CompositeType) key = ((CompositeType)table.partitionKeyType).decompose(partitionKeys); else key = table.partitionKeyType.decomposeUntyped(partitionKeys[0]); diff --git a/test/unit/org/apache/cassandra/index/sai/utils/TypeUtilTest.java b/test/unit/org/apache/cassandra/index/sai/utils/IndexTermTypeTest.java similarity index 55% rename from test/unit/org/apache/cassandra/index/sai/utils/TypeUtilTest.java rename to test/unit/org/apache/cassandra/index/sai/utils/IndexTermTypeTest.java index 69922ee4de..315adfdac5 100644 --- a/test/unit/org/apache/cassandra/index/sai/utils/TypeUtilTest.java +++ b/test/unit/org/apache/cassandra/index/sai/utils/IndexTermTypeTest.java @@ -15,11 +15,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.cassandra.index.sai.utils; import java.math.BigInteger; import java.nio.ByteBuffer; import java.util.Arrays; +import java.util.Collections; import java.util.function.BiConsumer; import java.util.function.BiFunction; @@ -40,16 +42,18 @@ import org.apache.cassandra.db.marshal.TupleType; import org.apache.cassandra.db.marshal.TypeParser; import org.apache.cassandra.db.marshal.UTF8Type; import org.apache.cassandra.db.marshal.UserType; +import org.apache.cassandra.index.sai.SAITester; import org.apache.cassandra.index.sai.StorageAttachedIndex; import org.apache.cassandra.schema.ColumnMetadata; import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.bytecomparable.ByteComparable; +import static org.apache.cassandra.index.sai.SAITester.getRandom; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; -public class TypeUtilTest extends SAIRandomizedTester +public class IndexTermTypeTest { @Test public void testSimpleType() @@ -58,13 +62,16 @@ public class TypeUtilTest extends SAIRandomizedTester { AbstractType type = cql3Type.getType(); AbstractType reversedType = ReversedType.getInstance(type); - - boolean isUTF8OrAscii = cql3Type == CQL3Type.Native.ASCII || cql3Type == CQL3Type.Native.TEXT || cql3Type == CQL3Type.Native.VARCHAR; - boolean isLiteral = cql3Type == CQL3Type.Native.ASCII || cql3Type == CQL3Type.Native.TEXT || cql3Type == CQL3Type.Native.VARCHAR || cql3Type == CQL3Type.Native.BOOLEAN; - assertEquals(isLiteral, TypeUtil.isLiteral(type)); - assertEquals(TypeUtil.isLiteral(type), TypeUtil.isLiteral(reversedType)); - assertEquals(isUTF8OrAscii, TypeUtil.isString(type)); - assertEquals(TypeUtil.isString(type), TypeUtil.isString(reversedType)); + IndexTermType indexTermType = SAITester.createIndexTermType(type); + IndexTermType reversedIndexTermType = SAITester.createIndexTermType(reversedType); + boolean isUTF8OrAscii = cql3Type == CQL3Type.Native.ASCII || cql3Type == CQL3Type.Native.TEXT || + cql3Type == CQL3Type.Native.VARCHAR; + boolean isLiteral = cql3Type == CQL3Type.Native.ASCII || cql3Type == CQL3Type.Native.TEXT || + cql3Type == CQL3Type.Native.VARCHAR || cql3Type == CQL3Type.Native.BOOLEAN; + assertEquals(isLiteral, indexTermType.isLiteral()); + assertEquals(indexTermType.isLiteral(), reversedIndexTermType.isLiteral()); + assertEquals(isUTF8OrAscii, indexTermType.isString()); + assertEquals(indexTermType.isString(), reversedIndexTermType.isString()); } } @@ -77,11 +84,12 @@ public class TypeUtilTest extends SAIRandomizedTester testCollectionType((valueType, multiCell) -> MapType.getInstance(keyType, valueType, multiCell), (valueType, nonFrozenMap) -> { - assertEquals(keyType, cellValueType(nonFrozenMap, IndexTarget.Type.KEYS)); - assertEquals(valueType, cellValueType(nonFrozenMap, IndexTarget.Type.VALUES)); - AbstractType entryType = cellValueType(nonFrozenMap, IndexTarget.Type.KEYS_AND_VALUES); - assertEquals(CompositeType.getInstance(keyType, valueType), entryType); - assertTrue(TypeUtil.isLiteral(entryType)); + assertEquals(keyType, indexTermType(nonFrozenMap, IndexTarget.Type.KEYS).indexType()); + assertEquals(valueType, indexTermType(nonFrozenMap, IndexTarget.Type.VALUES).indexType()); + IndexTermType entryIndexTermType = indexTermType(nonFrozenMap, IndexTarget.Type.KEYS_AND_VALUES); + assertEquals(CompositeType.getInstance(keyType, valueType), entryIndexTermType.indexType()); + assertTrue(entryIndexTermType.isComposite()); + assertTrue(entryIndexTermType.isLiteral()); }); } } @@ -104,9 +112,17 @@ public class TypeUtilTest extends SAIRandomizedTester for (CQL3Type elementType : StorageAttachedIndex.SUPPORTED_TYPES) { TupleType type = TupleType.getInstance(new TypeParser(String.format("(%s, %s)", elementType.getType(), elementType.getType()))); - assertFalse(TypeUtil.isFrozenCollection(type)); - assertTrue(TypeUtil.isFrozen(type)); - assertTrue(TypeUtil.isLiteral(type)); + IndexTermType indexTermType = indexTermType(type, IndexTarget.Type.SIMPLE); + assertFalse(indexTermType.isFrozenCollection()); + assertTrue(indexTermType.isFrozen()); + assertTrue(indexTermType.isLiteral()); + assertFalse(indexTermType.isReversed()); + + IndexTermType reversedIndexTermType = indexTermType(ReversedType.getInstance(type), IndexTarget.Type.SIMPLE); + assertFalse(reversedIndexTermType.isFrozenCollection()); + assertTrue(reversedIndexTermType.isFrozen()); + assertTrue(reversedIndexTermType.isLiteral()); + assertTrue(reversedIndexTermType.isReversed()); } } @@ -119,18 +135,32 @@ public class TypeUtilTest extends SAIRandomizedTester Arrays.asList(FieldIdentifier.forQuoted("f1"), FieldIdentifier.forQuoted("f2")), Arrays.asList(elementType.getType(), elementType.getType()), true); + IndexTermType indexTermType = indexTermType(type, IndexTarget.Type.SIMPLE); + assertFalse(indexTermType.isFrozenCollection()); + assertFalse(indexTermType.isFrozen()); + assertFalse(indexTermType.isLiteral()); + assertFalse(indexTermType.isReversed()); - assertFalse(TypeUtil.isFrozenCollection(type)); - assertFalse(TypeUtil.isFrozen(type)); - assertFalse(TypeUtil.isLiteral(type)); + IndexTermType reversedIndexTermType = indexTermType(ReversedType.getInstance(type), IndexTarget.Type.SIMPLE); + assertFalse(reversedIndexTermType.isFrozenCollection()); + assertFalse(reversedIndexTermType.isFrozen()); + assertFalse(reversedIndexTermType.isLiteral()); + assertTrue(reversedIndexTermType.isReversed()); type = new UserType("ks", ByteBufferUtil.bytes("myType"), Arrays.asList(FieldIdentifier.forQuoted("f1"), FieldIdentifier.forQuoted("f2")), Arrays.asList(elementType.getType(), elementType.getType()), false); - assertFalse(TypeUtil.isFrozenCollection(type)); - assertTrue(TypeUtil.isFrozen(type)); - assertTrue(TypeUtil.isLiteral(type)); + indexTermType = indexTermType(type, IndexTarget.Type.SIMPLE); + assertFalse(indexTermType.isFrozenCollection()); + assertTrue(indexTermType.isFrozen()); + assertTrue(indexTermType.isLiteral()); + + reversedIndexTermType = indexTermType(ReversedType.getInstance(type), IndexTarget.Type.SIMPLE); + assertFalse(reversedIndexTermType.isFrozenCollection()); + assertTrue(reversedIndexTermType.isFrozen()); + assertTrue(reversedIndexTermType.isLiteral()); + assertTrue(reversedIndexTermType.isReversed()); } } @@ -142,25 +172,25 @@ public class TypeUtilTest extends SAIRandomizedTester AbstractType frozenCollection = init.apply(elementType.getType(), false); AbstractType reversedFrozenCollection = ReversedType.getInstance(frozenCollection); - AbstractType type = TypeUtil.cellValueType(column(frozenCollection), IndexTarget.Type.FULL); - assertTrue(TypeUtil.isFrozenCollection(type)); - assertTrue(TypeUtil.isLiteral(type)); - assertFalse(type.isReversed()); + IndexTermType indexTermType = indexTermType(frozenCollection, IndexTarget.Type.FULL); + assertTrue(indexTermType.isFrozenCollection()); + assertTrue(indexTermType.isLiteral()); + assertFalse(indexTermType.isReversed()); - type = TypeUtil.cellValueType(column(reversedFrozenCollection), IndexTarget.Type.FULL); - assertTrue(TypeUtil.isFrozenCollection(type)); - assertTrue(TypeUtil.isLiteral(type)); - assertTrue(type.isReversed()); + IndexTermType reversedIndexTermType = indexTermType(reversedFrozenCollection, IndexTarget.Type.FULL); + assertTrue(reversedIndexTermType.isFrozenCollection()); + assertTrue(reversedIndexTermType.isLiteral()); + assertTrue(reversedIndexTermType.isReversed()); AbstractType nonFrozenCollection = init.apply(elementType.getType(), true); - assertEquals(elementType.getType(), cellValueType(nonFrozenCollection, IndexTarget.Type.VALUES)); + assertEquals(elementType.getType(), indexTermType(nonFrozenCollection, IndexTarget.Type.VALUES).indexType()); nonFrozenCollectionTester.accept(elementType.getType(), nonFrozenCollection); } } - private static AbstractType cellValueType(AbstractType type, IndexTarget.Type indexType) + private static IndexTermType indexTermType(AbstractType type, IndexTarget.Type indexType) { - return TypeUtil.cellValueType(column(type), indexType); + return IndexTermType.create(column(type), Collections.emptyList(), indexType); } private static ColumnMetadata column(AbstractType type) @@ -171,22 +201,24 @@ public class TypeUtilTest extends SAIRandomizedTester @Test public void shouldCompareByteBuffers() { + IndexTermType indexTermType = indexTermType(Int32Type.instance, IndexTarget.Type.SIMPLE); + final ByteBuffer a = Int32Type.instance.decompose(1); final ByteBuffer b = Int32Type.instance.decompose(2); - assertEquals(a, TypeUtil.min(a, b, Int32Type.instance)); - assertEquals(a, TypeUtil.min(b, a, Int32Type.instance)); - assertEquals(a, TypeUtil.min(a, a, Int32Type.instance)); - assertEquals(b, TypeUtil.min(b, b, Int32Type.instance)); - assertEquals(b, TypeUtil.min(null, b, Int32Type.instance)); - assertEquals(a, TypeUtil.min(a, null, Int32Type.instance)); + assertEquals(a, indexTermType.min(a, b)); + assertEquals(a, indexTermType.min(b, a)); + assertEquals(a, indexTermType.min(a, a)); + assertEquals(b, indexTermType.min(b, b)); + assertEquals(b, indexTermType.min(null, b)); + assertEquals(a, indexTermType.min(a, null)); - assertEquals(b, TypeUtil.max(b, a, Int32Type.instance)); - assertEquals(b, TypeUtil.max(a, b, Int32Type.instance)); - assertEquals(a, TypeUtil.max(a, a, Int32Type.instance)); - assertEquals(b, TypeUtil.max(b, b, Int32Type.instance)); - assertEquals(b, TypeUtil.max(null, b, Int32Type.instance)); - assertEquals(a, TypeUtil.max(a, null, Int32Type.instance)); + assertEquals(b, indexTermType.max(b, a)); + assertEquals(b, indexTermType.max(a, b)); + assertEquals(a, indexTermType.max(a, a)); + assertEquals(b, indexTermType.max(b, b)); + assertEquals(b, indexTermType.max(null, b)); + assertEquals(a, indexTermType.max(a, null)); } @Test @@ -204,15 +236,18 @@ public class TypeUtilTest extends SAIRandomizedTester Arrays.sort(data, BigInteger::compareTo); + IndexTermType indexTermType = indexTermType(IntegerType.instance, IndexTarget.Type.SIMPLE); + assertTrue(indexTermType.supportsRounding()); + for (int i = 1; i < data.length; i++) { BigInteger i0 = data[i - 1]; BigInteger i1 = data[i]; assertTrue("#" + i, i0.compareTo(i1) <= 0); - ByteBuffer b0 = TypeUtil.asIndexBytes(ByteBuffer.wrap(i0.toByteArray()), IntegerType.instance); - ByteBuffer b1 = TypeUtil.asIndexBytes(ByteBuffer.wrap(i1.toByteArray()), IntegerType.instance); - assertTrue("#" + i, TypeUtil.compare(b0, b1, IntegerType.instance) <= 0); + ByteBuffer b0 = indexTermType.asIndexBytes(ByteBuffer.wrap(i0.toByteArray())); + ByteBuffer b1 = indexTermType.asIndexBytes(ByteBuffer.wrap(i1.toByteArray())); + assertTrue("#" + i, indexTermType.compare(b0, b1) <= 0); } } @@ -220,6 +255,7 @@ public class TypeUtilTest extends SAIRandomizedTester public void testMapEntryEncoding() { CompositeType type = CompositeType.getInstance(UTF8Type.instance, Int32Type.instance); + IndexTermType indexTermType = indexTermType(type, IndexTarget.Type.SIMPLE); // simulate: index memtable insertion String[] data = new String[10000]; @@ -230,7 +266,7 @@ public class TypeUtilTest extends SAIRandomizedTester String v1 = new String(temp); int v2 = getRandom().nextInt(); - data[i] = TypeUtil.getString(type.decompose(v1, v2), type); + data[i] = indexTermType.asString(type.decompose(v1, v2)); } Arrays.sort(data, String::compareTo); @@ -238,9 +274,9 @@ public class TypeUtilTest extends SAIRandomizedTester for (int i = 1; i < data.length; i++) { // simulate: index memtable flush - ByteBuffer b0 = TypeUtil.fromString(data[i - 1], type); - ByteBuffer b1 = TypeUtil.fromString(data[i], type); - assertTrue("#" + i, TypeUtil.compare(b0, b1, type) <= 0); + ByteBuffer b0 = indexTermType.fromString(data[i - 1]); + ByteBuffer b1 = indexTermType.fromString(data[i]); + assertTrue("#" + i, indexTermType.compare(b0, b1) <= 0); // simulate: saving into on-disk trie ByteComparable t0 = ByteComparable.fixedLength(b0); diff --git a/test/unit/org/apache/cassandra/index/sai/virtual/IndexesSystemViewTest.java b/test/unit/org/apache/cassandra/index/sai/virtual/IndexesSystemViewTest.java index 19d4cbfdf1..d52d1d5395 100644 --- a/test/unit/org/apache/cassandra/index/sai/virtual/IndexesSystemViewTest.java +++ b/test/unit/org/apache/cassandra/index/sai/virtual/IndexesSystemViewTest.java @@ -25,7 +25,6 @@ import org.apache.cassandra.cql3.CQLTester; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.virtual.VirtualKeyspace; import org.apache.cassandra.db.virtual.VirtualKeyspaceRegistry; -import org.apache.cassandra.index.sai.IndexContext; import org.apache.cassandra.index.sai.SAITester; import org.apache.cassandra.index.sai.StorageAttachedIndex; import org.apache.cassandra.inject.Injections; @@ -121,7 +120,6 @@ public class IndexesSystemViewTest extends SAITester { ColumnFamilyStore cfs = getCurrentColumnFamilyStore(); StorageAttachedIndex sai = (StorageAttachedIndex) cfs.indexManager.getIndexByName(indexName); - IndexContext context = sai.getIndexContext(); return row(indexName, currentTable(), @@ -129,6 +127,6 @@ public class IndexesSystemViewTest extends SAITester isQueryable, isBuilding, isString, - context.getAnalyzerFactory().toString()); + sai.hasAnalyzer() ? sai.analyzer().toString() : "NoOpAnalyzer"); } } diff --git a/test/unit/org/apache/cassandra/index/sai/virtual/SSTablesSystemViewTest.java b/test/unit/org/apache/cassandra/index/sai/virtual/SSTablesSystemViewTest.java index 2150d356a2..a308d806d0 100644 --- a/test/unit/org/apache/cassandra/index/sai/virtual/SSTablesSystemViewTest.java +++ b/test/unit/org/apache/cassandra/index/sai/virtual/SSTablesSystemViewTest.java @@ -177,7 +177,7 @@ public class SSTablesSystemViewTest extends SAITester ColumnFamilyStore cfs = getCurrentColumnFamilyStore(); StorageAttachedIndex sai = (StorageAttachedIndex) cfs.indexManager.getIndexByName(indexName); - for (SSTableIndex sstableIndex : sai.getIndexContext().getView()) + for (SSTableIndex sstableIndex : sai.view()) { SSTableReader sstable = sstableIndex.getSSTable(); diff --git a/test/unit/org/apache/cassandra/index/sai/virtual/SegmentsSystemViewTest.java b/test/unit/org/apache/cassandra/index/sai/virtual/SegmentsSystemViewTest.java index 910f9f11fa..d6fcb132ca 100644 --- a/test/unit/org/apache/cassandra/index/sai/virtual/SegmentsSystemViewTest.java +++ b/test/unit/org/apache/cassandra/index/sai/virtual/SegmentsSystemViewTest.java @@ -34,25 +34,23 @@ import org.apache.cassandra.db.marshal.UTF8Type; import org.apache.cassandra.db.virtual.VirtualKeyspace; import org.apache.cassandra.db.virtual.VirtualKeyspaceRegistry; import org.apache.cassandra.index.Index; -import org.apache.cassandra.index.sai.IndexContext; import org.apache.cassandra.index.sai.SAITester; -import org.apache.cassandra.index.sai.disk.SSTableIndex; import org.apache.cassandra.index.sai.StorageAttachedIndex; +import org.apache.cassandra.index.sai.disk.SSTableIndex; import org.apache.cassandra.index.sai.disk.format.IndexComponent; import org.apache.cassandra.index.sai.disk.format.IndexDescriptor; import org.apache.cassandra.index.sai.disk.v1.segment.SegmentBuilder; import org.apache.cassandra.index.sai.disk.v1.segment.SegmentMetadata; -import org.apache.cassandra.index.sai.utils.TypeUtil; +import org.apache.cassandra.index.sai.utils.IndexIdentifier; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.schema.SchemaConstants; import org.apache.cassandra.service.StorageService; +import static org.apache.cassandra.config.CassandraRelevantProperties.TEST_ENCRYPTION; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertTrue; -import static org.apache.cassandra.config.CassandraRelevantProperties.TEST_ENCRYPTION; - /** * Tests the virtual table exposing SSTable index segment metadata. */ @@ -193,17 +191,17 @@ public class SegmentsSystemViewTest extends SAITester { StorageAttachedIndex index = (StorageAttachedIndex) idx; - for (SSTableIndex sstableIndex : index.getIndexContext().getView().getIndexes()) + for (SSTableIndex sstableIndex : index.view().getIndexes()) { SSTableReader sstable = sstableIndex.getSSTable(); IndexDescriptor indexDescriptor = IndexDescriptor.create(sstable); - indexDescriptor.hasComponent(IndexComponent.COLUMN_COMPLETION_MARKER, index.getIndexContext()); + indexDescriptor.hasComponent(IndexComponent.COLUMN_COMPLETION_MARKER, index.identifier()); - if (TypeUtil.isLiteral(sstableIndex.getIndexContext().getValidator())) + if (sstableIndex.getIndexTermType().isLiteral()) { - addComponentSizeToMap(lengths, IndexComponent.TERMS_DATA, index.getIndexContext(), indexDescriptor); - addComponentSizeToMap(lengths, IndexComponent.POSTING_LISTS, index.getIndexContext(), indexDescriptor); + addComponentSizeToMap(lengths, IndexComponent.TERMS_DATA, index.identifier(), indexDescriptor); + addComponentSizeToMap(lengths, IndexComponent.POSTING_LISTS, index.identifier(), indexDescriptor); } } } @@ -211,10 +209,10 @@ public class SegmentsSystemViewTest extends SAITester return lengths; } - private static void addComponentSizeToMap(HashMap map, IndexComponent key, IndexContext indexContext, IndexDescriptor indexDescriptor) + private static void addComponentSizeToMap(HashMap map, IndexComponent key, IndexIdentifier indexIdentifier, IndexDescriptor indexDescriptor) { map.compute(key.name(), (typeName, acc) -> { - final long size = indexDescriptor.sizeOnDiskOfPerIndexComponent(key, indexContext); + final long size = indexDescriptor.sizeOnDiskOfPerIndexComponent(key, indexIdentifier); return acc == null ? size : size + acc; }); }