In-memory index implementation with query path

This includes the following elements of the Storage Attached Index:
- Memtable-attached indexes backed by an in-memory trie structure for byte-comparable values
- Query path for the in-memory index
- Index status propagation
- Randomized testing for Memtable-attached indexes

patch my Mike Adamson; reviewed by Caleb Rackliffe and Andres de la Peña for CASSANDRA-18058

Co-authored-by: Mike Adamson <madamson@datastax.com>
Co-authored-by: Caleb Rackliffe <calebrackliffe@gmail.com>

.build/cassandra-build-deps-template.xml

@@ -135,5 +135,9 @@
       <groupId>de.jflex</groupId>
       <artifactId>jflex</artifactId>
     </dependency>
+    <dependency>
+      <groupId>com.carrotsearch.randomizedtesting</groupId>
+      <artifactId>randomizedtesting-runner</artifactId>
+    </dependency>
   </dependencies>
 </project>

.build/cassandra-deps-template.xml

@@ -372,5 +372,9 @@
       <groupId>org.agrona</groupId>
       <artifactId>agrona</artifactId>
     </dependency>
+    <dependency>
+      <groupId>org.apache.lucene</groupId>
+      <artifactId>lucene-core</artifactId>
+    </dependency>
   </dependencies>
 </project>

.build/parent-pom-template.xml

@@ -1050,6 +1050,23 @@
         <artifactId>agrona</artifactId>
         <version>1.17.1</version>
       </dependency>
+      <dependency>
+        <groupId>org.apache.lucene</groupId>
+        <artifactId>lucene-core</artifactId>
+        <version>7.5.0</version>
+      </dependency>
+      <dependency>
+        <groupId>com.carrotsearch.randomizedtesting</groupId>
+        <artifactId>randomizedtesting-runner</artifactId>
+        <version>2.1.2</version>
+        <scope>test</scope>
+        <exclusions>
+          <exclusion>
+            <groupId>junit</groupId>
+            <artifactId>junit</artifactId>
+          </exclusion>
+        </exclusions>
+      </dependency>
       <dependency>
         <groupId>ch.obermuhlner</groupId>
         <artifactId>big-math</artifactId>

src/java/org/apache/cassandra/config/CassandraRelevantProperties.java

@@ -415,10 +415,16 @@ public enum CassandraRelevantProperties
      */
     RESET_BOOTSTRAP_PROGRESS("cassandra.reset_bootstrap_progress"),
     RING_DELAY("cassandra.ring_delay_ms"),
+
+    // SAI specific properties
+    /** Controls the maximum number of index query intersections that will take part in a query */
+    SAI_INTERSECTION_CLAUSE_LIMIT("cassandra.sai.intersection.clause.limit", "2"),
+
     /** Defines how often schema definitions are pulled from the other nodes */
     SCHEMA_PULL_INTERVAL_MS("cassandra.schema_pull_interval_ms", "60000"),
     SCHEMA_UPDATE_HANDLER_FACTORY_CLASS("cassandra.schema.update_handler_factory.class"),
     SEARCH_CONCURRENCY_FACTOR("cassandra.search_concurrency_factor", "1"),
+
     /**
      * The maximum number of seeds returned by a seed provider before emmitting a warning.
      * A large seed list may impact effectiveness of the third gossip round.

src/java/org/apache/cassandra/cql3/restrictions/StatementRestrictions.java

@@ -678,7 +678,7 @@ public final class StatementRestrictions
     public RowFilter getRowFilter(IndexRegistry indexRegistry, QueryOptions options)
     {
         if (filterRestrictions.isEmpty())
-            return RowFilter.NONE;
+            return RowFilter.none();
 
         RowFilter filter = RowFilter.create();
         for (Restrictions restrictions : filterRestrictions.getRestrictions())

src/java/org/apache/cassandra/cql3/statements/CQL3CasRequest.java

@@ -194,7 +194,7 @@ public class CQL3CasRequest implements CASRequest
             return SinglePartitionReadCommand.create(metadata,
                                                    nowInSec,
                                                    columnFilter,
-                                                   RowFilter.NONE,
+                                                   RowFilter.none(),
                                                    DataLimits.cqlLimits(1),
                                                    key,
                                                    new ClusteringIndexSliceFilter(Slices.ALL, false));

src/java/org/apache/cassandra/cql3/statements/ModificationStatement.java

@@ -441,7 +441,7 @@ public abstract class ModificationStatement implements CQLStatement.SingleKeyspa
             commands.add(SinglePartitionReadCommand.create(metadata(),
                                                            nowInSeconds,
                                                            ColumnFilter.selection(this.requiresRead),
-                                                           RowFilter.NONE,
+                                                           RowFilter.none(),
                                                            limits,
                                                            metadata().partitioner.decorateKey(key),
                                                            filter));

src/java/org/apache/cassandra/cql3/statements/schema/CreateIndexStatement.java

@@ -48,6 +48,9 @@ import static com.google.common.collect.Iterables.tryFind;
 
 public final class CreateIndexStatement extends AlterSchemaStatement
 {
+    public static final String INVALID_CUSTOM_INDEX_TARGET = "Column '%s' is longer than the permissible name length of %d characters or" +
+                                                             " contains non-alphanumeric-underscore characters";
+
     private final String indexName;
     private final String tableName;
     private final List<IndexTarget.Raw> rawIndexTargets;
@@ -138,12 +141,12 @@ public final class CreateIndexStatement extends AlterSchemaStatement
                     throw ire("Duplicate column '%s' in index target list", target.column);
         }
 
-        indexTargets.forEach(t -> validateIndexTarget(table, t));
+        IndexMetadata.Kind kind = attrs.isCustom ? IndexMetadata.Kind.CUSTOM : IndexMetadata.Kind.COMPOSITES;
+
+        indexTargets.forEach(t -> validateIndexTarget(table, kind, t));
 
         String name = null == indexName ? generateIndexName(keyspace, indexTargets) : indexName;
 
-        IndexMetadata.Kind kind = attrs.isCustom ? IndexMetadata.Kind.CUSTOM : IndexMetadata.Kind.COMPOSITES;
-
         Map<String, String> options = attrs.isCustom ? attrs.getOptions() : Collections.emptyMap();
 
         IndexMetadata index = IndexMetadata.fromIndexTargets(indexTargets, name, kind, options);
@@ -173,13 +176,16 @@ public final class CreateIndexStatement extends AlterSchemaStatement
         return ImmutableSet.of();
     }
 
-    private void validateIndexTarget(TableMetadata table, IndexTarget target)
+    private void validateIndexTarget(TableMetadata table, IndexMetadata.Kind kind, IndexTarget target)
     {
         ColumnMetadata column = table.getColumn(target.column);
 
         if (null == column)
             throw ire("Column '%s' doesn't exist", target.column);
 
+        if ((kind == IndexMetadata.Kind.CUSTOM) && !SchemaConstants.isValidName(target.column.toString()))
+            throw ire(INVALID_CUSTOM_INDEX_TARGET, target.column, SchemaConstants.NAME_LENGTH);
+
         if (column.type.referencesDuration())
         {
             if (column.type.isCollection())

src/java/org/apache/cassandra/db/PartitionRangeReadCommand.java

@@ -158,7 +158,7 @@ public class PartitionRangeReadCommand extends ReadCommand implements PartitionR
                       metadata,
                       nowInSec,
                       ColumnFilter.all(metadata),
-                      RowFilter.NONE,
+                      RowFilter.none(),
                       DataLimits.NONE,
                       DataRange.allData(metadata.partitioner),
                       null,

src/java/org/apache/cassandra/db/ReadQuery.java

@@ -107,7 +107,7 @@ public interface ReadQuery
             @Override
             public RowFilter rowFilter()
             {
-                return RowFilter.NONE;
+                return RowFilter.none();
             }
 
             @Override

src/java/org/apache/cassandra/db/SinglePartitionReadCommand.java

@@ -237,7 +237,7 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar
                                                     ColumnFilter columnFilter,
                                                     ClusteringIndexFilter filter)
     {
-        return create(metadata, nowInSec, columnFilter, RowFilter.NONE, DataLimits.NONE, key, filter);
+        return create(metadata, nowInSec, columnFilter, RowFilter.none(), DataLimits.NONE, key, filter);
     }
 
     /**
@@ -298,7 +298,7 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar
     public static SinglePartitionReadCommand create(TableMetadata metadata, long nowInSec, DecoratedKey key, Slices slices)
     {
         ClusteringIndexSliceFilter filter = new ClusteringIndexSliceFilter(slices, false);
-        return create(metadata, nowInSec, ColumnFilter.all(metadata), RowFilter.NONE, DataLimits.NONE, key, filter);
+        return create(metadata, nowInSec, ColumnFilter.all(metadata), RowFilter.none(), DataLimits.NONE, key, filter);
     }
 
     /**
@@ -331,7 +331,7 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar
     public static SinglePartitionReadCommand create(TableMetadata metadata, long nowInSec, DecoratedKey key, NavigableSet<Clustering<?>> names)
     {
         ClusteringIndexNamesFilter filter = new ClusteringIndexNamesFilter(names, false);
-        return create(metadata, nowInSec, ColumnFilter.all(metadata), RowFilter.NONE, DataLimits.NONE, key, filter);
+        return create(metadata, nowInSec, ColumnFilter.all(metadata), RowFilter.none(), DataLimits.NONE, key, filter);
     }
 
     /**

src/java/org/apache/cassandra/db/SinglePartitionReadQuery.java

@@ -74,7 +74,7 @@ public interface SinglePartitionReadQuery extends ReadQuery
                                                   ColumnFilter columnFilter,
                                                   ClusteringIndexFilter filter)
     {
-        return create(metadata, nowInSec, columnFilter, RowFilter.NONE, DataLimits.NONE, key, filter);
+        return create(metadata, nowInSec, columnFilter, RowFilter.none(), DataLimits.NONE, key, filter);
     }
 
     /**

src/java/org/apache/cassandra/db/filter/RowFilter.java

@@ -23,6 +23,8 @@ import java.util.*;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Predicate;
+import java.util.stream.Collectors;
 
 import com.google.common.base.Objects;
 import org.slf4j.Logger;
@@ -63,7 +65,6 @@ public abstract class RowFilter implements Iterable<RowFilter.Expression>
     private static final Logger logger = LoggerFactory.getLogger(RowFilter.class);
 
     public static final Serializer serializer = new Serializer();
-    public static final RowFilter NONE = new CQLFilter(Collections.emptyList());
 
     protected final List<Expression> expressions;
 
@@ -82,6 +83,11 @@ public abstract class RowFilter implements Iterable<RowFilter.Expression>
         return new CQLFilter(new ArrayList<>(capacity));
     }
 
+    public static RowFilter none()
+    {
+        return CQLFilter.NONE;
+    }
+
     public SimpleExpression add(ColumnMetadata def, Operator op, ByteBuffer value)
     {
         SimpleExpression expression = new SimpleExpression(def, op, value);
@@ -229,7 +235,7 @@ public abstract class RowFilter implements Iterable<RowFilter.Expression>
     {
         assert expressions.contains(expression);
         if (expressions.size() == 1)
-            return RowFilter.NONE;
+            return RowFilter.none();
 
         List<Expression> newExpressions = new ArrayList<>(expressions.size() - 1);
         for (Expression e : expressions)
@@ -261,6 +267,11 @@ public abstract class RowFilter implements Iterable<RowFilter.Expression>
         return withNewExpressions(Collections.emptyList());
     }
 
+    public RowFilter restrict(Predicate<Expression> filter)
+    {
+        return withNewExpressions(expressions.stream().filter(filter).collect(Collectors.toList()));
+    }
+
     protected abstract RowFilter withNewExpressions(List<Expression> expressions);
 
     public boolean isEmpty()
@@ -303,6 +314,8 @@ public abstract class RowFilter implements Iterable<RowFilter.Expression>
 
     private static class CQLFilter extends RowFilter
     {
+        static CQLFilter NONE = new CQLFilter(Collections.emptyList());
+
         private CQLFilter(List<Expression> expressions)
         {
             super(expressions);

src/java/org/apache/cassandra/db/view/TableViews.java

@@ -462,7 +462,7 @@ public class TableViews extends AbstractCollection<View>
         // TODO: we could still make sense to special case for when there is a single view and a small number of updates (and
         // no deletions). Indeed, in that case we could check whether any of the update modify any of the restricted regular
         // column, and if that's not the case we could use view filter. We keep it simple for now though.
-        RowFilter rowFilter = RowFilter.NONE;
+        RowFilter rowFilter = RowFilter.none();
         return SinglePartitionReadCommand.create(metadata, nowInSec, queriedColumns, rowFilter, DataLimits.NONE, key, clusteringFilter);
     }
 

src/java/org/apache/cassandra/db/virtual/SystemViewsKeyspace.java

@@ -19,6 +19,8 @@ package org.apache.cassandra.db.virtual;
 
 import com.google.common.collect.ImmutableList;
 
+import org.apache.cassandra.index.sai.virtual.StorageAttachedIndexTables;
+
 import static org.apache.cassandra.schema.SchemaConstants.VIRTUAL_VIEWS;
 
 public final class SystemViewsKeyspace extends VirtualKeyspace
@@ -51,6 +53,7 @@ public final class SystemViewsKeyspace extends VirtualKeyspace
                     .add(new LogMessagesTable(VIRTUAL_VIEWS))
                     .add(new SnapshotsTable(VIRTUAL_VIEWS))
                     .addAll(LocalRepairTables.getAll(VIRTUAL_VIEWS))
+                    .addAll(StorageAttachedIndexTables.getAll(VIRTUAL_VIEWS))
                     .build());
     }
 }

src/java/org/apache/cassandra/dht/Murmur3Partitioner.java

@@ -203,6 +203,12 @@ public class Murmur3Partitioner implements IPartitioner
             return token;
         }
 
+        @Override
+        public long getLongValue()
+        {
+            return token;
+        }
+
         @Override
         public double size(Token next)
         {

src/java/org/apache/cassandra/dht/Token.java

@@ -124,6 +124,21 @@ public abstract class Token implements RingPosition<Token>, Serializable
     abstract public long getHeapSize();
     abstract public Object getTokenValue();
 
+    /**
+     * This method exists so that callers can access the primitive {@code long} value for this {@link Token}, if
+     * one exits. It is especially useful when the auto-boxing induced by a call to {@link #getTokenValue()} would
+     * be unacceptable for reasons of performance.
+     *
+     * @return the primitive {@code long} value of this token, if one exists
+     *
+     * @throws UnsupportedOperationException if this {@link Token} is not backed by a primitive {@code long} value
+     */
+    public long getLongValue()
+    {
+        throw new UnsupportedOperationException();
+    }
+
+
     /**
      * Produce a weakly prefix-free byte-comparable representation of the token, i.e. such a sequence of bytes that any
      * pair x, y of valid tokens of this type and any bytes b1, b2 between 0x10 and 0xEF,

src/java/org/apache/cassandra/exceptions/RequestFailureReason.java

@@ -35,7 +35,8 @@ public enum RequestFailureReason
     TIMEOUT                  (2),
     INCOMPATIBLE_SCHEMA      (3),
     READ_SIZE                (4),
-    NODE_DOWN                (5);
+    NODE_DOWN                (5),
+    INDEX_NOT_AVAILABLE      (6);
 
     public static final Serializer serializer = new Serializer();
 

src/java/org/apache/cassandra/gms/ApplicationState.java

@@ -57,6 +57,7 @@ public enum ApplicationState
      **/
     SSTABLE_VERSIONS,
     DISK_USAGE,
+    INDEX_STATUS,
     // DO NOT EDIT OR REMOVE PADDING STATES BELOW - only add new states above.  See CASSANDRA-16484
     X1,
     X2,

src/java/org/apache/cassandra/gms/VersionedValue.java

@@ -269,6 +269,11 @@ public class VersionedValue implements Comparable<VersionedValue>
             return new VersionedValue(VersionedValue.SHUTDOWN + VersionedValue.DELIMITER + value);
         }
 
+        public VersionedValue indexStatus(String status)
+        {
+            return new VersionedValue(status);
+        }
+
         public VersionedValue datacenter(String dcId)
         {
             return new VersionedValue(dcId);

src/java/org/apache/cassandra/index/Index.java

@@ -512,19 +512,19 @@ public interface Index
          * Notification of the start of a partition update.
          * This event always occurs before any other during the update.
          */
-        public void begin();
+        default void begin() {}
 
         /**
          * Notification of a top level partition delete.
          */
-        public void partitionDelete(DeletionTime deletionTime);
+        default void partitionDelete(DeletionTime deletionTime) {}
 
         /**
          * Notification of a RangeTombstone.
          * An update of a single partition may contain multiple RangeTombstones,
          * and a notification will be passed for each of them.
          */
-        public void rangeTombstone(RangeTombstone tombstone);
+        default void rangeTombstone(RangeTombstone tombstone) {}
 
         /**
          * Notification that a new row was inserted into the Memtable holding the partition.
@@ -534,7 +534,7 @@ public interface Index
          *
          * @param row the Row being inserted into the base table's Memtable.
          */
-        public void insertRow(Row row);
+        default void insertRow(Row row) {}
 
         /**
          * Notification of a modification to a row in the base table's Memtable.
@@ -555,7 +555,7 @@ public interface Index
          * @param newRowData data that was not present in the existing row and is being inserted
          *                   into the base table's Memtable
          */
-        public void updateRow(Row oldRowData, Row newRowData);
+        default void updateRow(Row oldRowData, Row newRowData) {}
 
         /**
          * Notification that a row was removed from the partition.
@@ -573,13 +573,13 @@ public interface Index
          *
          * @param row data being removed from the base table
          */
-        public void removeRow(Row row);
+        default void removeRow(Row row) {}
 
         /**
          * Notification of the end of the partition update.
          * This event always occurs after all others for the particular update.
          */
-        public void finish();
+        default void finish() {}
     }
 
     /*

src/java/org/apache/cassandra/index/IndexStatusManager.java

@@ -0,0 +1,242 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.cassandra.concurrent.ExecutorPlus;
+import org.apache.cassandra.db.ConsistencyLevel;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.exceptions.ReadFailureException;
+import org.apache.cassandra.exceptions.RequestFailureReason;
+import org.apache.cassandra.gms.ApplicationState;
+import org.apache.cassandra.gms.Gossiper;
+import org.apache.cassandra.gms.VersionedValue;
+import org.apache.cassandra.locator.Endpoints;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.serializers.MarshalException;
+import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.JsonUtils;
+
+import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory;
+
+/**
+ * Handles the status of an index across the ring, updating the status per index and endpoint
+ * in a per-endpoint map.
+ *
+ * Peer status changes are recieved via the {@link StorageService} {@link org.apache.cassandra.gms.IEndpointStateChangeSubscriber}.
+ *
+ * Local status changes are propagated to the {@link Gossiper} using an async executor.
+ */
+public class IndexStatusManager
+{
+    private static final Logger logger = LoggerFactory.getLogger(IndexStatusManager.class);
+
+    public static final IndexStatusManager instance = new IndexStatusManager();
+
+    // executes index status propagation task asynchronously to avoid potential deadlock on SIM
+    private final ExecutorPlus statusPropagationExecutor = executorFactory().withJmxInternal()
+                                                                            .sequential("StatusPropagationExecutor");
+    // used to produce a status string from the current endpoint states in JSON format for Gossip propagation
+    private final ObjectMapper objectMapper = new ObjectMapper();
+
+    /**
+     * A map of per-endpoint index statuses: the key of inner map is the identifier "keyspace.index"
+     */
+    public final Map<InetAddressAndPort, Map<String, Index.Status>> peerIndexStatus = new HashMap<>();
+
+    private IndexStatusManager()
+    {}
+
+    /**
+     * Remove endpoints whose indexes are not queryable for the specified {@link Index.QueryPlan}.
+     *
+     * @param liveEndpoints current live endpoints where non-queryable endpoints will be removed
+     * @param keyspace to be queried
+     * @param indexQueryPlan index query plan used in the read command
+     * @param level consistency level of read command
+     */
+    public <E extends Endpoints<E>> E filterForQuery(E liveEndpoints, Keyspace keyspace, Index.QueryPlan indexQueryPlan, ConsistencyLevel level)
+    {
+        E queryableEndpoints = liveEndpoints.filter(replica -> {
+
+            for (Index index : indexQueryPlan.getIndexes())
+            {
+                Index.Status status = getIndexStatus(replica.endpoint(), keyspace.getName(), index.getIndexMetadata().name);
+                if (!index.isQueryable(status))
+                    return false;
+            }
+
+            return true;
+        });
+
+        int initial = liveEndpoints.size();
+        int filtered = queryableEndpoints.size();
+
+        // Throw ReadFailureException if read request cannot satisfy Consistency Level due to non-queryable indexes.
+        // It is to provide a better UX, compared to throwing UnavailableException when the nodes are actually alive.
+        if (initial != filtered)
+        {
+            int required = level.blockFor(keyspace.getReplicationStrategy());
+            if (required <= initial && required > filtered)
+            {
+                Map<InetAddressAndPort, RequestFailureReason> failureReasons = new HashMap<>();
+                liveEndpoints.without(queryableEndpoints.endpoints())
+                             .forEach(replica -> failureReasons.put(replica.endpoint(), RequestFailureReason.INDEX_NOT_AVAILABLE));
+
+                throw new ReadFailureException(level, filtered, required, false, failureReasons);
+            }
+        }
+
+        return queryableEndpoints;
+    }
+
+    /**
+     * Recieve a new index status map from a peer. This will include the status for all the indexes on the peer.
+     *
+     * @param endpoint the {@link InetAddressAndPort} the index status map is coming from
+     * @param versionedValue the {@link VersionedValue} containing the index status map
+     */
+    public synchronized void receivePeerIndexStatus(InetAddressAndPort endpoint, VersionedValue versionedValue)
+    {
+        try
+        {
+            if (versionedValue == null)
+                return;
+            if (endpoint.equals(FBUtilities.getBroadcastAddressAndPort()))
+                return;
+
+            @SuppressWarnings("unchecked")
+            Map<String, String> peerStatus = (Map<String, String>) JsonUtils.decodeJson(versionedValue.value);
+            Map<String, Index.Status> indexStatus = new HashMap<>();
+
+            for (Map.Entry<String, String> e : peerStatus.entrySet())
+            {
+                String keyspaceIndex = e.getKey();
+                Index.Status status = Index.Status.valueOf(e.getValue());
+                indexStatus.put(keyspaceIndex, status);
+            }
+
+            Map<String, Index.Status> oldStatus = peerIndexStatus.put(endpoint, indexStatus);
+            Map<String, Index.Status> updated = updatedIndexStatuses(oldStatus, indexStatus);
+            Set<String> removed = removedIndexStatuses(oldStatus, indexStatus);
+            if (!updated.isEmpty() || !removed.isEmpty())
+                logger.debug("Received index status for peer {}:\n    Updated: {}\n    Removed: {}",
+                             endpoint, updated, removed);
+        }
+        catch (MarshalException | IllegalArgumentException e)
+        {
+            logger.warn("Unable to parse index status: {}", e.getMessage());
+        }
+    }
+
+    /**
+     * Propagate a new index status to the ring. The new index status is added to the current index status map
+     * and the whole map is sent to the ring as a {@link VersionedValue}.
+     *
+     * @param keyspace the keyspace name for the index
+     * @param index the index name
+     * @param status the new {@link Index.Status}
+     */
+    public synchronized void propagateLocalIndexStatus(String keyspace, String index, Index.Status status)
+    {
+        try
+        {
+            Map<String, Index.Status> states = peerIndexStatus.computeIfAbsent(FBUtilities.getBroadcastAddressAndPort(),
+                                                                               k -> new HashMap<>());
+            String keyspaceIndex = identifier(keyspace, index);
+
+            if (status == Index.Status.DROPPED)
+                states.remove(keyspaceIndex);
+            else
+                states.put(keyspaceIndex, status);
+
+            // Don't try and propagate if the gossiper isn't enabled. This is primarily for tests where the
+            // Gossiper has not been started. If we attempt to propagate when not started an exception is
+            // logged and this causes a number of dtests to fail.
+            if (Gossiper.instance.isEnabled())
+            {
+                String newStatus = objectMapper.writeValueAsString(states);
+                statusPropagationExecutor.submit(() -> {
+                    // schedule gossiper update asynchronously to avoid potential deadlock when another thread is holding
+                    // gossiper taskLock.
+                    VersionedValue value = StorageService.instance.valueFactory.indexStatus(newStatus);
+                    Gossiper.instance.addLocalApplicationState(ApplicationState.INDEX_STATUS, value);
+                });
+            }
+        }
+        catch (Throwable e)
+        {
+            logger.warn("Unable to propagate index status: {}", e.getMessage());
+        }
+    }
+
+    @VisibleForTesting
+    public synchronized Index.Status getIndexStatus(InetAddressAndPort peer, String keyspace, String index)
+    {
+        return peerIndexStatus.getOrDefault(peer, Collections.emptyMap())
+                              .getOrDefault(identifier(keyspace, index), Index.Status.UNKNOWN);
+    }
+
+    /**
+     * Returns the names of indexes that are present in oldStatus but absent in newStatus.
+     */
+    private @Nonnull Set<String> removedIndexStatuses(@Nullable Map<String, Index.Status> oldStatus,
+                                                      @Nonnull Map<String, Index.Status> newStatus)
+    {
+        if (oldStatus == null)
+            return Collections.emptySet();
+        Set<String> result = new HashSet<>(oldStatus.keySet());
+        result.removeAll(newStatus.keySet());
+        return result;
+    }
+
+    /**
+     * Returns a new map containing only the entries from newStatus that differ from corresponding entries in oldStatus.
+     */
+    private @Nonnull Map<String, Index.Status> updatedIndexStatuses(@Nullable Map<String, Index.Status> oldStatus,
+                                                                    @Nonnull Map<String, Index.Status> newStatus)
+    {
+        Map<String, Index.Status> delta = new HashMap<>();
+        for (Map.Entry<String, Index.Status> e : newStatus.entrySet())
+        {
+            if (oldStatus == null || e.getValue() != oldStatus.get(e.getKey()))
+                delta.put(e.getKey(), e.getValue());
+        }
+        return delta;
+    }
+
+    private String identifier(String keyspace, String index)
+    {
+        return keyspace + '.' + index;
+    }
+}

src/java/org/apache/cassandra/index/SecondaryIndexManager.java

@@ -48,7 +48,6 @@ import org.apache.cassandra.concurrent.ExecutorPlus;
 import org.apache.cassandra.concurrent.FutureTask;
 import org.apache.cassandra.concurrent.ImmediateExecutor;
 import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.cql3.statements.schema.IndexTarget;
 import org.apache.cassandra.db.*;
 import org.apache.cassandra.db.compaction.CompactionManager;
 import org.apache.cassandra.db.filter.ClusteringIndexSliceFilter;
@@ -214,7 +213,6 @@ public class SecondaryIndexManager implements IndexRegistry, INotificationConsum
                : blockingExecutor.submit(reloadTask);
     }
 
-    @SuppressWarnings("unchecked")
     private synchronized Future<Void> createIndex(IndexMetadata indexDef, boolean isNewCF)
     {
         final Index index = createInstance(indexDef);
@@ -654,7 +652,8 @@ public class SecondaryIndexManager implements IndexRegistry, INotificationConsum
      * @param isNewCF {@code true} if this method is invoked when initializing a new table/columnfamily (i.e. loading a CF at startup),
      * {@code false} for all other cases (i.e. newly added index)
      */
-    private synchronized void markIndexesBuilding(Set<Index> indexes, boolean isFullRebuild, boolean isNewCF)
+    @VisibleForTesting
+    public synchronized void markIndexesBuilding(Set<Index> indexes, boolean isFullRebuild, boolean isNewCF)
     {
         String keyspaceName = baseCfs.getKeyspaceName();
 
@@ -766,6 +765,8 @@ public class SecondaryIndexManager implements IndexRegistry, INotificationConsum
         writableIndexes.remove(indexName);
         needsFullRebuild.remove(indexName);
         inProgressBuilds.remove(indexName);
+        // remove existing indexing status
+        IndexStatusManager.instance.propagateLocalIndexStatus(keyspace.getName(), indexName, Index.Status.DROPPED);
     }
 
     public Index getIndexByName(String indexName)
@@ -779,8 +780,8 @@ public class SecondaryIndexManager implements IndexRegistry, INotificationConsum
         if (indexDef.isCustom())
         {
             assert indexDef.options != null;
-            // Find any aliases to the fully qualified index class name:
-            String className = IndexMetadata.expandAliases(indexDef.options.get(IndexTarget.CUSTOM_INDEX_OPTION_NAME));
+            // Get the fully qualified index class name from the index metadata
+            String className = indexDef.getIndexClassName();
             assert !Strings.isNullOrEmpty(className);
 
             try
@@ -943,7 +944,7 @@ public class SecondaryIndexManager implements IndexRegistry, INotificationConsum
             SinglePartitionReadCommand cmd = SinglePartitionReadCommand.create(baseCfs.metadata(),
                                                                                FBUtilities.nowInSeconds(),
                                                                                ColumnFilter.selection(columns),
-                                                                               RowFilter.NONE,
+                                                                               RowFilter.none(),
                                                                                DataLimits.NONE,
                                                                                key,
                                                                                new ClusteringIndexSliceFilter(Slices.ALL, false));
@@ -1180,12 +1181,20 @@ public class SecondaryIndexManager implements IndexRegistry, INotificationConsum
         return indexes.stream().map(i -> i.getIndexMetadata().name).collect(Collectors.joining(","));
     }
 
-
     public Optional<Index> getBestIndexFor(RowFilter.Expression expression)
     {
         return indexes.values().stream().filter((i) -> i.supportsExpression(expression.column(), expression.operator())).findFirst();
     }
 
+    public <T extends Index> Set<T> getBestIndexFor(RowFilter.Expression expression, Class<T> indexType)
+    {
+        return indexes.values()
+                      .stream()
+                      .filter(i -> indexType.isInstance(i) && i.supportsExpression(expression.column(), expression.operator()))
+                      .map(indexType::cast)
+                      .collect(Collectors.toSet());
+    }
+
     /**
      * Called at write time to ensure that values present in the update
      * are valid according to the rules of all registered indexes which
@@ -1711,9 +1720,13 @@ public class SecondaryIndexManager implements IndexRegistry, INotificationConsum
 
     public void makeIndexNonQueryable(Index index, Index.Status status)
     {
+        if (status == Index.Status.BUILD_SUCCEEDED)
+            throw new IllegalStateException("Index cannot be marked non-queryable with status " + status);
+
         String name = index.getIndexMetadata().name;
         if (indexes.get(name) == index)
         {
+            IndexStatusManager.instance.propagateLocalIndexStatus(keyspace.getName(), name, status);
             if (!index.isQueryable(status))
                 queryableIndexes.remove(name);
         }
@@ -1721,9 +1734,13 @@ public class SecondaryIndexManager implements IndexRegistry, INotificationConsum
 
     public void makeIndexQueryable(Index index, Index.Status status)
     {
+        if (status != Index.Status.BUILD_SUCCEEDED)
+            throw new IllegalStateException("Index cannot be marked queryable with status " + status);
+
         String name = index.getIndexMetadata().name;
         if (indexes.get(name) == index)
         {
+            IndexStatusManager.instance.propagateLocalIndexStatus(keyspace.getName(), name, status);
             if (index.isQueryable(status))
             {
                 if (queryableIndexes.add(name))

src/java/org/apache/cassandra/index/internal/composites/CompositesSearcher.java

@@ -118,7 +118,7 @@ public class CompositesSearcher extends CassandraIndexSearcher
                         dataCmd = SinglePartitionReadCommand.create(index.baseCfs.metadata(),
                                                                     command.nowInSec(),
                                                                     command.columnFilter(),
-                                                                    RowFilter.NONE,
+                                                                    RowFilter.none(),
                                                                     DataLimits.NONE,
                                                                     partitionKey,
                                                                     command.clusteringIndexFilter(partitionKey));

src/java/org/apache/cassandra/index/sai/IndexContext.java

@@ -0,0 +1,453 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai;
+
+import java.nio.ByteBuffer;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.Objects;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.TimeUnit;
+import javax.annotation.Nullable;
+
+import com.google.common.base.MoreObjects;
+import com.google.common.collect.ImmutableSet;
+
+import org.apache.cassandra.cql3.Operator;
+import org.apache.cassandra.cql3.statements.schema.IndexTarget;
+import org.apache.cassandra.db.ClusteringComparator;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.PartitionPosition;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.AsciiType;
+import org.apache.cassandra.db.marshal.BooleanType;
+import org.apache.cassandra.db.marshal.CompositeType;
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.db.marshal.UUIDType;
+import org.apache.cassandra.db.memtable.Memtable;
+import org.apache.cassandra.db.rows.Cell;
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.dht.AbstractBounds;
+import org.apache.cassandra.index.sai.analyzer.AbstractAnalyzer;
+import org.apache.cassandra.index.sai.memory.MemtableIndex;
+import org.apache.cassandra.index.sai.metrics.IndexMetrics;
+import org.apache.cassandra.index.sai.plan.Expression;
+import org.apache.cassandra.index.sai.utils.PrimaryKey;
+import org.apache.cassandra.index.sai.utils.PrimaryKeyFactory;
+import org.apache.cassandra.index.sai.utils.KeyRangeIterator;
+import org.apache.cassandra.index.sai.utils.KeyRangeUnionIterator;
+import org.apache.cassandra.index.sai.utils.TypeUtil;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.IndexMetadata;
+import org.apache.cassandra.utils.Clock;
+import org.apache.cassandra.utils.FBUtilities;
+
+/**
+ * Manage metadata for each column index.
+ */
+public class IndexContext
+{
+    private static final Set<AbstractType<?>> EQ_ONLY_TYPES = ImmutableSet.of(UTF8Type.instance,
+                                                                              AsciiType.instance,
+                                                                              BooleanType.instance,
+                                                                              UUIDType.instance);
+
+    private final AbstractType<?> partitionKeyType;
+    private final ClusteringComparator clusteringComparator;
+
+    private final String keyspace;
+    private final String table;
+    private final ColumnMetadata columnMetadata;
+    private final IndexTarget.Type indexType;
+    private final AbstractType<?> validator;
+
+    // Config can be null if the column context is "fake" (i.e. created for a filtering expression).
+    @Nullable
+    private final IndexMetadata config;
+
+    private final ConcurrentMap<Memtable, MemtableIndex> liveMemtableIndexMap;
+
+    private final IndexMetrics indexMetrics;
+
+    private final AbstractAnalyzer.AnalyzerFactory indexAnalyzerFactory;
+    private final AbstractAnalyzer.AnalyzerFactory queryAnalyzerFactory;
+    private final PrimaryKeyFactory primaryKeyFactory;
+
+    public IndexContext(String keyspace,
+                        String table,
+                        AbstractType<?> partitionKeyType,
+                        ClusteringComparator clusteringComparator,
+                        ColumnMetadata columnMetadata,
+                        IndexTarget.Type indexType,
+                        @Nullable IndexMetadata config)
+    {
+        this.keyspace = Objects.requireNonNull(keyspace);
+        this.table = Objects.requireNonNull(table);
+        this.partitionKeyType = Objects.requireNonNull(partitionKeyType);
+        this.clusteringComparator = Objects.requireNonNull(clusteringComparator);
+        this.columnMetadata = Objects.requireNonNull(columnMetadata);
+        this.indexType = Objects.requireNonNull(indexType);
+        this.validator = TypeUtil.cellValueType(columnMetadata, indexType);
+        this.primaryKeyFactory = PrimaryKey.factory(clusteringComparator);
+
+        this.config = config;
+        this.indexMetrics = config == null ? null : new IndexMetrics(this);
+        this.liveMemtableIndexMap = config == null ? null : new ConcurrentHashMap<>();
+
+        // We currently only support the NoOpAnalyzer
+        this.indexAnalyzerFactory = AbstractAnalyzer.fromOptions(getValidator(), Collections.emptyMap());
+        this.queryAnalyzerFactory = AbstractAnalyzer.fromOptions(getValidator(), Collections.emptyMap());
+    }
+
+    public AbstractType<?> keyValidator()
+    {
+        return partitionKeyType;
+    }
+
+    public PrimaryKeyFactory keyFactory()
+    {
+        return primaryKeyFactory;
+    }
+
+    public ClusteringComparator comparator()
+    {
+        return clusteringComparator;
+    }
+
+    public String getKeyspace()
+    {
+        return keyspace;
+    }
+
+    public String getTable()
+    {
+        return table;
+    }
+
+    public IndexMetadata getIndexMetadata()
+    {
+        return config;
+    }
+
+    public long index(DecoratedKey key, Row row, Memtable mt)
+    {
+        assert config != null : "Attempt to index on a non-indexing context";
+
+        MemtableIndex current = liveMemtableIndexMap.get(mt);
+
+        // We expect the relevant IndexMemtable to be present most of the time, so only make the
+        // call to computeIfAbsent() if it's not. (see https://bugs.openjdk.java.net/browse/JDK-8161372)
+        MemtableIndex target = (current != null)
+                               ? current
+                               : liveMemtableIndexMap.computeIfAbsent(mt, memtable -> new MemtableIndex(this));
+
+        long start = Clock.Global.nanoTime();
+
+        long bytes = 0;
+
+        if (isNonFrozenCollection())
+        {
+            Iterator<ByteBuffer> bufferIterator = getValuesOf(row, FBUtilities.nowInSeconds());
+            if (bufferIterator != null)
+            {
+                while (bufferIterator.hasNext())
+                {
+                    ByteBuffer value = bufferIterator.next();
+                    bytes += target.index(key, row.clustering(), value);
+                }
+            }
+        }
+        else
+        {
+            ByteBuffer value = getValueOf(key, row, FBUtilities.nowInSeconds());
+            target.index(key, row.clustering(), value);
+        }
+        indexMetrics.memtableIndexWriteLatency.update(Clock.Global.nanoTime() - start, TimeUnit.NANOSECONDS);
+        return bytes;
+    }
+
+    public void renewMemtable(Memtable renewed)
+    {
+        assert liveMemtableIndexMap != null : "Attempt to renew memtable on non-indexing context";
+
+        for (Memtable memtable : liveMemtableIndexMap.keySet())
+        {
+            // remove every index but the one that corresponds to the post-truncate Memtable
+            if (renewed != memtable)
+            {
+                liveMemtableIndexMap.remove(memtable);
+            }
+        }
+    }
+
+    public void discardMemtable(Memtable discarded)
+    {
+        assert liveMemtableIndexMap != null : "Attempt to discard memtable from non-indexing context";
+
+        liveMemtableIndexMap.remove(discarded);
+    }
+
+    public KeyRangeIterator searchMemtableIndexes(Expression e, AbstractBounds<PartitionPosition> keyRange)
+    {
+        assert liveMemtableIndexMap != null : "Attempt to perform search on non-indexing context";
+
+        Collection<MemtableIndex> memtableIndexes = liveMemtableIndexMap.values();
+
+        if (memtableIndexes.isEmpty())
+        {
+            return KeyRangeIterator.empty();
+        }
+
+        KeyRangeIterator.Builder builder = KeyRangeUnionIterator.builder(memtableIndexes.size());
+
+        for (MemtableIndex memtableIndex : memtableIndexes)
+        {
+            builder.add(memtableIndex.search(e, keyRange));
+        }
+
+        return builder.build();
+    }
+
+    public long liveMemtableWriteCount()
+    {
+        assert liveMemtableIndexMap != null : "Attempt to get metrics from non-indexing context";
+
+        return liveMemtableIndexMap.values().stream().mapToLong(MemtableIndex::writeCount).sum();
+    }
+
+    public long estimatedMemIndexMemoryUsed()
+    {
+        assert liveMemtableIndexMap != null : "Attempt to get metrics from non-indexing context";
+
+        return liveMemtableIndexMap.values().stream().mapToLong(MemtableIndex::estimatedMemoryUsed).sum();
+    }
+
+    public ColumnMetadata getDefinition()
+    {
+        return columnMetadata;
+    }
+
+    public AbstractType<?> getValidator()
+    {
+        return validator;
+    }
+
+    public boolean isNonFrozenCollection()
+    {
+        return TypeUtil.isNonFrozenCollection(columnMetadata.type);
+    }
+
+    public boolean isFrozen()
+    {
+        return TypeUtil.isFrozen(columnMetadata.type);
+    }
+
+    public String getColumnName()
+    {
+        return columnMetadata.name.toString();
+    }
+
+    @Nullable
+    public String getIndexName()
+    {
+        return config == null ? null : config.name;
+    }
+
+    /**
+     * Returns an {@code AnalyzerFactory} for use by the write path to transform incoming literal
+     * during indexing. The analyzers can be tokenising or non-tokenising. Tokenising analyzers
+     * will split the incoming terms into multiple terms in the index while non-tokenising analyzers
+     * will not split the incoming term but will transform the term (e.g. case-insensitive)
+     */
+    public AbstractAnalyzer.AnalyzerFactory getIndexAnalyzerFactory()
+    {
+        return indexAnalyzerFactory;
+    }
+
+    /**
+     * Return an {@code AnalyzerFactory} for use by the query path to transform query terms before
+     * searching for them in the index. This can be the same as the indexAnalyzerFactory.
+     */
+    public AbstractAnalyzer.AnalyzerFactory getQueryAnalyzerFactory()
+    {
+        return queryAnalyzerFactory;
+    }
+
+    public boolean isIndexed()
+    {
+        return config != null;
+    }
+
+    /**
+     * Called when index is dropped. Clear all live in-memory indexes and close
+     * analyzer factories.
+     */
+    public void invalidate()
+    {
+        if (liveMemtableIndexMap != null)
+            liveMemtableIndexMap.clear();
+        if (indexMetrics != null)
+            indexMetrics.release();
+        indexAnalyzerFactory.close();
+        if (queryAnalyzerFactory != indexAnalyzerFactory)
+        {
+            queryAnalyzerFactory.close();
+        }
+    }
+
+    public boolean supports(Operator op)
+    {
+        if (op == Operator.LIKE ||
+            op == Operator.LIKE_CONTAINS ||
+            op == Operator.LIKE_PREFIX ||
+            op == Operator.LIKE_MATCHES ||
+            op == Operator.LIKE_SUFFIX) return false;
+
+        Expression.IndexOperator operator = Expression.IndexOperator.valueOf(op);
+
+        if (isNonFrozenCollection())
+        {
+            if (indexType == IndexTarget.Type.KEYS) return operator == Expression.IndexOperator.CONTAINS_KEY;
+            if (indexType == IndexTarget.Type.VALUES) return operator == Expression.IndexOperator.CONTAINS_VALUE;
+            return indexType == IndexTarget.Type.KEYS_AND_VALUES && operator == Expression.IndexOperator.EQ;
+        }
+
+        if (indexType == IndexTarget.Type.FULL)
+            return operator == Expression.IndexOperator.EQ;
+
+        AbstractType<?> validator = getValidator();
+
+        if (operator != Expression.IndexOperator.EQ && EQ_ONLY_TYPES.contains(validator)) return false;
+
+        // RANGE only applicable to non-literal indexes
+        return (operator != null) && !(TypeUtil.isLiteral(validator) && operator == Expression.IndexOperator.RANGE);
+    }
+
+    public ByteBuffer getValueOf(DecoratedKey key, Row row, long nowInSecs)
+    {
+        if (row == null)
+            return null;
+
+        switch (columnMetadata.kind)
+        {
+            case PARTITION_KEY:
+                return partitionKeyType instanceof CompositeType
+                       ? CompositeType.extractComponent(key.getKey(), columnMetadata.position())
+                       : key.getKey();
+            case CLUSTERING:
+                // skip indexing of static clustering when regular column is indexed
+                return row.isStatic() ? null : row.clustering().bufferAt(columnMetadata.position());
+
+            // treat static cell retrieval the same was as regular
+            // only if row kind is STATIC otherwise return null
+            case STATIC:
+                if (!row.isStatic())
+                    return null;
+            case REGULAR:
+                Cell<?> cell = row.getCell(columnMetadata);
+                return cell == null || !cell.isLive(nowInSecs) ? null : cell.buffer();
+
+            default:
+                return null;
+        }
+    }
+
+    public Iterator<ByteBuffer> getValuesOf(Row row, long nowInSecs)
+    {
+        if (row == null)
+            return null;
+
+        switch (columnMetadata.kind)
+        {
+            // treat static cell retrieval the same was as regular
+            // only if row kind is STATIC otherwise return null
+            case STATIC:
+                if (!row.isStatic())
+                    return null;
+            case REGULAR:
+                return TypeUtil.collectionIterator(validator,
+                                                   row.getComplexColumnData(columnMetadata),
+                                                   columnMetadata,
+                                                   indexType,
+                                                   nowInSecs);
+
+            default:
+                return null;
+        }
+    }
+
+    @Override
+    public String toString()
+    {
+        return MoreObjects.toStringHelper(this)
+                          .add("columnName", getColumnName())
+                          .add("indexName", getIndexName())
+                          .toString();
+    }
+
+    public boolean isLiteral()
+    {
+        return TypeUtil.isLiteral(getValidator());
+    }
+
+    @Override
+    public boolean equals(Object obj)
+    {
+        if (obj == this)
+            return true;
+
+        if (!(obj instanceof IndexContext))
+            return false;
+
+        IndexContext other = (IndexContext) obj;
+
+        return Objects.equals(columnMetadata, other.columnMetadata) &&
+               (indexType == other.indexType) &&
+               Objects.equals(config, other.config) &&
+               Objects.equals(partitionKeyType, other.partitionKeyType) &&
+               Objects.equals(clusteringComparator, other.clusteringComparator);
+    }
+
+    @Override
+    public int hashCode()
+    {
+        return Objects.hash(columnMetadata, indexType, config, partitionKeyType, clusteringComparator);
+    }
+
+    /**
+     * A helper method for constructing consistent log messages for specific column indexes.
+     *
+     * Example: For the index "idx" in keyspace "ks" on table "tb", calling this method with the raw message
+     * "Flushing new index segment..." will produce...
+     *
+     * "[ks.tb.idx] Flushing new index segment..."
+     *
+     * @param message The raw content of a logging message, without information identifying it with an index.
+     *
+     * @return A log message with the proper keyspace, table and index name prepended to it.
+     */
+    public String logMessage(String message)
+    {
+        // Index names are unique only within a keyspace.
+        return String.format("[%s.%s.%s] %s", keyspace, table, config == null ? "?" : config.name, message);
+    }
+}

src/java/org/apache/cassandra/index/sai/QueryContext.java

@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai;
+
+import java.util.concurrent.TimeUnit;
+import javax.annotation.concurrent.NotThreadSafe;
+
+import org.apache.cassandra.db.ReadCommand;
+import org.apache.cassandra.exceptions.QueryCancelledException;
+import org.apache.cassandra.utils.Clock;
+
+/**
+ * Tracks state relevant to the execution of a single query, including metrics and timeout monitoring.
+ *
+ * Fields here are non-volatile, as they are accessed from a single thread.
+ */
+@NotThreadSafe
+public class QueryContext
+{
+    private static final boolean DISABLE_TIMEOUT = Boolean.getBoolean("cassandra.sai.test.disable.timeout");
+
+    private final ReadCommand readCommand;
+    private final long queryStartTimeNanos;
+
+    public final long executionQuotaNano;
+
+    public long partitionsRead = 0;
+    public long rowsFiltered = 0;
+
+    public long queryTimeouts = 0;
+
+    public QueryContext(ReadCommand readCommand, long executionQuotaMs)
+    {
+        this.readCommand = readCommand;
+        executionQuotaNano = TimeUnit.MILLISECONDS.toNanos(executionQuotaMs);
+        queryStartTimeNanos = Clock.Global.nanoTime();
+    }
+
+    public long totalQueryTimeNs()
+    {
+        return Clock.Global.nanoTime() - queryStartTimeNanos;
+    }
+
+    public void checkpoint()
+    {
+        if (totalQueryTimeNs() >= executionQuotaNano && !DISABLE_TIMEOUT)
+        {
+            queryTimeouts++;
+            throw new QueryCancelledException(readCommand);
+        }
+    }
+}

src/java/org/apache/cassandra/index/sai/README.md

@@ -0,0 +1,115 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements.  See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership.  The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License.  You may obtain a copy of the License at
+ 
+     http://www.apache.org/licenses/LICENSE-2.0
+ 
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+# Storage-Attached Indexing
+
+## Overview
+Storage-attached indexing is a column based secondary indexing apparatus for Cassandra.
+
+The project was inspired by SASI (SSTable-Attached Secondary Indexes) and retains some of its high-level
+architectural character (and even some actual code), but makes significant improvements in a number of areas:
+
+- The on-disk/SSTable index formats for both string and numeric data have been completely replaced. Strings are indexed
+  on disk using our proprietary on-disk byte-ordered trie data structure, while numeric types are indexed using Lucene's
+  balanced kd-tree.
+- While indexes continue to be managed at the column level from the user's perspective, the storage design at the column
+  index level is row-based, with related offset and token information stored only once at the SSTable level. This
+  drastically reduces our on-disk footprint when several columns are indexed on the same table.
+- Tracing, metrics, virtual table-based metadata and snapshot-based backup/restore are supported out of the box.
+
+Many similarities with standard secondary indexes remain:
+
+- The full set of C* consistency levels is supported for both reads and writes.
+- Index updates are synchronous with mutations and do not require any kind of read-before-write.
+- Queries are implemented on the back of C* range reads.
+- Paging is supported.
+- Only token ordering of results is supported.
+- Index builds are visible to operators as compactions and are executed on compaction threads.
+- All DML and DDL statements are CQL-based.
+- Single-node management operations are available via nodetool. (ex. stop & rebuild_index)
+
+## Quick Start
+
+The following short tutorial will get you up-and-running with storage-attached indexing.
+
+### Build and Start Cassandra
+
+Follow the instructions to build and start Cassandra in README.asc in root folder of the Cassandra repository
+
+### Create a Simple Data Model
+
+1.) Run the following DDL statements to create a table and two indexes:
+
+`CREATE KEYSPACE test WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '1'};`
+
+`USE test;`
+
+`CREATE TABLE person (id int, name text, age int, PRIMARY KEY (id));`
+
+`CREATE CUSTOM INDEX ON person (name) USING 'StorageAttachedIndex' WITH OPTIONS = {'case_sensitive': false};`
+
+`CREATE CUSTOM INDEX ON person (age) USING 'StorageAttachedIndex';`
+
+2.) Add some data.
+
+`INSERT INTO person (id, name, age) VALUES (1, 'John', 21);`
+
+`INSERT INTO person (id, name, age) VALUES (2, 'john', 50);`
+
+`INSERT INTO person (id, name, age) VALUES (3, 'Boris', 43);`
+
+`INSERT INTO person (id, name, age) VALUES (4, 'Caleb', 34);`
+
+### Make Some Queries
+
+1.) Query for everyone named "John", ignoring case.
+
+`SELECT * FROM person WHERE name = 'John';`
+
+```
+ id | age | name
+----+-----+------
+  1 |  21 | John
+  2 |  50 | john
+```
+
+2.) Query for everyone between the ages of 18 and 25.
+
+`SELECT * FROM person WHERE age >= 18 AND age <= 35;`
+
+```
+ id | age | name
+----+-----+-------
+  1 |  21 |  John
+  4 |  34 | Caleb
+```
+
+## Contributors
+
+- Marc Selwan
+- Caleb Rackliffe
+- Zhao Yang
+- Jason Rutherglen
+- Maciej Zasada
+- Andres de la Peña
+- Mike Adamson
+- Zahir Patni
+- Tomek Lasica
+- Berenguer Blasi
+- Rocco Varela
+- Piotr Kołaczkowski

src/java/org/apache/cassandra/index/sai/StorageAttachedIndex.java

@@ -0,0 +1,449 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CompletableFuture; // checkstyle: permit this import
+import java.util.concurrent.Future;
+
+import com.google.common.collect.ImmutableSet;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.cql3.CQL3Type;
+import org.apache.cassandra.cql3.Operator;
+import org.apache.cassandra.cql3.statements.schema.IndexTarget;
+import org.apache.cassandra.db.CassandraWriteContext;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.ReadCommand;
+import org.apache.cassandra.db.RegularAndStaticColumns;
+import org.apache.cassandra.db.WriteContext;
+import org.apache.cassandra.db.compaction.CompactionInfo;
+import org.apache.cassandra.db.filter.RowFilter;
+import org.apache.cassandra.db.lifecycle.LifecycleNewTracker;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.memtable.Memtable;
+import org.apache.cassandra.db.partitions.PartitionUpdate;
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.dht.ByteOrderedPartitioner;
+import org.apache.cassandra.dht.IPartitioner;
+import org.apache.cassandra.dht.LocalPartitioner;
+import org.apache.cassandra.dht.OrderPreservingPartitioner;
+import org.apache.cassandra.dht.RandomPartitioner;
+import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.index.Index;
+import org.apache.cassandra.index.IndexRegistry;
+import org.apache.cassandra.index.SecondaryIndexBuilder;
+import org.apache.cassandra.index.TargetParser;
+import org.apache.cassandra.index.sai.analyzer.AbstractAnalyzer;
+import org.apache.cassandra.index.sai.utils.TypeUtil;
+import org.apache.cassandra.index.transactions.IndexTransaction;
+import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.SSTableFlushObserver;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.IndexMetadata;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.utils.Pair;
+import org.apache.cassandra.utils.concurrent.OpOrder;
+
+public class StorageAttachedIndex implements Index
+{
+    private static final Logger logger = LoggerFactory.getLogger(StorageAttachedIndex.class);
+
+    private static class StorageAttachedIndexBuildingSupport implements IndexBuildingSupport
+    {
+        @Override
+        public SecondaryIndexBuilder getIndexBuildTask(ColumnFamilyStore cfs,
+                                                       Set<Index> indexes,
+                                                       Collection<SSTableReader> sstablesToRebuild,
+                                                       boolean isFullRebuild)
+        {
+            // Return an empty builder for in-memory implementation
+            return new SecondaryIndexBuilder()
+            {
+                @Override
+                public void build()
+                {
+                }
+
+                @Override
+                public CompactionInfo getCompactionInfo()
+                {
+                    return null;
+                }
+            };
+        }
+    }
+
+    // Used to build indexes on newly added SSTables:
+    private static final StorageAttachedIndexBuildingSupport INDEX_BUILDER_SUPPORT = new StorageAttachedIndexBuildingSupport();
+
+    private static final Set<String> VALID_OPTIONS = ImmutableSet.of(IndexTarget.TARGET_OPTION_NAME,
+                                                                     IndexTarget.CUSTOM_INDEX_OPTION_NAME);
+
+    public static final Set<CQL3Type> SUPPORTED_TYPES = ImmutableSet.of(CQL3Type.Native.ASCII, CQL3Type.Native.BIGINT, CQL3Type.Native.DATE,
+                                                                        CQL3Type.Native.DOUBLE, CQL3Type.Native.FLOAT, CQL3Type.Native.INT,
+                                                                        CQL3Type.Native.SMALLINT, CQL3Type.Native.TEXT, CQL3Type.Native.TIME,
+                                                                        CQL3Type.Native.TIMESTAMP, CQL3Type.Native.TIMEUUID, CQL3Type.Native.TINYINT,
+                                                                        CQL3Type.Native.UUID, CQL3Type.Native.VARCHAR, CQL3Type.Native.INET,
+                                                                        CQL3Type.Native.VARINT, CQL3Type.Native.DECIMAL, CQL3Type.Native.BOOLEAN);
+
+    private static final Set<Class<? extends IPartitioner>> ILLEGAL_PARTITIONERS =
+            ImmutableSet.of(OrderPreservingPartitioner.class, LocalPartitioner.class, ByteOrderedPartitioner.class, RandomPartitioner.class);
+
+    private final ColumnFamilyStore baseCfs;
+    private final IndexContext indexContext;
+
+    public StorageAttachedIndex(ColumnFamilyStore baseCfs, IndexMetadata config)
+    {
+        this.baseCfs = baseCfs;
+        TableMetadata tableMetadata = baseCfs.metadata();
+        Pair<ColumnMetadata, IndexTarget.Type> target = TargetParser.parse(tableMetadata, config);
+        this.indexContext = new IndexContext(tableMetadata.keyspace,
+                                             tableMetadata.name,
+                                             tableMetadata.partitionKeyType,
+                                             tableMetadata.comparator,
+                                             target.left,
+                                             target.right,
+                                             config);
+    }
+
+    /**
+     * Used via reflection in {@link IndexMetadata}
+     */
+    @SuppressWarnings({ "unused" })
+    public static Map<String, String> validateOptions(Map<String, String> options, TableMetadata metadata)
+    {
+        Map<String, String> unknown = new HashMap<>(2);
+
+        for (Map.Entry<String, String> option : options.entrySet())
+        {
+            if (!VALID_OPTIONS.contains(option.getKey()))
+            {
+                unknown.put(option.getKey(), option.getValue());
+            }
+        }
+
+        if (!unknown.isEmpty())
+        {
+            return unknown;
+        }
+
+        if (ILLEGAL_PARTITIONERS.contains(metadata.partitioner.getClass()))
+        {
+            throw new InvalidRequestException("Storage-attached index does not support the following IPartitioner implementations: " + ILLEGAL_PARTITIONERS);
+        }
+
+        String targetColumn = options.get(IndexTarget.TARGET_OPTION_NAME);
+
+        if (targetColumn == null)
+        {
+            throw new InvalidRequestException("Missing target column");
+        }
+
+        if (targetColumn.split(",").length > 1)
+        {
+            throw new InvalidRequestException("A storage-attached index cannot be created over multiple columns: " + targetColumn);
+        }
+
+        Pair<ColumnMetadata, IndexTarget.Type> target = TargetParser.parse(metadata, targetColumn);
+
+        if (target == null)
+        {
+            throw new InvalidRequestException("Failed to retrieve target column for: " + targetColumn);
+        }
+
+        // In order to support different index targets on non-frozen map, ie. KEYS, VALUE, ENTRIES, we need to put index
+        // name as part of index file name instead of column name. We only need to check that the target is different
+        // between indexes. This will only allow indexes in the same column with a different IndexTarget.Type.
+        //
+        // Note that: "metadata.indexes" already includes current index
+        if (metadata.indexes.stream().filter(index -> index.getIndexClassName().equals(StorageAttachedIndex.class.getName()))
+                            .map(index -> TargetParser.parse(metadata, index.options.get(IndexTarget.TARGET_OPTION_NAME)))
+                            .filter(Objects::nonNull).filter(t -> t.equals(target)).count() > 1)
+        {
+            throw new InvalidRequestException("Cannot create more than one storage-attached index on the same column: " + target.left);
+        }
+
+        AbstractType<?> type = TypeUtil.cellValueType(target.left, target.right);
+
+        // If we are indexing map entries we need to validate the subtypes
+        if (TypeUtil.isComposite(type))
+        {
+            for (AbstractType<?> subType : type.subTypes())
+            {
+                if (!SUPPORTED_TYPES.contains(subType.asCQL3Type()) && !TypeUtil.isFrozen(subType))
+                    throw new InvalidRequestException("Unsupported type: " + subType.asCQL3Type());
+            }
+        }
+        else if (!SUPPORTED_TYPES.contains(type.asCQL3Type()) && !TypeUtil.isFrozen(type))
+        {
+            throw new InvalidRequestException("Unsupported type: " + type.asCQL3Type());
+        }
+
+        AbstractAnalyzer.fromOptions(type, options);
+
+        return Collections.emptyMap();
+    }
+
+    @Override
+    public void register(IndexRegistry registry)
+    {
+        // index will be available for writes
+        registry.registerIndex(this, StorageAttachedIndexGroup.class, () -> new StorageAttachedIndexGroup(baseCfs));
+    }
+
+    @Override
+    public IndexMetadata getIndexMetadata()
+    {
+        return indexContext.getIndexMetadata();
+    }
+
+    @Override
+    public Callable<?> getInitializationTask()
+    {
+        // New storage-attached indexes will be available for queries after on disk index data are built.
+        // Memtable data will be indexed via flushing triggered by schema change
+        return () -> startInitialBuild(baseCfs).get();
+    }
+
+    private Future<?> startInitialBuild(ColumnFamilyStore baseCfs)
+    {
+        if (baseCfs.indexManager.isIndexQueryable(this))
+        {
+            logger.debug(indexContext.logMessage("Skipping validation and building in initialization task, as pre-join has already made the storage attached index queryable..."));
+        }
+        baseCfs.indexManager.makeIndexQueryable(this, Status.BUILD_SUCCEEDED);
+        return CompletableFuture.completedFuture(null);
+    }
+
+    @Override
+    public Callable<?> getMetadataReloadTask(IndexMetadata indexMetadata)
+    {
+        return null;
+    }
+
+    @Override
+    public Callable<?> getBlockingFlushTask()
+    {
+        return null; // storage-attached indexes are flushed alongside memtable
+    }
+
+    @Override
+    public Callable<?> getInvalidateTask()
+    {
+        return () ->
+        {
+            indexContext.invalidate();
+            return null;
+        };
+    }
+
+    @Override
+    public Callable<?> getPreJoinTask(boolean hadBootstrap)
+    {
+        /*
+         * During bootstrap, streamed SSTable are already built for existing indexes via {@link StorageAttachedIndexBuildingSupport}
+         * from {@link org.apache.cassandra.streaming.StreamReceiveTask.OnCompletionRunnable}.
+         *
+         * For indexes created during bootstrapping, we don't have to block bootstrap for them.
+         */
+
+        return this::startPreJoinTask;
+    }
+
+    private Future<?> startPreJoinTask()
+    {
+        try
+        {
+            if (baseCfs.indexManager.isIndexQueryable(this))
+                logger.debug(indexContext.logMessage("Skipping validation in pre-join task, as the initialization task has already made the index queryable..."));
+
+            baseCfs.indexManager.makeIndexQueryable(this, Status.BUILD_SUCCEEDED);
+        }
+        catch (Throwable t)
+        {
+            logger.error(indexContext.logMessage("Failed in pre-join task!"), t);
+        }
+
+        return null;
+    }
+
+    @Override
+    public Callable<?> getTruncateTask(long truncatedAt)
+    {
+        /*
+         * index files will be removed as part of base sstable lifecycle in
+         * {@link LogTransaction#delete(java.io.File)} asynchronously.
+         */
+        return null;
+    }
+
+    @Override
+    public boolean shouldBuildBlocking()
+    {
+        return true;
+    }
+
+    @Override
+    public Optional<ColumnFamilyStore> getBackingTable()
+    {
+        return Optional.empty();
+    }
+
+    @Override
+    public boolean dependsOn(ColumnMetadata column)
+    {
+        return indexContext.getDefinition().compareTo(column) == 0;
+    }
+
+    @Override
+    public boolean supportsExpression(ColumnMetadata column, Operator operator)
+    {
+        return dependsOn(column) && indexContext.supports(operator);
+    }
+
+    @Override
+    public AbstractType<?> customExpressionValueType()
+    {
+        return null;
+    }
+
+    @Override
+    public RowFilter getPostIndexQueryFilter(RowFilter filter)
+    {
+        // it should be executed from the SAI query plan, this is only used by the singleton index query plan
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public long getEstimatedResultRows()
+    {
+        throw new UnsupportedOperationException("Use StorageAttachedIndexQueryPlan#getEstimatedResultRows() instead.");
+    }
+
+    @Override
+    public boolean isQueryable(Status status)
+    {
+        // consider unknown status as queryable, because gossip may not be up-to-date for newly joining nodes.
+        return status == Status.BUILD_SUCCEEDED || status == Status.UNKNOWN;
+    }
+
+    @Override
+    public void validate(PartitionUpdate update) throws InvalidRequestException
+    {}
+
+    @Override
+    public Searcher searcherFor(ReadCommand command) throws InvalidRequestException
+    {
+        // searchers should be created from the query plan, this is only used by the singleton index query plan
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public SSTableFlushObserver getFlushObserver(Descriptor descriptor, LifecycleNewTracker tracker)
+    {
+        // flush observers should be created from the index group, this is only used by the singleton index group
+        throw new UnsupportedOperationException("Storage-attached index flush observers should never be created directly.");
+    }
+
+    @Override
+    public Indexer indexerFor(DecoratedKey key,
+                              RegularAndStaticColumns columns,
+                              long nowInSec,
+                              WriteContext writeContext,
+                              IndexTransaction.Type transactionType,
+                              Memtable memtable)
+    {
+        if (transactionType == IndexTransaction.Type.UPDATE)
+        {
+            return new UpdateIndexer(key, memtable, writeContext);
+        }
+
+        // we are only interested in the data from Memtable
+        // everything else is going to be handled by SSTableWriter observers
+        return null;
+    }
+
+    @Override
+    public IndexBuildingSupport getBuildTaskSupport()
+    {
+        return INDEX_BUILDER_SUPPORT;
+    }
+
+    public IndexContext getIndexContext()
+    {
+        return indexContext;
+    }
+
+    @Override
+    public String toString()
+    {
+        return String.format("%s.%s.%s", baseCfs.keyspace.getName(), baseCfs.name, getIndexMetadata() == null ? "?" : getIndexMetadata());
+    }
+
+    /**
+     * Removes this index from the {@code SecondaryIndexManager}'s set of queryable indexes.
+     */
+    public void makeIndexNonQueryable()
+    {
+        baseCfs.indexManager.makeIndexNonQueryable(this, Status.BUILD_FAILED);
+        logger.warn(indexContext.logMessage("Storage-attached index is no longer queryable. Please restart this node to repair it."));
+    }
+
+    private class UpdateIndexer implements Index.Indexer
+    {
+        private final DecoratedKey key;
+        private final Memtable memtable;
+        private final WriteContext writeContext;
+
+        UpdateIndexer(DecoratedKey key, Memtable memtable, WriteContext writeContext)
+        {
+            this.key = key;
+            this.memtable = memtable;
+            this.writeContext = writeContext;
+        }
+
+        @Override
+        public void insertRow(Row row)
+        {
+            adjustMemtableSize(indexContext.index(key, row, memtable), CassandraWriteContext.fromContext(writeContext).getGroup());
+        }
+
+        @Override
+        public void updateRow(Row oldRow, Row newRow)
+        {
+            insertRow(newRow);
+        }
+
+        void adjustMemtableSize(long additionalSpace, OpOrder.Group opGroup)
+        {
+            memtable.markExtraOnHeapUsed(additionalSpace, opGroup);
+        }
+    }
+}

src/java/org/apache/cassandra/index/sai/StorageAttachedIndexGroup.java

@@ -0,0 +1,237 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Objects;
+import java.util.Set;
+import java.util.function.Predicate;
+import java.util.stream.Collectors;
+import javax.annotation.Nullable;
+import javax.annotation.concurrent.ThreadSafe;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.Sets;
+import com.google.common.primitives.Ints;
+
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.RegularAndStaticColumns;
+import org.apache.cassandra.db.WriteContext;
+import org.apache.cassandra.db.filter.RowFilter;
+import org.apache.cassandra.db.lifecycle.LifecycleNewTracker;
+import org.apache.cassandra.db.lifecycle.Tracker;
+import org.apache.cassandra.db.memtable.Memtable;
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.index.Index;
+import org.apache.cassandra.index.sai.metrics.TableQueryMetrics;
+import org.apache.cassandra.index.sai.metrics.TableStateMetrics;
+import org.apache.cassandra.index.sai.plan.StorageAttachedIndexQueryPlan;
+import org.apache.cassandra.index.transactions.IndexTransaction;
+import org.apache.cassandra.io.sstable.Component;
+import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.SSTableFlushObserver;
+import org.apache.cassandra.notifications.INotification;
+import org.apache.cassandra.notifications.INotificationConsumer;
+import org.apache.cassandra.notifications.MemtableDiscardedNotification;
+import org.apache.cassandra.notifications.MemtableRenewedNotification;
+import org.apache.cassandra.schema.TableMetadata;
+
+/**
+ * Orchestrates building of storage-attached indices, and manages lifecycle of resources shared between them.
+ */
+@ThreadSafe
+public class StorageAttachedIndexGroup implements Index.Group, INotificationConsumer
+{
+    private final TableQueryMetrics queryMetrics;
+    private final TableStateMetrics stateMetrics;
+    private final Set<Index> indexes = Sets.newConcurrentHashSet();
+    private final ColumnFamilyStore baseCfs;
+
+    StorageAttachedIndexGroup(ColumnFamilyStore baseCfs)
+    {
+        this.baseCfs = baseCfs;
+        this.queryMetrics = new TableQueryMetrics(baseCfs.metadata());
+        this.stateMetrics = new TableStateMetrics(baseCfs.metadata(), this);
+
+        Tracker tracker = baseCfs.getTracker();
+        tracker.subscribe(this);
+    }
+
+    @Nullable
+    public static StorageAttachedIndexGroup getIndexGroup(ColumnFamilyStore cfs)
+    {
+        return (StorageAttachedIndexGroup) cfs.indexManager.getIndexGroup(StorageAttachedIndexGroup.class);
+    }
+
+    @Override
+    public Set<Index> getIndexes()
+    {
+        return indexes;
+    }
+
+    @Override
+    public void addIndex(Index index)
+    {
+        assert index instanceof StorageAttachedIndex;
+        indexes.add(index);
+    }
+
+    @Override
+    public void removeIndex(Index index)
+    {
+        assert index instanceof StorageAttachedIndex;
+        boolean removed = indexes.remove(index);
+        assert removed : "Cannot remove non-existing index " + index;
+        /*
+         * For the in-memory implementation we only need to unsubscribe from the tracker
+         */
+        if (indexes.isEmpty())
+        {
+            baseCfs.getTracker().unsubscribe(this);
+        }
+    }
+
+    @Override
+    public void invalidate()
+    {
+        // in case of dropping table, sstable contexts should already been removed by SSTableListChangedNotification.
+        queryMetrics.release();
+        stateMetrics.release();
+        baseCfs.getTracker().unsubscribe(this);
+    }
+
+    @Override
+    public boolean containsIndex(Index index)
+    {
+        return indexes.contains(index);
+    }
+
+    @Override
+    public Index.Indexer indexerFor(Predicate<Index> indexSelector,
+                                    DecoratedKey key,
+                                    RegularAndStaticColumns columns,
+                                    long nowInSec,
+                                    WriteContext ctx,
+                                    IndexTransaction.Type transactionType,
+                                    Memtable memtable)
+    {
+        final Set<Index.Indexer> indexers =
+                indexes.stream().filter(indexSelector)
+                       .map(i -> i.indexerFor(key, columns, nowInSec, ctx, transactionType, memtable))
+                       .filter(Objects::nonNull)
+                       .collect(Collectors.toSet());
+
+        return indexers.isEmpty() ? null : new Index.Indexer()
+        {
+            @Override
+            public void insertRow(Row row)
+            {
+                for (Index.Indexer indexer : indexers)
+                    indexer.insertRow(row);
+            }
+
+            @Override
+            public void updateRow(Row oldRow, Row newRow)
+            {
+                for (Index.Indexer indexer : indexers)
+                    indexer.updateRow(oldRow, newRow);
+            }
+        };
+    }
+
+    @Override
+    public StorageAttachedIndexQueryPlan queryPlanFor(RowFilter rowFilter)
+    {
+        return StorageAttachedIndexQueryPlan.create(baseCfs, queryMetrics, indexes, rowFilter);
+    }
+
+    @Override
+    public SSTableFlushObserver getFlushObserver(Descriptor descriptor, LifecycleNewTracker tracker, TableMetadata tableMetadata)
+    {
+        return null;
+    }
+
+    @Override
+    public boolean handles(IndexTransaction.Type type)
+    {
+        // to skip CleanupGCTransaction and IndexGCTransaction
+        return type == IndexTransaction.Type.UPDATE;
+    }
+
+    @Override
+    public Set<Component> getComponents()
+    {
+        return getComponents(indexes);
+    }
+
+    // TODO: Currently returns an empty set for in-memory only implementation
+    static Set<Component> getComponents(Collection<Index> indices)
+    {
+        return Collections.emptySet();
+    }
+
+    @Override
+    public void handleNotification(INotification notification, Object sender)
+    {
+        if (notification instanceof MemtableRenewedNotification)
+        {
+            indexes.forEach(index -> ((StorageAttachedIndex)index).getIndexContext().renewMemtable(((MemtableRenewedNotification) notification).renewed));
+        }
+        else if (notification instanceof MemtableDiscardedNotification)
+        {
+            indexes.forEach(index -> ((StorageAttachedIndex)index).getIndexContext().discardMemtable(((MemtableDiscardedNotification) notification).memtable));
+        }
+    }
+
+    /**
+     * @return count of queryable indexes
+     */
+    public int totalQueryableIndexCount()
+    {
+        return Ints.checkedCast(indexes.stream().filter(baseCfs.indexManager::isIndexQueryable).count());
+    }
+
+    /**
+     * @return count of indexes
+     */
+    public int totalIndexCount()
+    {
+        return indexes.size();
+    }
+
+    public TableMetadata metadata()
+    {
+        return baseCfs.metadata();
+    }
+
+    public ColumnFamilyStore table()
+    {
+        return baseCfs;
+    }
+
+    /**
+     * Simulate the index going through a restart of node
+     */
+    @VisibleForTesting
+    public void reset()
+    {
+        indexes.forEach(i -> ((StorageAttachedIndex) i).makeIndexNonQueryable());
+    }
+}

src/java/org/apache/cassandra/index/sai/analyzer/AbstractAnalyzer.java

@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.analyzer;
+
+import java.nio.ByteBuffer;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.NoSuchElementException;
+
+import org.apache.cassandra.db.marshal.AbstractType;
+
+public abstract class AbstractAnalyzer implements Iterator<ByteBuffer>
+{
+    protected ByteBuffer next = null;
+    String nextLiteral = null;
+
+    /**
+     * @return true if index value is transformed, e.g. normalized or lower-cased or tokenized.
+     */
+    public abstract boolean transformValue();
+
+    /**
+     * Call when tokenization is finished.  Used by the LuceneAnalyzer.
+     */
+    public void end()
+    {
+    }
+
+    /**
+     * Note: This method does not advance, as we rely on {@link #hasNext()} to buffer the next value.
+     *
+     * @return the raw value currently buffered by this iterator
+     */
+    @Override
+    public ByteBuffer next()
+    {
+        if (next == null)
+            throw new NoSuchElementException();
+        return next;
+    }
+
+    @Override
+    public void remove()
+    {
+        throw new UnsupportedOperationException();
+    }
+
+    protected abstract void resetInternal(ByteBuffer input);
+
+    public void reset(ByteBuffer input)
+    {
+        this.next = null;
+        this.nextLiteral = null;
+
+        resetInternal(input);
+    }
+
+    public interface AnalyzerFactory
+    {
+        AbstractAnalyzer create();
+
+        default void close()
+        {
+        }
+    }
+
+    public static AnalyzerFactory fromOptions(AbstractType<?> type, Map<String, String> options)
+    {
+        return NoOpAnalyzer::new;
+    }
+}

src/java/org/apache/cassandra/index/sai/analyzer/NoOpAnalyzer.java

@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.analyzer;
+
+import java.nio.ByteBuffer;
+
+import com.google.common.base.MoreObjects;
+
+/**
+ * Default NoOp tokenizer. The iterator will iterate only once returning the unmodified input
+ */
+public class NoOpAnalyzer extends AbstractAnalyzer
+{
+    private ByteBuffer input;
+    private boolean hasNext = false;
+
+    public NoOpAnalyzer() {}
+
+    @Override
+    public boolean hasNext()
+    {
+        if (hasNext)
+        {
+            this.next = input;
+            this.hasNext = false;
+            return true;
+        }
+        this.next = null;
+        return false;
+    }
+
+    @Override
+    protected void resetInternal(ByteBuffer input)
+    {
+        this.input = input;
+        this.hasNext = true;
+    }
+
+    @Override
+    public boolean transformValue()
+    {
+        return false;
+    }
+
+    @Override
+    public String toString()
+    {
+        return MoreObjects.toStringHelper(this).toString();
+    }
+}

src/java/org/apache/cassandra/index/sai/memory/FilteringInMemoryKeyRangeIterator.java

@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.memory;
+
+import java.util.SortedSet;
+
+import org.apache.cassandra.db.PartitionPosition;
+import org.apache.cassandra.dht.AbstractBounds;
+import org.apache.cassandra.index.sai.utils.PrimaryKey;
+
+/**
+ * An {@link InMemoryKeyRangeIterator} that filters the returned {@link PrimaryKey}s based on the provided keyRange
+ */
+public class FilteringInMemoryKeyRangeIterator extends InMemoryKeyRangeIterator
+{
+    private final AbstractBounds<PartitionPosition> keyRange;
+
+    public FilteringInMemoryKeyRangeIterator(SortedSet<PrimaryKey> keys, AbstractBounds<PartitionPosition> keyRange)
+    {
+        super(keys);
+        this.keyRange = keyRange;
+    }
+
+    @Override
+    protected PrimaryKey computeNext()
+    {
+        PrimaryKey key = computeNextKey();
+        while (key != null && !keyRange.contains(key.partitionKey()))
+            key = computeNextKey();
+        return key == null ? endOfData() : key;
+    }
+}

src/java/org/apache/cassandra/index/sai/memory/InMemoryKeyRangeIterator.java

@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.memory;
+
+import java.util.PriorityQueue;
+import java.util.SortedSet;
+import javax.annotation.concurrent.NotThreadSafe;
+
+import org.apache.cassandra.index.sai.utils.KeyRangeIterator;
+import org.apache.cassandra.index.sai.utils.PrimaryKey;
+
+@NotThreadSafe
+public class InMemoryKeyRangeIterator extends KeyRangeIterator
+{
+    private final PriorityQueue<PrimaryKey> keys;
+    private final boolean uniqueKeys;
+    private PrimaryKey lastKey;
+
+    /**
+     * An in-memory {@link KeyRangeIterator} that uses a {@link PriorityQueue} built from a {@link SortedSet}
+     * which has no duplication as its backing store.
+     */
+    public InMemoryKeyRangeIterator(SortedSet<PrimaryKey> keys)
+    {
+        super(keys.first(), keys.last(), keys.size());
+        this.keys = new PriorityQueue<>(keys);
+        this.uniqueKeys = true;
+    }
+
+    /**
+     * An in-memory {@link KeyRangeIterator} that uses a {@link PriorityQueue} which may
+     * contain duplicated keys as its backing store.
+     */
+    public InMemoryKeyRangeIterator(PrimaryKey min, PrimaryKey max, PriorityQueue<PrimaryKey> keys)
+    {
+        super(min, max, keys.size());
+        this.keys = keys;
+        this.uniqueKeys = false;
+    }
+
+    @Override
+    protected PrimaryKey computeNext()
+    {
+        PrimaryKey key = computeNextKey();
+        return key == null ? endOfData() : key;
+    }
+
+    protected PrimaryKey computeNextKey()
+    {
+        PrimaryKey next = null;
+
+        while (!keys.isEmpty())
+        {
+            PrimaryKey key = keys.poll();
+            if (uniqueKeys)
+                return key;
+
+            if (lastKey == null || lastKey.compareTo(key) != 0)
+            {
+                next = key;
+                lastKey = key;
+                break;
+            }
+        }
+
+        return next;
+    }
+
+    @Override
+    protected void performSkipTo(PrimaryKey nextKey)
+    {
+        while (!keys.isEmpty())
+        {
+            PrimaryKey key = keys.peek();
+            if (key.compareTo(nextKey) >= 0)
+                break;
+
+            // consume smaller key
+            keys.poll();
+        }
+    }
+
+    @Override
+    public void close()
+    {}
+}

src/java/org/apache/cassandra/index/sai/memory/MemtableIndex.java

@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.memory;
+
+import java.nio.ByteBuffer;
+import java.util.Iterator;
+import java.util.concurrent.atomic.LongAdder;
+
+import org.apache.cassandra.db.Clustering;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.PartitionPosition;
+import org.apache.cassandra.dht.AbstractBounds;
+import org.apache.cassandra.index.sai.IndexContext;
+import org.apache.cassandra.index.sai.plan.Expression;
+import org.apache.cassandra.index.sai.utils.PrimaryKeys;
+import org.apache.cassandra.index.sai.utils.KeyRangeIterator;
+import org.apache.cassandra.utils.Pair;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+
+public class MemtableIndex
+{
+    private final TrieMemoryIndex index;
+    private final LongAdder writeCount = new LongAdder();
+    private final LongAdder estimatedMemoryUsed = new LongAdder();
+
+    public MemtableIndex(IndexContext indexContext)
+    {
+        this.index = new TrieMemoryIndex(indexContext);
+    }
+
+    public long writeCount()
+    {
+        return writeCount.sum();
+    }
+
+    public long estimatedMemoryUsed()
+    {
+        return estimatedMemoryUsed.sum();
+    }
+
+    public boolean isEmpty()
+    {
+        return getMinTerm() == null;
+    }
+
+    public ByteBuffer getMinTerm()
+    {
+        return index.getMinTerm();
+    }
+
+    public ByteBuffer getMaxTerm()
+    {
+        return index.getMaxTerm();
+    }
+
+    public long index(DecoratedKey key, Clustering<?> clustering, ByteBuffer value)
+    {
+        if (value == null || value.remaining() == 0)
+            return 0;
+
+        long ram = index.add(key, clustering, value);
+        writeCount.increment();
+        estimatedMemoryUsed.add(ram);
+        return ram;
+    }
+
+    public KeyRangeIterator search(Expression expression, AbstractBounds<PartitionPosition> keyRange)
+    {
+        return index.search(expression, keyRange);
+    }
+
+    public Iterator<Pair<ByteComparable, PrimaryKeys>> iterator()
+    {
+        return index.iterator();
+    }
+}

src/java/org/apache/cassandra/index/sai/memory/TrieMemoryIndex.java

@@ -0,0 +1,383 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.memory;
+
+import java.nio.ByteBuffer;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.PriorityQueue;
+import java.util.SortedSet;
+import java.util.concurrent.atomic.LongAdder;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import io.netty.util.concurrent.FastThreadLocal;
+import org.apache.cassandra.db.Clustering;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.PartitionPosition;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.memtable.TrieMemtable;
+import org.apache.cassandra.db.tries.InMemoryTrie;
+import org.apache.cassandra.db.tries.Trie;
+import org.apache.cassandra.dht.AbstractBounds;
+import org.apache.cassandra.index.sai.IndexContext;
+import org.apache.cassandra.index.sai.analyzer.AbstractAnalyzer;
+import org.apache.cassandra.index.sai.plan.Expression;
+import org.apache.cassandra.index.sai.utils.PrimaryKey;
+import org.apache.cassandra.index.sai.utils.PrimaryKeys;
+import org.apache.cassandra.index.sai.utils.KeyRangeIterator;
+import org.apache.cassandra.index.sai.utils.TypeUtil;
+import org.apache.cassandra.utils.Pair;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+import org.apache.cassandra.utils.bytecomparable.ByteSource;
+import org.apache.cassandra.utils.bytecomparable.ByteSourceInverse;
+
+/**
+ * This is an in-memory index using the {@link InMemoryTrie} to store a {@link ByteComparable}
+ * representation of the indexed values. Data is stored on-heap or off-heap and follows the
+ * settings of the {@link TrieMemtable} to determine where.
+ *
+ *
+ */
+public class TrieMemoryIndex
+{
+    private static final Logger logger = LoggerFactory.getLogger(TrieMemoryIndex.class);
+    private static final int MAX_RECURSIVE_KEY_LENGTH = 128;
+
+    private final IndexContext indexContext;
+    private final InMemoryTrie<PrimaryKeys> data;
+    private final PrimaryKeysReducer primaryKeysReducer;
+    private final AbstractAnalyzer.AnalyzerFactory analyzerFactory;
+    private final AbstractType<?> validator;
+    private final boolean isLiteral;
+
+    private ByteBuffer minTerm;
+    private ByteBuffer maxTerm;
+
+    public TrieMemoryIndex(IndexContext indexContext)
+    {
+        this.indexContext = indexContext;
+        this.data = new InMemoryTrie<>(TrieMemtable.BUFFER_TYPE);
+        this.primaryKeysReducer = new PrimaryKeysReducer();
+        // The use of the analyzer is within a synchronized block so can be considered thread-safe
+        this.analyzerFactory = indexContext.getIndexAnalyzerFactory();
+        this.validator = indexContext.getValidator();
+        this.isLiteral = TypeUtil.isLiteral(validator);
+    }
+
+    /**
+     * Adds an index value to the in-memory index
+     *
+     * @param key partition key for the indexed value
+     * @param clustering clustering for the indexed value
+     * @param value indexed value
+     * @return amount of heap allocated by the new value
+     */
+    public synchronized long add(DecoratedKey key, Clustering<?> clustering, ByteBuffer value)
+    {
+        AbstractAnalyzer analyzer = analyzerFactory.create();
+        try
+        {
+            value = TypeUtil.asIndexBytes(value, validator);
+            analyzer.reset(value);
+            final PrimaryKey primaryKey = indexContext.keyFactory().create(key, clustering);
+            final long initialSizeOnHeap = data.sizeOnHeap();
+            final long initialSizeOffHeap = data.sizeOffHeap();
+            final long reducerHeapSize = primaryKeysReducer.heapAllocations();
+
+            while (analyzer.hasNext())
+            {
+                final ByteBuffer term = analyzer.next();
+
+                setMinMaxTerm(term.duplicate());
+
+                final ByteComparable comparableBytes = asComparableBytes(term);
+
+                try
+                {
+                    if (term.limit() <= MAX_RECURSIVE_KEY_LENGTH)
+                    {
+                        data.putRecursive(comparableBytes, primaryKey, primaryKeysReducer);
+                    }
+                    else
+                    {
+                        data.apply(Trie.singleton(comparableBytes, primaryKey), primaryKeysReducer);
+                    }
+                }
+                catch (InMemoryTrie.SpaceExhaustedException e)
+                {
+                    throw new RuntimeException(e);
+                }
+            }
+
+            long onHeap = data.sizeOnHeap();
+            long offHeap = data.sizeOffHeap();
+            long heapAllocations = primaryKeysReducer.heapAllocations();
+            return (onHeap - initialSizeOnHeap) + (offHeap - initialSizeOffHeap) + (heapAllocations - reducerHeapSize);
+        }
+        finally
+        {
+            analyzer.end();
+        }
+    }
+
+    /**
+     * Search for an expression in the in-memory index within the {@link AbstractBounds} defined
+     * by keyRange. This can either be an exact match or a range match.
+
+     * @param expression the {@link Expression} to search for
+     * @param keyRange the {@link AbstractBounds} containing the key range to restrict the search to
+     * @return a {@link KeyRangeIterator} containing the search results
+     */
+    public KeyRangeIterator search(Expression expression, AbstractBounds<PartitionPosition> keyRange)
+    {
+        if (logger.isTraceEnabled())
+            logger.trace("Searching memtable index on expression '{}'...", expression);
+
+        switch (expression.getOp())
+        {
+            case EQ:
+            case CONTAINS_KEY:
+            case CONTAINS_VALUE:
+                return exactMatch(expression, keyRange);
+            case RANGE:
+                return rangeMatch(expression, keyRange);
+            default:
+                throw new IllegalArgumentException("Unsupported expression: " + expression);
+        }
+    }
+
+    /**
+     * Returns an {@link Iterator} over the entire dataset contained in the trie. This is used
+     * when the index is flushed to disk.
+     *
+     * @return the iterator containing the trie data
+     */
+    public Iterator<Pair<ByteComparable, PrimaryKeys>> iterator()
+    {
+        Iterator<Map.Entry<ByteComparable, PrimaryKeys>> iterator = data.entrySet().iterator();
+        return new Iterator<Pair<ByteComparable, PrimaryKeys>>()
+        {
+            @Override
+            public boolean hasNext()
+            {
+                return iterator.hasNext();
+            }
+
+            @Override
+            public Pair<ByteComparable, PrimaryKeys> next()
+            {
+                Map.Entry<ByteComparable, PrimaryKeys> entry = iterator.next();
+                return Pair.create(decode(entry.getKey()), entry.getValue());
+            }
+        };
+    }
+
+    public ByteBuffer getMinTerm()
+    {
+        return minTerm;
+    }
+
+    public ByteBuffer getMaxTerm()
+    {
+        return maxTerm;
+    }
+
+    private void setMinMaxTerm(ByteBuffer term)
+    {
+        assert term != null;
+
+        minTerm = TypeUtil.min(term, minTerm, indexContext.getValidator());
+        maxTerm = TypeUtil.max(term, maxTerm, indexContext.getValidator());
+    }
+
+    private ByteComparable asComparableBytes(ByteBuffer input)
+    {
+        return isLiteral ? version -> terminated(ByteSource.of(input, version))
+                         : version -> TypeUtil.asComparableBytes(input, validator, version);
+    }
+
+    private ByteComparable decode(ByteComparable term)
+    {
+        return isLiteral ? version -> ByteSourceInverse.unescape(ByteSource.peekable(term.asComparableBytes(version)))
+                         : term;
+
+    }
+
+    private ByteSource terminated(ByteSource src)
+    {
+        return new ByteSource()
+        {
+            boolean done = false;
+
+            @Override
+            public int next()
+            {
+                if (done)
+                    return END_OF_STREAM;
+                int n = src.next();
+                if (n != END_OF_STREAM)
+                    return n;
+
+                done = true;
+                return ByteSource.TERMINATOR;
+            }
+        };
+    }
+
+    private KeyRangeIterator exactMatch(Expression expression, AbstractBounds<PartitionPosition> keyRange)
+    {
+        ByteComparable comparableMatch = expression.lower == null ? ByteComparable.EMPTY
+                                                                  : asComparableBytes(expression.lower.value.encoded);
+        PrimaryKeys primaryKeys = data.get(comparableMatch);
+        return primaryKeys == null ? KeyRangeIterator.empty()
+                                   : new FilteringInMemoryKeyRangeIterator(primaryKeys.keys(), keyRange);
+    }
+
+    private static class Collector
+    {
+        private static final int MINIMUM_QUEUE_SIZE = 128;
+
+        // Maintain the last queue size used on this index to use for the next range match.
+        // This allows for receiving a stream of wide range queries where the queue size
+        // is larger than we would want to default the size to.
+        // TODO Investigate using a decaying histogram here to avoid the effect of outliers.
+        private static final FastThreadLocal<Integer> lastQueueSize = new FastThreadLocal<Integer>()
+        {
+            protected Integer initialValue()
+            {
+                return MINIMUM_QUEUE_SIZE;
+            }
+        };
+
+        PrimaryKey minimumKey = null;
+        PrimaryKey maximumKey = null;
+        final PriorityQueue<PrimaryKey> mergedKeys = new PriorityQueue<>(lastQueueSize.get());
+
+        final AbstractBounds<PartitionPosition> keyRange;
+
+        public Collector(AbstractBounds<PartitionPosition> keyRange)
+        {
+            this.keyRange = keyRange;
+        }
+
+        public void processContent(PrimaryKeys keys)
+        {
+            if (keys.isEmpty())
+                return;
+
+            SortedSet<PrimaryKey> primaryKeys = keys.keys();
+
+            // shortcut to avoid generating iterator
+            if (primaryKeys.size() == 1)
+            {
+                processKey(primaryKeys.first());
+                return;
+            }
+
+            // skip entire partition keys if they don't overlap
+            if (!keyRange.right.isMinimum() && primaryKeys.first().partitionKey().compareTo(keyRange.right) > 0
+                || primaryKeys.last().partitionKey().compareTo(keyRange.left) < 0)
+                return;
+
+            primaryKeys.forEach(this::processKey);
+        }
+
+        public void updateLastQueueSize()
+        {
+            lastQueueSize.set(Math.max(MINIMUM_QUEUE_SIZE, mergedKeys.size()));
+        }
+
+        private void processKey(PrimaryKey key)
+        {
+            if (keyRange.contains(key.partitionKey()))
+            {
+                mergedKeys.add(key);
+
+                minimumKey = minimumKey == null ? key : key.compareTo(minimumKey) < 0 ? key : minimumKey;
+                maximumKey = maximumKey == null ? key : key.compareTo(maximumKey) > 0 ? key : maximumKey;
+            }
+        }
+    }
+
+    private KeyRangeIterator rangeMatch(Expression expression, AbstractBounds<PartitionPosition> keyRange)
+    {
+        ByteComparable lowerBound, upperBound;
+        boolean lowerInclusive, upperInclusive;
+        if (expression.lower != null)
+        {
+            lowerBound = asComparableBytes(expression.lower.value.encoded);
+            lowerInclusive = expression.lower.inclusive;
+        }
+        else
+        {
+            lowerBound = ByteComparable.EMPTY;
+            lowerInclusive = false;
+        }
+
+        if (expression.upper != null)
+        {
+            upperBound = asComparableBytes(expression.upper.value.encoded);
+            upperInclusive = expression.upper.inclusive;
+        }
+        else
+        {
+            upperBound = null;
+            upperInclusive = false;
+        }
+
+        Collector cd = new Collector(keyRange);
+
+        data.subtrie(lowerBound, lowerInclusive, upperBound, upperInclusive)
+            .values()
+            .forEach(cd::processContent);
+
+        if (cd.mergedKeys.isEmpty())
+        {
+            return KeyRangeIterator.empty();
+        }
+
+        cd.updateLastQueueSize();
+
+        return new InMemoryKeyRangeIterator(cd.minimumKey, cd.maximumKey, cd.mergedKeys);
+    }
+
+    private static class PrimaryKeysReducer implements InMemoryTrie.UpsertTransformer<PrimaryKeys, PrimaryKey>
+    {
+        private final LongAdder heapAllocations = new LongAdder();
+
+        @Override
+        public PrimaryKeys apply(PrimaryKeys existing, PrimaryKey neww)
+        {
+            if (existing == null)
+            {
+                existing = new PrimaryKeys();
+                heapAllocations.add(existing.unsharedHeapSize());
+            }
+            heapAllocations.add(existing.add(neww));
+            return existing;
+        }
+
+        long heapAllocations()
+        {
+            return heapAllocations.longValue();
+        }
+    }
+}

src/java/org/apache/cassandra/index/sai/metrics/AbstractMetrics.java

@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.metrics;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.cassandra.metrics.CassandraMetricsRegistry;
+import org.apache.cassandra.metrics.DefaultNameFactory;
+
+import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics;
+
+public abstract class AbstractMetrics
+{
+    public static final String TYPE = "StorageAttachedIndex";
+
+    protected final String keyspace;
+    protected final String table;
+    private final String index;
+    private final String scope;
+    protected final List<CassandraMetricsRegistry.MetricName> tracked = new ArrayList<>();
+
+    AbstractMetrics(String keyspace, String table, String scope)
+    {
+        this(keyspace, table, null, scope);
+    }
+
+    AbstractMetrics(String keyspace, String table, String index, String scope)
+    {
+        assert keyspace != null && table != null : "SAI metrics must include keyspace and table";
+        this.keyspace = keyspace;
+        this.table = table;
+        this.index = index;
+        this.scope = scope;
+    }
+
+    public void release()
+    {
+        tracked.forEach(Metrics::remove);
+        tracked.clear();
+    }
+
+    protected CassandraMetricsRegistry.MetricName createMetricName(String name)
+    {
+        return createMetricName(name, scope);
+    }
+
+    protected CassandraMetricsRegistry.MetricName createMetricName(String name, String scope)
+    {
+        String metricScope = keyspace + '.' + table;
+        if (index != null)
+        {
+            metricScope += '.' + index;
+        }
+        metricScope += '.' + scope + '.' + name;
+
+        CassandraMetricsRegistry.MetricName metricName = new CassandraMetricsRegistry.MetricName(DefaultNameFactory.GROUP_NAME,
+                                                                                                 TYPE, name, metricScope, createMBeanName(name, scope));
+        tracked.add(metricName);
+        return metricName;
+    }
+
+    private String createMBeanName(String name, String scope)
+    {
+        StringBuilder builder = new StringBuilder();
+        builder.append(DefaultNameFactory.GROUP_NAME);
+        builder.append(":type=").append(TYPE);
+        builder.append(',').append("keyspace=").append(keyspace);
+        builder.append(',').append("table=").append(table);
+        if (index != null)
+            builder.append(',').append("index=").append(index);
+        builder.append(',').append("scope=").append(scope);
+        builder.append(',').append("name=").append(name);
+        return builder.toString();
+    }
+}

src/java/org/apache/cassandra/index/sai/metrics/IndexMetrics.java

@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.metrics;
+
+import com.codahale.metrics.Gauge;
+import com.codahale.metrics.Timer;
+import org.apache.cassandra.index.sai.IndexContext;
+
+import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics;
+
+public class IndexMetrics extends AbstractMetrics
+{
+    public final Timer memtableIndexWriteLatency;
+    public final Gauge<Long> liveMemtableIndexWriteCount;
+    public final Gauge<Long> memtableIndexBytes;
+
+    public IndexMetrics(IndexContext context)
+    {
+        super(context.getKeyspace(), context.getTable(), context.getIndexName(), "IndexMetrics");
+
+        memtableIndexWriteLatency = Metrics.timer(createMetricName("MemtableIndexWriteLatency"));
+        liveMemtableIndexWriteCount = Metrics.register(createMetricName("LiveMemtableIndexWriteCount"), context::liveMemtableWriteCount);
+        memtableIndexBytes = Metrics.register(createMetricName("MemtableIndexBytes"), context::estimatedMemIndexMemoryUsed);
+    }
+}

src/java/org/apache/cassandra/index/sai/metrics/TableQueryMetrics.java

@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.metrics;
+
+import java.util.concurrent.TimeUnit;
+
+import com.codahale.metrics.Counter;
+import com.codahale.metrics.Histogram;
+import com.codahale.metrics.Timer;
+import org.apache.cassandra.index.sai.QueryContext;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.tracing.Tracing;
+
+import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics;
+
+public class TableQueryMetrics extends AbstractMetrics
+{
+    public static final String TABLE_QUERY_METRIC_TYPE = "TableQueryMetrics";
+
+    private final PerQueryMetrics perQueryMetrics;
+    private final Counter totalQueryTimeouts;
+    private final Counter totalPartitionReads;
+    private final Counter totalRowsFiltered;
+    private final Counter totalQueriesCompleted;
+
+    public TableQueryMetrics(TableMetadata table)
+    {
+        super(table.keyspace, table.name, TABLE_QUERY_METRIC_TYPE);
+
+        perQueryMetrics = new PerQueryMetrics(table);
+
+        totalPartitionReads = Metrics.counter(createMetricName("TotalPartitionReads"));
+        totalRowsFiltered = Metrics.counter(createMetricName("TotalRowsFiltered"));
+        totalQueriesCompleted = Metrics.counter(createMetricName("TotalQueriesCompleted"));
+        totalQueryTimeouts = Metrics.counter(createMetricName("TotalQueryTimeouts"));
+    }
+
+    public void record(QueryContext queryContext)
+    {
+        if (queryContext.queryTimeouts > 0)
+        {
+            assert queryContext.queryTimeouts == 1;
+
+            totalQueryTimeouts.inc();
+        }
+
+        perQueryMetrics.record(queryContext);
+    }
+
+    public void release()
+    {
+        super.release();
+        perQueryMetrics.release();
+    }
+
+    public class PerQueryMetrics extends AbstractMetrics
+    {
+        private final Timer queryLatency;
+
+        /**
+         * Global metrics for all indexes hit during the query.
+         */
+        private final Histogram partitionReads;
+        private final Histogram rowsFiltered;
+
+        public PerQueryMetrics(TableMetadata table)
+        {
+            super(table.keyspace, table.name, "PerQuery");
+
+            queryLatency = Metrics.timer(createMetricName("QueryLatency"));
+
+            partitionReads = Metrics.histogram(createMetricName("PartitionReads"), false);
+            rowsFiltered = Metrics.histogram(createMetricName("RowsFiltered"), false);
+        }
+
+        public void record(QueryContext queryContext)
+        {
+            final long totalQueryTimeNs = queryContext.totalQueryTimeNs();
+            queryLatency.update(totalQueryTimeNs, TimeUnit.NANOSECONDS);
+            final long queryLatencyMicros = TimeUnit.NANOSECONDS.toMicros(totalQueryTimeNs);
+
+            final long partitionsRead = queryContext.partitionsRead;
+            final long rowsFiltered = queryContext.rowsFiltered;
+
+            partitionReads.update(partitionsRead);
+            totalPartitionReads.inc(partitionsRead);
+
+            this.rowsFiltered.update(rowsFiltered);
+            totalRowsFiltered.inc(rowsFiltered);
+
+            if (Tracing.isTracing())
+            {
+                Tracing.trace("Index query accessed memtable indexes and took {} microseconds.",
+                              queryLatencyMicros);
+            }
+
+            totalQueriesCompleted.inc();
+        }
+    }
+}

src/java/org/apache/cassandra/index/sai/metrics/TableStateMetrics.java

@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.metrics;
+
+import com.codahale.metrics.Gauge;
+import org.apache.cassandra.index.sai.StorageAttachedIndexGroup;
+import org.apache.cassandra.schema.TableMetadata;
+
+import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics;
+
+public class TableStateMetrics extends AbstractMetrics
+{
+    public static final String TABLE_STATE_METRIC_TYPE = "TableStateMetrics";
+
+    public final Gauge<Integer> totalIndexCount;
+    public final Gauge<Integer> totalQueryableIndexCount;
+
+    public TableStateMetrics(TableMetadata table, StorageAttachedIndexGroup group)
+    {
+        super(table.keyspace, table.name, TABLE_STATE_METRIC_TYPE);
+        totalQueryableIndexCount = Metrics.register(createMetricName("TotalQueryableIndexCount"), group::totalQueryableIndexCount);
+        totalIndexCount = Metrics.register(createMetricName("TotalIndexCount"), group::totalIndexCount);
+    }
+}

src/java/org/apache/cassandra/index/sai/plan/Expression.java

@@ -0,0 +1,378 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.plan;
+
+import java.nio.ByteBuffer;
+import java.util.Objects;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.commons.lang3.builder.HashCodeBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.cql3.Operator;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.index.sai.IndexContext;
+import org.apache.cassandra.index.sai.analyzer.AbstractAnalyzer;
+import org.apache.cassandra.index.sai.utils.TypeUtil;
+
+public class Expression
+{
+    private static final Logger logger = LoggerFactory.getLogger(Expression.class);
+
+    public enum IndexOperator
+    {
+        EQ, RANGE, CONTAINS_KEY, CONTAINS_VALUE;
+
+        public static IndexOperator valueOf(Operator operator)
+        {
+            switch (operator)
+            {
+                case EQ:
+                    return EQ;
+
+                case CONTAINS:
+                    return CONTAINS_VALUE; // non-frozen map: value contains term;
+
+                case CONTAINS_KEY:
+                    return CONTAINS_KEY; // non-frozen map: value contains key term;
+
+                case LT:
+                case GT:
+                case LTE:
+                case GTE:
+                    return RANGE;
+
+                default:
+                    return null;
+            }
+        }
+    }
+
+    public final AbstractAnalyzer.AnalyzerFactory analyzerFactory;
+
+    public final IndexContext context;
+    public final AbstractType<?> validator;
+
+    @VisibleForTesting
+    protected IndexOperator operator;
+
+    public Bound lower, upper;
+    // The upperInclusive and lowerInclusive flags are maintained separately to the inclusive flags
+    // in the upper and lower bounds because the upper and lower bounds have their inclusivity relaxed
+    // if the datatype being filtered is rounded in the index. These flags are used in the post-filtering
+    // process to remove values equal to the bounds.
+    public boolean upperInclusive, lowerInclusive;
+
+    public Expression(IndexContext indexContext)
+    {
+        this.context = indexContext;
+        this.analyzerFactory = indexContext.getQueryAnalyzerFactory();
+        this.validator = indexContext.getValidator();
+    }
+
+    /**
+     * This adds an operation to the current {@link Expression} instance and
+     * returns the current instance.
+     *
+     * @param op the CQL3 operation
+     * @param value the expression value
+     * @return the current expression with the added operation
+     */
+    public Expression add(Operator op, ByteBuffer value)
+    {
+        boolean lowerInclusive, upperInclusive;
+        // If the type supports rounding then we need to make sure that index
+        // range search is always inclusive, otherwise we run the risk of
+        // missing values that are within the exclusive range but are rejected
+        // because their rounded value is the same as the value being queried.
+        lowerInclusive = upperInclusive = TypeUtil.supportsRounding(validator);
+        switch (op)
+        {
+            case EQ:
+            case CONTAINS:
+            case CONTAINS_KEY:
+                lower = new Bound(value, validator, true);
+                upper = lower;
+                operator = IndexOperator.valueOf(op);
+                break;
+
+            case LTE:
+                if (context.getDefinition().isReversedType())
+                {
+                    this.lowerInclusive = true;
+                    lowerInclusive = true;
+                }
+                else
+                {
+                    this.upperInclusive = true;
+                    upperInclusive = true;
+                }
+            case LT:
+                operator = IndexOperator.RANGE;
+                if (context.getDefinition().isReversedType())
+                    lower = new Bound(value, validator, lowerInclusive);
+                else
+                    upper = new Bound(value, validator, upperInclusive);
+                break;
+
+            case GTE:
+                if (context.getDefinition().isReversedType())
+                {
+                    this.upperInclusive = true;
+                    upperInclusive = true;
+                }
+                else
+                {
+                    this.lowerInclusive = true;
+                    lowerInclusive = true;
+                }
+            case GT:
+                operator = IndexOperator.RANGE;
+                if (context.getDefinition().isReversedType())
+                    upper = new Bound(value, validator, upperInclusive);
+                else
+                    lower = new Bound(value, validator, lowerInclusive);
+                break;
+        }
+
+        assert operator != null;
+
+        return this;
+    }
+
+    /**
+     * Used in post-filtering to determine is an indexed value matches the expression
+     */
+    public boolean isSatisfiedBy(ByteBuffer columnValue)
+    {
+        if (!TypeUtil.isValid(columnValue, validator))
+        {
+            logger.error(context.logMessage("Value is not valid for indexed column {} with {}"), context.getColumnName(), validator);
+            return false;
+        }
+
+        Value value = new Value(columnValue, validator);
+
+        if (lower != null)
+        {
+            // suffix check
+            if (TypeUtil.isLiteral(validator))
+                return validateStringValue(value.raw, lower.value.raw);
+            else
+            {
+                // range or (not-)equals - (mainly) for numeric values
+                int cmp = TypeUtil.comparePostFilter(lower.value, value, validator);
+
+                // in case of EQ lower == upper
+                if (operator == IndexOperator.EQ || operator == IndexOperator.CONTAINS_KEY || operator == IndexOperator.CONTAINS_VALUE)
+                    return cmp == 0;
+
+                if (cmp > 0 || (cmp == 0 && !lowerInclusive))
+                    return false;
+            }
+        }
+
+        if (upper != null && lower != upper)
+        {
+            // string (prefix or suffix) check
+            if (TypeUtil.isLiteral(validator))
+                return validateStringValue(value.raw, upper.value.raw);
+            else
+            {
+                // range - mainly for numeric values
+                int cmp = TypeUtil.comparePostFilter(upper.value, value, validator);
+                return (cmp > 0 || (cmp == 0 && upperInclusive));
+            }
+        }
+
+        return true;
+    }
+
+    private boolean validateStringValue(ByteBuffer columnValue, ByteBuffer requestedValue)
+    {
+        AbstractAnalyzer analyzer = analyzerFactory.create();
+        analyzer.reset(columnValue.duplicate());
+        try
+        {
+            while (analyzer.hasNext())
+            {
+                final ByteBuffer term = analyzer.next();
+
+                boolean isMatch = false;
+                switch (operator)
+                {
+                    case EQ:
+                    case CONTAINS_KEY:
+                    case CONTAINS_VALUE:
+                        isMatch = validator.compare(term, requestedValue) == 0;
+                        break;
+                    case RANGE:
+                        isMatch = isLowerSatisfiedBy(term) && isUpperSatisfiedBy(term);
+                        break;
+                }
+
+                if (isMatch)
+                    return true;
+            }
+            return false;
+        }
+        finally
+        {
+            analyzer.end();
+        }
+    }
+
+    public IndexOperator getOp()
+    {
+        return operator;
+    }
+
+    private boolean hasLower()
+    {
+        return lower != null;
+    }
+
+    private boolean hasUpper()
+    {
+        return upper != null;
+    }
+
+    private boolean isLowerSatisfiedBy(ByteBuffer value)
+    {
+        if (!hasLower())
+            return true;
+
+        int cmp = validator.compare(value, lower.value.raw);
+        return cmp > 0 || cmp == 0 && lower.inclusive;
+    }
+
+    private boolean isUpperSatisfiedBy(ByteBuffer value)
+    {
+        if (!hasUpper())
+            return true;
+
+        int cmp = validator.compare(value, upper.value.raw);
+        return cmp < 0 || cmp == 0 && upper.inclusive;
+    }
+
+    @Override
+    public String toString()
+    {
+        return String.format("Expression{name: %s, op: %s, lower: (%s, %s), upper: (%s, %s)}",
+                             context.getColumnName(),
+                             operator,
+                             lower == null ? "null" : validator.getString(lower.value.raw),
+                             lower != null && lower.inclusive,
+                             upper == null ? "null" : validator.getString(upper.value.raw),
+                             upper != null && upper.inclusive);
+    }
+
+    @Override
+    public int hashCode()
+    {
+        return new HashCodeBuilder().append(context.getColumnName())
+                                    .append(operator)
+                                    .append(validator)
+                                    .append(lower).append(upper).build();
+    }
+
+    @Override
+    public boolean equals(Object other)
+    {
+        if (!(other instanceof Expression))
+            return false;
+
+        if (this == other)
+            return true;
+
+        Expression o = (Expression) other;
+
+        return Objects.equals(context.getColumnName(), o.context.getColumnName())
+               && validator.equals(o.validator)
+               && operator == o.operator
+               && Objects.equals(lower, o.lower)
+               && Objects.equals(upper, o.upper);
+    }
+
+    /**
+     * A representation of a column value in its raw and encoded form.
+     */
+    public static class Value
+    {
+        public final ByteBuffer raw;
+        public final ByteBuffer encoded;
+
+        public Value(ByteBuffer value, AbstractType<?> type)
+        {
+            this.raw = value;
+            this.encoded = TypeUtil.asIndexBytes(value, type);
+        }
+
+        @Override
+        public boolean equals(Object other)
+        {
+            if (!(other instanceof Value))
+                return false;
+
+            Value o = (Value) other;
+            return raw.equals(o.raw) && encoded.equals(o.encoded);
+        }
+
+        @Override
+        public int hashCode()
+        {
+            HashCodeBuilder builder = new HashCodeBuilder();
+            builder.append(raw);
+            builder.append(encoded);
+            return builder.toHashCode();
+        }
+    }
+
+    public static class Bound
+    {
+        public final Value value;
+        public final boolean inclusive;
+
+        public Bound(ByteBuffer value, AbstractType<?> type, boolean inclusive)
+        {
+            this.value = new Value(value, type);
+            this.inclusive = inclusive;
+        }
+
+        @Override
+        public boolean equals(Object other)
+        {
+            if (!(other instanceof Bound))
+                return false;
+
+            Bound o = (Bound) other;
+            return value.equals(o.value) && inclusive == o.inclusive;
+        }
+
+        @Override
+        public int hashCode()
+        {
+            HashCodeBuilder builder = new HashCodeBuilder();
+            builder.append(value);
+            builder.append(inclusive);
+            return builder.toHashCode();
+        }
+    }
+}

src/java/org/apache/cassandra/index/sai/plan/FilterTree.java

@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.plan;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.ListIterator;
+
+import com.google.common.collect.ListMultimap;
+
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.db.rows.Unfiltered;
+import org.apache.cassandra.index.sai.utils.TypeUtil;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.ColumnMetadata.Kind;
+import org.apache.cassandra.utils.FBUtilities;
+
+import static org.apache.cassandra.index.sai.plan.Operation.BooleanOperator;
+
+/**
+ * Tree-like structure to filter base table data using indexed expressions and non-user-defined filters.
+ *
+ * This is needed because:
+ * 1. SAI doesn't index tombstones, base data may have been shadowed.
+ * 2. Replica filter protecting may fetch data that doesn't match index expressions.
+ */
+public class FilterTree
+{
+    protected final BooleanOperator op;
+    protected final ListMultimap<ColumnMetadata, Expression> expressions;
+    protected final List<FilterTree> children = new ArrayList<>();
+
+    FilterTree(BooleanOperator operation,
+               ListMultimap<ColumnMetadata, Expression> expressions)
+    {
+        this.op = operation;
+        this.expressions = expressions;
+    }
+
+    void addChild(FilterTree child)
+    {
+        children.add(child);
+    }
+
+    public boolean isSatisfiedBy(DecoratedKey key, Unfiltered unfiltered, Row staticRow)
+    {
+        boolean result = localSatisfiedBy(key, unfiltered, staticRow);
+
+        for (FilterTree child : children)
+            result = op.apply(result, child.isSatisfiedBy(key, unfiltered, staticRow));
+
+        return result;
+    }
+
+    private boolean localSatisfiedBy(DecoratedKey key, Unfiltered unfiltered, Row staticRow)
+    {
+        if (unfiltered == null || !unfiltered.isRow())
+            return false;
+
+        final long now = FBUtilities.nowInSeconds();
+        boolean result = op == BooleanOperator.AND;
+
+        Iterator<ColumnMetadata> columnIterator = expressions.keySet().iterator();
+        while (columnIterator.hasNext())
+        {
+            ColumnMetadata column = columnIterator.next();
+            Row row = column.kind == Kind.STATIC ? staticRow : (Row) unfiltered;
+
+            // If there is a column with multiple expressions that can mean an OR or (in the case of map
+            // collections) it can mean different map indexes.
+            List<Expression> filters = expressions.get(column);
+
+            // We do a reverse iteration over the filters because NOT_EQ operations will be at the end
+            // of the filter list, and we want to check them first.
+            ListIterator<Expression> filterIterator = filters.listIterator(filters.size());
+            while (filterIterator.hasPrevious())
+            {
+                Expression filter = filterIterator.previous();
+
+                if (TypeUtil.isNonFrozenCollection(column.type))
+                {
+                    Iterator<ByteBuffer> valueIterator = filter.context.getValuesOf(row, now);
+                    result = op.apply(result, collectionMatch(valueIterator, filter));
+                }
+                else
+                {
+                    ByteBuffer value = filter.context.getValueOf(key, row, now);
+                    result = op.apply(result, singletonMatch(value, filter));
+                }
+
+                // If the operation is an AND then exit early if we get a single false
+                if (op == BooleanOperator.AND && !result)
+                    return false;
+            }
+        }
+        return result;
+    }
+
+    private boolean singletonMatch(ByteBuffer value, Expression filter)
+    {
+        return value != null && filter.isSatisfiedBy(value);
+    }
+
+    private boolean collectionMatch(Iterator<ByteBuffer> valueIterator, Expression filter)
+    {
+        if (valueIterator == null)
+            return false;
+
+        while (valueIterator.hasNext())
+        {
+            ByteBuffer value = valueIterator.next();
+            if (value == null)
+                continue;
+
+            if (filter.isSatisfiedBy(value))
+                return true;
+        }
+        return false;
+    }
+}

src/java/org/apache/cassandra/index/sai/plan/Operation.java

@@ -0,0 +1,351 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.plan;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.function.BiFunction;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ArrayListMultimap;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.ListMultimap;
+
+import org.apache.cassandra.cql3.Operator;
+import org.apache.cassandra.db.filter.RowFilter;
+import org.apache.cassandra.index.sai.IndexContext;
+import org.apache.cassandra.index.sai.analyzer.AbstractAnalyzer;
+import org.apache.cassandra.index.sai.utils.KeyRangeIterator;
+import org.apache.cassandra.index.sai.utils.TypeUtil;
+import org.apache.cassandra.schema.ColumnMetadata;
+
+public class Operation
+{
+    public enum BooleanOperator
+    {
+        AND((a, b) -> a & b);
+
+        private final BiFunction<Boolean, Boolean, Boolean> func;
+
+        BooleanOperator(BiFunction<Boolean, Boolean, Boolean> func)
+        {
+            this.func = func;
+        }
+
+        public boolean apply(boolean a, boolean b)
+        {
+            return func.apply(a, b);
+        }
+    }
+
+    @VisibleForTesting
+    protected static ListMultimap<ColumnMetadata, Expression> buildIndexExpressions(QueryController controller,
+                                                                                    BooleanOperator booleanOperator,
+                                                                                    List<RowFilter.Expression> expressions)
+    {
+        ListMultimap<ColumnMetadata, Expression> analyzed = ArrayListMultimap.create();
+
+        // sort all the expressions in the operation by name and priority of the logical operator
+        // this gives us an efficient way to handle inequality and combining into ranges without extra processing
+        // and converting expressions from one type to another.
+        expressions.sort((a, b) -> {
+            int cmp = a.column().compareTo(b.column());
+            return cmp == 0 ? -Integer.compare(getPriority(a.operator()), getPriority(b.operator())) : cmp;
+        });
+
+        for (final RowFilter.Expression e : expressions)
+        {
+            IndexContext indexContext = controller.getContext(e);
+            List<Expression> perColumn = analyzed.get(e.column());
+
+            AbstractAnalyzer analyzer = indexContext.getQueryAnalyzerFactory().create();
+            try
+            {
+                analyzer.reset(e.getIndexValue().duplicate());
+
+                // EQ can have multiple expressions e.g. text = "Hello World",
+                // becomes text = "Hello" OR text = "World" because "space" is always interpreted as a split point (by analyzer),
+                // CONTAINS/CONTAINS_KEY are always treated as multiple expressions since they currently only targetting
+                // collections.
+                boolean isMultiExpression = false;
+                switch (e.operator())
+                {
+                    case EQ:
+                        // EQ operator will always be a multiple expression because it is being used by
+                        // map entries
+                        isMultiExpression = indexContext.isNonFrozenCollection();
+                        break;
+
+                    case CONTAINS:
+                    case CONTAINS_KEY:
+                        isMultiExpression = true;
+                        break;
+                }
+                if (isMultiExpression)
+                {
+                    while (analyzer.hasNext())
+                    {
+                        final ByteBuffer token = analyzer.next();
+                        perColumn.add(new Expression(indexContext).add(e.operator(), token.duplicate()));
+                    }
+                }
+                else
+                // "range" or not-equals operator, combines both bounds together into the single expression,
+                // if operation of the group is AND, otherwise we are forced to create separate expressions,
+                // not-equals is combined with the range iff operator is AND.
+                {
+                    Expression range;
+                    if (perColumn.size() == 0 || booleanOperator != BooleanOperator.AND)
+                    {
+                        range = new Expression(indexContext);
+                        perColumn.add(range);
+                    }
+                    else
+                    {
+                        range = Iterables.getLast(perColumn);
+                    }
+
+                    if (!TypeUtil.isLiteral(indexContext.getValidator()))
+                    {
+                        range.add(e.operator(), e.getIndexValue().duplicate());
+                    }
+                    else
+                    {
+                        while (analyzer.hasNext())
+                        {
+                            ByteBuffer term = analyzer.next();
+                            range.add(e.operator(), term.duplicate());
+                        }
+                    }
+                }
+            }
+            finally
+            {
+                analyzer.end();
+            }
+        }
+
+        return analyzed;
+    }
+
+    private static int getPriority(Operator op)
+    {
+        switch (op)
+        {
+            case EQ:
+            case CONTAINS:
+            case CONTAINS_KEY:
+                return 5;
+
+            case GTE:
+            case GT:
+                return 3;
+
+            case LTE:
+            case LT:
+                return 2;
+
+            default:
+                return 0;
+        }
+    }
+
+    /**
+     * Converts expressions into filter tree for query.
+     *
+     * @return a KeyRangeIterator over the index query results
+     */
+    static KeyRangeIterator buildIterator(QueryController controller)
+    {
+        return Node.buildTree(controller.filterOperation()).analyzeTree(controller).rangeIterator(controller);
+    }
+
+    /**
+     * Converts expressions into filter tree (which is currently just a single AND).
+     *
+     * Filter tree allows us to do a couple of important optimizations
+     * namely, group flattening for AND operations (query rewrite), expression bounds checks,
+     * "satisfies by" checks for resulting rows with an early exit.
+     *
+     * @return root of the filter tree.
+     */
+    static FilterTree buildFilter(QueryController controller)
+    {
+        return Node.buildTree(controller.filterOperation()).buildFilter(controller);
+    }
+
+    static abstract class Node
+    {
+        ListMultimap<ColumnMetadata, Expression> expressionMap;
+
+        boolean canFilter()
+        {
+            return (expressionMap != null && !expressionMap.isEmpty()) || !children().isEmpty();
+        }
+
+        List<Node> children()
+        {
+            return Collections.emptyList();
+        }
+
+        void add(Node child)
+        {
+            throw new UnsupportedOperationException();
+        }
+
+        RowFilter.Expression expression()
+        {
+            throw new UnsupportedOperationException();
+        }
+
+        abstract void analyze(List<RowFilter.Expression> expressionList, QueryController controller);
+
+        abstract FilterTree filterTree();
+
+        abstract KeyRangeIterator rangeIterator(QueryController controller);
+
+        static Node buildTree(RowFilter filterOperation)
+        {
+            OperatorNode node = new AndNode();
+            for (RowFilter.Expression expression : filterOperation.getExpressions())
+                node.add(buildExpression(expression));
+            return node;
+        }
+
+        static Node buildExpression(RowFilter.Expression expression)
+        {
+            return new ExpressionNode(expression);
+        }
+
+        Node analyzeTree(QueryController controller)
+        {
+            List<RowFilter.Expression> expressionList = new ArrayList<>();
+            doTreeAnalysis(this, expressionList, controller);
+            if (!expressionList.isEmpty())
+                this.analyze(expressionList, controller);
+            return this;
+        }
+
+        void doTreeAnalysis(Node node, List<RowFilter.Expression> expressions, QueryController controller)
+        {
+            if (node.children().isEmpty())
+                expressions.add(node.expression());
+            else
+            {
+                List<RowFilter.Expression> expressionList = new ArrayList<>();
+                for (Node child : node.children())
+                    doTreeAnalysis(child, expressionList, controller);
+                node.analyze(expressionList, controller);
+            }
+        }
+
+        FilterTree buildFilter(QueryController controller)
+        {
+            analyzeTree(controller);
+            FilterTree tree = filterTree();
+            for (Node child : children())
+                if (child.canFilter())
+                    tree.addChild(child.buildFilter(controller));
+            return tree;
+        }
+    }
+
+    static abstract class OperatorNode extends Node
+    {
+        final List<Node> children = new ArrayList<>();
+
+        @Override
+        public List<Node> children()
+        {
+            return children;
+        }
+
+        @Override
+        public void add(Node child)
+        {
+            children.add(child);
+        }
+    }
+
+    static class AndNode extends OperatorNode
+    {
+        @Override
+        public void analyze(List<RowFilter.Expression> expressionList, QueryController controller)
+        {
+            expressionMap = buildIndexExpressions(controller, BooleanOperator.AND, expressionList);
+        }
+
+        @Override
+        FilterTree filterTree()
+        {
+            return new FilterTree(BooleanOperator.AND, expressionMap);
+        }
+
+        @Override
+        KeyRangeIterator rangeIterator(QueryController controller)
+        {
+            KeyRangeIterator.Builder builder = controller.getIndexQueryResults(expressionMap.values());
+            for (Node child : children)
+            {
+                boolean canFilter = child.canFilter();
+                if (canFilter)
+                    builder.add(child.rangeIterator(controller));
+            }
+            return builder.build();
+        }
+    }
+
+    static class ExpressionNode extends Node
+    {
+        final RowFilter.Expression expression;
+
+        @Override
+        public void analyze(List<RowFilter.Expression> expressionList, QueryController controller)
+        {
+            expressionMap = buildIndexExpressions(controller, BooleanOperator.AND, expressionList);
+        }
+
+        @Override
+        FilterTree filterTree()
+        {
+            return new FilterTree(BooleanOperator.AND, expressionMap);
+        }
+
+        public ExpressionNode(RowFilter.Expression expression)
+        {
+            this.expression = expression;
+        }
+
+        @Override
+        public RowFilter.Expression expression()
+        {
+            return expression;
+        }
+
+        @Override
+        KeyRangeIterator rangeIterator(QueryController controller)
+        {
+            assert canFilter() : "Cannot process query with no expressions";
+
+            return controller.getIndexQueryResults(expressionMap.values()).build();
+        }
+    }
+}

src/java/org/apache/cassandra/index/sai/plan/QueryController.java

@@ -0,0 +1,246 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.plan;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.Set;
+
+import com.google.common.collect.Lists;
+
+import org.apache.cassandra.cql3.statements.schema.IndexTarget;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.DataRange;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.PartitionPosition;
+import org.apache.cassandra.db.PartitionRangeReadCommand;
+import org.apache.cassandra.db.ReadCommand;
+import org.apache.cassandra.db.ReadExecutionController;
+import org.apache.cassandra.db.SinglePartitionReadCommand;
+import org.apache.cassandra.db.filter.ClusteringIndexFilter;
+import org.apache.cassandra.db.filter.ClusteringIndexNamesFilter;
+import org.apache.cassandra.db.filter.DataLimits;
+import org.apache.cassandra.db.filter.RowFilter;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.dht.AbstractBounds;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.index.sai.IndexContext;
+import org.apache.cassandra.index.sai.QueryContext;
+import org.apache.cassandra.index.sai.StorageAttachedIndex;
+import org.apache.cassandra.index.sai.metrics.TableQueryMetrics;
+import org.apache.cassandra.index.sai.utils.PrimaryKey;
+import org.apache.cassandra.index.sai.utils.KeyRangeIntersectionIterator;
+import org.apache.cassandra.index.sai.utils.KeyRangeIterator;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.utils.FBUtilities;
+
+public class QueryController
+{
+    private final ColumnFamilyStore cfs;
+    private final ReadCommand command;
+    private final QueryContext queryContext;
+    private final TableQueryMetrics tableQueryMetrics;
+    private final RowFilter filterOperation;
+    private final List<DataRange> ranges;
+    private final AbstractBounds<PartitionPosition> mergeRange;
+
+    public QueryController(ColumnFamilyStore cfs,
+                           ReadCommand command,
+                           RowFilter filterOperation,
+                           QueryContext queryContext,
+                           TableQueryMetrics tableQueryMetrics)
+    {
+        this.cfs = cfs;
+        this.command = command;
+        this.queryContext = queryContext;
+        this.tableQueryMetrics = tableQueryMetrics;
+        this.filterOperation = filterOperation;
+        this.ranges = dataRanges(command);
+        DataRange first = ranges.get(0);
+        DataRange last = ranges.get(ranges.size() - 1);
+        this.mergeRange = ranges.size() == 1 ? first.keyRange() : first.keyRange().withNewRight(last.keyRange().right);
+    }
+
+    public TableMetadata metadata()
+    {
+        return command.metadata();
+    }
+
+    RowFilter filterOperation()
+    {
+        return this.filterOperation;
+    }
+
+    /**
+     * @return token ranges used in the read command
+     */
+    List<DataRange> dataRanges()
+    {
+        return ranges;
+    }
+
+    /**
+     * Note: merged range may contain subrange that no longer belongs to the local node after range movement.
+     * It should only be used as an optimization to reduce search space. Use {@link #dataRanges()} instead to filter data.
+     *
+     * @return merged token range
+     */
+    AbstractBounds<PartitionPosition> mergeRange()
+    {
+        return mergeRange;
+    }
+
+    /**
+     * @return indexed {@code IndexContext} if index is found; otherwise return non-indexed {@code IndexContext}.
+     */
+    public IndexContext getContext(RowFilter.Expression expression)
+    {
+        Set<StorageAttachedIndex> indexes = cfs.indexManager.getBestIndexFor(expression, StorageAttachedIndex.class);
+
+        return indexes.isEmpty() ? new IndexContext(cfs.metadata().keyspace,
+                                                    cfs.metadata().name,
+                                                    cfs.metadata().partitionKeyType,
+                                                    cfs.metadata().comparator,
+                                                    expression.column(),
+                                                    IndexTarget.Type.VALUES,
+                                                    null)
+                                 : indexes.iterator().next().getIndexContext();
+    }
+
+    public UnfilteredRowIterator queryStorage(PrimaryKey key, ReadExecutionController executionController)
+    {
+        if (key == null)
+            throw new IllegalArgumentException("non-null key required");
+
+        try
+        {
+            SinglePartitionReadCommand partition = SinglePartitionReadCommand.create(cfs.metadata(),
+                                                                                     command.nowInSec(),
+                                                                                     command.columnFilter(),
+                                                                                     RowFilter.none(),
+                                                                                     DataLimits.NONE,
+                                                                                     key.partitionKey(),
+                                                                                     makeFilter(key));
+
+            return partition.queryMemtableAndDisk(cfs, executionController);
+        }
+        finally
+        {
+            queryContext.checkpoint();
+        }
+    }
+
+    /**
+     * Build a {@link KeyRangeIterator.Builder} from the given list of expressions by applying given operation (AND).
+     * Building of such builder involves index search, results of which are persisted in the internal resources list
+     *
+     * @param expressions The expressions to build range iterator from (expressions with not results are ignored).
+     *
+     * @return range iterator builder based on given expressions and operation type.
+     */
+    public KeyRangeIterator.Builder getIndexQueryResults(Collection<Expression> expressions)
+    {
+        KeyRangeIterator.Builder builder = KeyRangeIntersectionIterator.builder(expressions.size());
+
+        try
+        {
+            for (Expression e : expressions)
+            {
+                if (e.context.isIndexed())
+                {
+                    @SuppressWarnings({"resource", "RedundantSuppression"}) // RangeIterators are closed by releaseIndexes
+                    KeyRangeIterator memtableIterator = e.context.searchMemtableIndexes(e, mergeRange);
+
+                    builder.add(memtableIterator);
+                }
+            }
+        }
+        catch (Throwable t)
+        {
+            // all sstable indexes in view have been referenced, need to clean up when exception is thrown
+            builder.cleanup();
+            throw t;
+        }
+        return builder;
+    }
+
+    /**
+     * Returns whether this query is not selecting the {@link PrimaryKey}.
+     * The query does not select the key if both of the following statements are false:
+     *  1. The table associated with the query is not using clustering keys
+     *  2. The clustering index filter for the command wants the row.
+     *
+     *  Item 2 is important in paged queries where the {@link org.apache.cassandra.db.filter.ClusteringIndexSliceFilter} for
+     *  subsequent paged queries may not select rows that are returned by the index
+     *  search because that is initially partition based.
+     *
+     * @param key The {@link PrimaryKey} to be tested
+     * @return true if the key is not selected by the query
+     */
+    public boolean doesNotSelect(PrimaryKey key)
+    {
+        return !key.hasEmptyClustering() && !command.clusteringIndexFilter(key.partitionKey()).selects(key.clustering());
+    }
+
+    // Note: This method assumes that the selects method has already been called for the
+    // key to avoid having to (potentially) call selects twice
+    private ClusteringIndexFilter makeFilter(PrimaryKey key)
+    {
+        ClusteringIndexFilter clusteringIndexFilter = command.clusteringIndexFilter(key.partitionKey());
+
+        if (key.hasEmptyClustering())
+            return clusteringIndexFilter;
+        else
+            return new ClusteringIndexNamesFilter(FBUtilities.singleton(key.clustering(), cfs.metadata().comparator),
+                                                  clusteringIndexFilter.isReversed());
+    }
+
+    /**
+     * Used to release all resources and record metrics when query finishes.
+     */
+    public void finish()
+    {
+        if (tableQueryMetrics != null) tableQueryMetrics.record(queryContext);
+    }
+
+    /**
+     * Returns the {@link DataRange} list covered by the specified {@link ReadCommand}.
+     *
+     * @param command a read command
+     * @return the data ranges covered by {@code command}
+     */
+    private static List<DataRange> dataRanges(ReadCommand command)
+    {
+        if (command instanceof SinglePartitionReadCommand)
+        {
+            SinglePartitionReadCommand cmd = (SinglePartitionReadCommand) command;
+            DecoratedKey key = cmd.partitionKey();
+            return Lists.newArrayList(new DataRange(new Range<>(key, key), cmd.clusteringIndexFilter()));
+        }
+        else if (command instanceof PartitionRangeReadCommand)
+        {
+            PartitionRangeReadCommand cmd = (PartitionRangeReadCommand) command;
+            return Lists.newArrayList(cmd.dataRange());
+        }
+        else
+        {
+            throw new AssertionError("Unsupported read command type: " + command.getClass().getName());
+        }
+    }
+}

src/java/org/apache/cassandra/index/sai/plan/StorageAttachedIndexQueryPlan.java

@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.plan;
+
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import javax.annotation.Nullable;
+
+import com.google.common.collect.ImmutableSet;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.ReadCommand;
+import org.apache.cassandra.db.filter.RowFilter;
+import org.apache.cassandra.index.Index;
+import org.apache.cassandra.index.sai.StorageAttachedIndex;
+import org.apache.cassandra.index.sai.metrics.TableQueryMetrics;
+import org.apache.cassandra.schema.TableMetadata;
+
+public class StorageAttachedIndexQueryPlan implements Index.QueryPlan
+{
+    private final ColumnFamilyStore cfs;
+    private final TableQueryMetrics queryMetrics;
+    private final RowFilter postIndexFilter;
+    private final RowFilter filterOperation;
+    private final Set<Index> indexes;
+
+    private StorageAttachedIndexQueryPlan(ColumnFamilyStore cfs,
+                                          TableQueryMetrics queryMetrics,
+                                          RowFilter postIndexFilter,
+                                          RowFilter filterOperation,
+                                          ImmutableSet<Index> indexes)
+    {
+        this.cfs = cfs;
+        this.queryMetrics = queryMetrics;
+        this.postIndexFilter = postIndexFilter;
+        this.filterOperation = filterOperation;
+        this.indexes = indexes;
+    }
+
+    @Nullable
+    public static StorageAttachedIndexQueryPlan create(ColumnFamilyStore cfs,
+                                                       TableQueryMetrics queryMetrics,
+                                                       Set<Index> indexes,
+                                                       RowFilter rowFilter)
+    {
+        ImmutableSet.Builder<Index> selectedIndexesBuilder = ImmutableSet.builder();
+
+        for (RowFilter.Expression expression : rowFilter)
+        {
+            // we ignore user-defined expressions here because we don't have a way to translate their #isSatifiedBy
+            // method, they will be included in the filter returned by QueryPlan#postIndexQueryFilter()
+            if (expression.isUserDefined())
+                continue;
+
+            for (Index index : indexes)
+            {
+                if (index.supportsExpression(expression.column(), expression.operator()))
+                {
+                    selectedIndexesBuilder.add(index);
+                }
+            }
+        }
+
+        ImmutableSet<Index> selectedIndexes = selectedIndexesBuilder.build();
+        if (selectedIndexes.isEmpty())
+            return null;
+
+        /*
+         * postIndexFilter comprised by those expressions in the read command row filter that can't be handled by
+         * {@link FilterTree#satisfiedBy(Unfiltered, Row, boolean)}. This includes expressions targeted
+         * at {@link RowFilter.UserExpression}s.
+         */
+        RowFilter postIndexFilter = rowFilter.restrict(RowFilter.Expression::isUserDefined);
+        return new StorageAttachedIndexQueryPlan(cfs, queryMetrics, postIndexFilter, rowFilter, selectedIndexes);
+    }
+
+    @Override
+    public Set<Index> getIndexes()
+    {
+        return indexes;
+    }
+
+    @Override
+    public long getEstimatedResultRows()
+    {
+        // this is temporary (until proper QueryPlan is integrated into Cassandra)
+        // and allows us to priority storage-attached indexes if any in the query since they
+        // are going to be more efficient, to query and intersect, than built-in indexes.
+        return Long.MIN_VALUE;
+    }
+
+    @Override
+    public boolean shouldEstimateInitialConcurrency()
+    {
+        return false;
+    }
+
+    @Override
+    public Index.Searcher searcherFor(ReadCommand command)
+    {
+        return new StorageAttachedIndexSearcher(cfs,
+                                                queryMetrics,
+                                                command,
+                                                filterOperation,
+                                                DatabaseDescriptor.getRangeRpcTimeout(TimeUnit.MILLISECONDS));
+    }
+
+    /**
+     * @return a filter with all the expressions that are user-defined or for a non-indexed partition key column
+     *
+     * (currently index on partition columns is not supported, see {@link StorageAttachedIndex#validateOptions(Map, TableMetadata)})
+     */
+    @Override
+    public RowFilter postIndexQueryFilter()
+    {
+        return postIndexFilter;
+    }
+}

src/java/org/apache/cassandra/index/sai/plan/StorageAttachedIndexSearcher.java

@@ -0,0 +1,566 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.plan;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.NoSuchElementException;
+import java.util.function.Supplier;
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+
+import com.google.common.collect.Iterators;
+
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.DataRange;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.PartitionPosition;
+import org.apache.cassandra.db.ReadCommand;
+import org.apache.cassandra.db.ReadExecutionController;
+import org.apache.cassandra.db.RegularAndStaticColumns;
+import org.apache.cassandra.db.filter.RowFilter;
+import org.apache.cassandra.db.partitions.PartitionIterator;
+import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
+import org.apache.cassandra.db.rows.AbstractUnfilteredRowIterator;
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.db.rows.RowIterator;
+import org.apache.cassandra.db.rows.Unfiltered;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.dht.AbstractBounds;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.exceptions.RequestTimeoutException;
+import org.apache.cassandra.index.Index;
+import org.apache.cassandra.index.sai.QueryContext;
+import org.apache.cassandra.index.sai.analyzer.AbstractAnalyzer;
+import org.apache.cassandra.index.sai.metrics.TableQueryMetrics;
+import org.apache.cassandra.index.sai.utils.KeyRangeIterator;
+import org.apache.cassandra.index.sai.utils.PrimaryKey;
+import org.apache.cassandra.index.sai.utils.PrimaryKeyFactory;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.utils.AbstractIterator;
+
+public class StorageAttachedIndexSearcher implements Index.Searcher
+{
+    private final ReadCommand command;
+    private final QueryController queryController;
+    private final QueryContext queryContext;
+    private final PrimaryKeyFactory keyFactory;
+
+    public StorageAttachedIndexSearcher(ColumnFamilyStore cfs,
+                                        TableQueryMetrics tableQueryMetrics,
+                                        ReadCommand command,
+                                        RowFilter filterOperation,
+                                        long executionQuotaMs)
+    {
+        this.command = command;
+        this.queryContext = new QueryContext(command, executionQuotaMs);
+        this.queryController = new QueryController(cfs, command, filterOperation, queryContext, tableQueryMetrics);
+        this.keyFactory = PrimaryKey.factory(cfs.metadata().comparator);
+    }
+
+    @Override
+    public ReadCommand command()
+    {
+        return command;
+    }
+
+    @Override
+    public PartitionIterator filterReplicaFilteringProtection(PartitionIterator fullResponse)
+    {
+        for (RowFilter.Expression expression : queryController.filterOperation())
+        {
+            AbstractAnalyzer analyzer = queryController.getContext(expression).getIndexAnalyzerFactory().create();
+            try
+            {
+                if (analyzer.transformValue())
+                    return applyIndexFilter(fullResponse, Operation.buildFilter(queryController), queryContext);
+            }
+            finally
+            {
+                analyzer.end();
+            }
+        }
+
+        // if no analyzer does transformation
+        return Index.Searcher.super.filterReplicaFilteringProtection(fullResponse);
+    }
+
+    @Override
+    public UnfilteredPartitionIterator search(ReadExecutionController executionController) throws RequestTimeoutException
+    {
+        return new ResultRetriever(queryController, executionController, queryContext, keyFactory);
+    }
+
+    private static class ResultRetriever extends AbstractIterator<UnfilteredRowIterator> implements UnfilteredPartitionIterator
+    {
+        private final PrimaryKey firstPrimaryKey;
+        private final PrimaryKey lastPrimaryKey;
+        private final Iterator<DataRange> keyRanges;
+        private AbstractBounds<PartitionPosition> currentKeyRange;
+
+        private final KeyRangeIterator resultKeyIterator;
+        private final FilterTree filterTree;
+        private final QueryController queryController;
+        private final ReadExecutionController executionController;
+        private final QueryContext queryContext;
+        private final PrimaryKeyFactory keyFactory;
+        private PrimaryKey lastKey;
+
+        private ResultRetriever(QueryController queryController,
+                                ReadExecutionController executionController,
+                                QueryContext queryContext,
+                                PrimaryKeyFactory keyFactory)
+        {
+            this.keyRanges = queryController.dataRanges().iterator();
+            this.currentKeyRange = keyRanges.next().keyRange();
+
+            this.resultKeyIterator = Operation.buildIterator(queryController);
+            this.filterTree = Operation.buildFilter(queryController);
+            this.queryController = queryController;
+            this.executionController = executionController;
+            this.queryContext = queryContext;
+            this.keyFactory = keyFactory;
+
+            this.firstPrimaryKey = keyFactory.createTokenOnly(queryController.mergeRange().left.getToken());
+            this.lastPrimaryKey = keyFactory.createTokenOnly(queryController.mergeRange().right.getToken());
+        }
+
+        @Override
+        @SuppressWarnings({"resource", "RedundantSuppression"}) // The iterator produced here has a nop close operation
+        public UnfilteredRowIterator computeNext()
+        {
+            // IMPORTANT: The correctness of the entire query pipeline relies on the fact that we consume a token
+            // and materialize its keys before moving on to the next token in the flow. This sequence must not be broken
+            // with toList() or similar. (Both the union and intersection flow constructs, to avoid excessive object
+            // allocation, reuse their token mergers as they process individual positions on the ring.)
+
+            if (resultKeyIterator == null)
+                return endOfData();
+
+            // If being called for the first time, skip to the beginning of the range.
+            // We can't put this code in the constructor because it may throw and the caller
+            // may not be prepared for that.
+            if (lastKey == null)
+                resultKeyIterator.skipTo(firstPrimaryKey);
+
+            // Theoretically we wouldn't need this if the caller of computeNext always ran the
+            // returned iterators to the completion. Unfortunately, we have no control over the caller behavior here.
+            // Hence, we skip to the next partition in order to comply to the unwritten partition iterator contract
+            // saying this iterator must not return the same partition twice.
+            skipToNextPartition();
+
+            UnfilteredRowIterator iterator = nextRowIterator(this::nextSelectedKeyInRange);
+            return iterator != null
+                   ? iteratePartition(iterator)
+                   : endOfData();
+        }
+
+        /**
+         * Tries to obtain a row iterator for one of the supplied keys by repeatedly calling
+         * {@link ResultRetriever#queryStorageAndFilter} until it gives a non-null result.
+         * The keySupplier should return the next key with every call to get() and
+         * null when there are no more keys to try.
+         *
+         * @return an iterator or null if all keys were tried with no success
+         */
+        private @Nullable UnfilteredRowIterator nextRowIterator(@Nonnull Supplier<PrimaryKey> keySupplier)
+        {
+            UnfilteredRowIterator iterator = null;
+            while (iterator == null)
+            {
+                PrimaryKey key = keySupplier.get();
+                if (key == null)
+                    return null;
+                iterator = queryStorageAndFilter(key);
+            }
+            return iterator;
+        }
+
+        /**
+         * Returns the next available key contained by one of the keyRanges.
+         * If the next key falls out of the current key range, it skips to the next key range, and so on.
+         * If no more keys or no more ranges are available, returns null.
+         */
+        private @Nullable PrimaryKey nextKeyInRange()
+        {
+            PrimaryKey key = nextKey();
+
+            while (key != null && !(currentKeyRange.contains(key.partitionKey())))
+            {
+                if (!currentKeyRange.right.isMinimum() && currentKeyRange.right.compareTo(key.partitionKey()) <= 0)
+                {
+                    // currentKeyRange before the currentKey so need to move currentKeyRange forward
+                    currentKeyRange = nextKeyRange();
+                    if (currentKeyRange == null)
+                        return null;
+                }
+                else
+                {
+                    // key either before the current range, so let's move the key forward
+                    skipTo(currentKeyRange.left.getToken());
+                    key = nextKey();
+                }
+            }
+            return key;
+        }
+
+        /**
+         * Returns the next available key contained by one of the keyRanges and selected by the queryController.
+         * If the next key falls out of the current key range, it skips to the next key range, and so on.
+         * If no more keys acceptd by the controller are available, returns null.
+         */
+        private @Nullable PrimaryKey nextSelectedKeyInRange()
+        {
+            PrimaryKey key;
+            do
+            {
+                key = nextKeyInRange();
+            }
+            while (key != null && queryController.doesNotSelect(key));
+            return key;
+        }
+
+        /**
+         * Retrieves the next primary key that belongs to the given partition and is selected by the query controller.
+         * The underlying key iterator is advanced only if the key belongs to the same partition.
+         * <p>
+         * Returns null if:
+         * <ul>
+         *   <li>there are no more keys</li>
+         *   <li>the next key is beyond the upper bound</li>
+         *   <li>the next key belongs to a different partition</li>
+         * </ul>
+         * </p>
+         */
+        private @Nullable PrimaryKey nextSelectedKeyInPartition(DecoratedKey partitionKey)
+        {
+            PrimaryKey key;
+            do
+            {
+                if (!resultKeyIterator.hasNext())
+                    return null;
+                if (!resultKeyIterator.peek().partitionKey().equals(partitionKey))
+                    return null;
+
+                key = nextKey();
+            }
+            while (key != null && queryController.doesNotSelect(key));
+            return key;
+        }
+
+        /**
+         * Gets the next key from the underlying operation.
+         * Returns null if there are no more keys <= lastPrimaryKey.
+         */
+        private @Nullable PrimaryKey nextKey()
+        {
+            if (!resultKeyIterator.hasNext())
+                return null;
+            PrimaryKey key = resultKeyIterator.next();
+            return isWithinUpperBound(key) ? key : null;
+        }
+
+        /**
+         * Returns true if the key is not greater than lastPrimaryKey
+         */
+        private boolean isWithinUpperBound(PrimaryKey key)
+        {
+            return lastPrimaryKey.token().isMinimum() || lastPrimaryKey.compareTo(key) >= 0;
+        }
+
+        /**
+         * Gets the next key range from the underlying range iterator.
+         */
+        private @Nullable AbstractBounds<PartitionPosition> nextKeyRange()
+        {
+            return keyRanges.hasNext() ? keyRanges.next().keyRange() : null;
+        }
+
+        /**
+         * Convenience function to skip to a given token.
+         */
+        private void skipTo(@Nonnull Token token)
+        {
+            resultKeyIterator.skipTo(keyFactory.createTokenOnly(token));
+        }
+
+        /**
+         * Skips to the key that belongs to a different partition than the last key we fetched.
+         */
+        private void skipToNextPartition()
+        {
+            if (lastKey == null)
+                return;
+            DecoratedKey lastPartitionKey = lastKey.partitionKey();
+            while (resultKeyIterator.hasNext() && resultKeyIterator.peek().partitionKey().equals(lastPartitionKey))
+                resultKeyIterator.next();
+        }
+
+
+        /**
+         * Returns an iterator over the rows in the partition associated with the given iterator.
+         * Initially, it retrieves the rows from the given iterator until it runs out of data.
+         * Then it iterates the primary keys obtained from the index until the end of the partition
+         * and lazily constructs new row itertors for each of the keys. At a given time, only one row iterator is open.
+         *
+         * The rows are retrieved in the order of primary keys provided by the underlying index.
+         * The iterator is complete when the next key to be fetched belongs to different partition
+         * (but the iterator does not consume that key).
+         *
+         * @param startIter an iterator positioned at the first row in the partition that we want to return
+         */
+        private @Nonnull UnfilteredRowIterator iteratePartition(@Nonnull UnfilteredRowIterator startIter)
+        {
+            return new AbstractUnfilteredRowIterator(startIter.metadata(),
+                                                     startIter.partitionKey(),
+                                                     startIter.partitionLevelDeletion(),
+                                                     startIter.columns(),
+                                                     startIter.staticRow(),
+                                                     startIter.isReverseOrder(),
+                                                     startIter.stats())
+            {
+                private UnfilteredRowIterator currentIter = startIter;
+                private final DecoratedKey partitionKey = startIter.partitionKey();
+
+                @Override
+                protected Unfiltered computeNext()
+                {
+                    while (!currentIter.hasNext())
+                    {
+                        currentIter.close();
+                        currentIter = nextRowIterator(() -> nextSelectedKeyInPartition(partitionKey));
+                        if (currentIter == null)
+                            return endOfData();
+                    }
+                    return currentIter.next();
+                }
+
+                @Override
+                public void close()
+                {
+                    FileUtils.closeQuietly(currentIter);
+                    super.close();
+                }
+            };
+        }
+
+        private UnfilteredRowIterator queryStorageAndFilter(PrimaryKey key)
+        {
+            // Key reads are lazy, delayed all the way to this point. Skip if we've already seen this one:
+            if (key.equals(lastKey))
+                return null;
+
+            lastKey = key;
+
+            try (UnfilteredRowIterator partition = queryController.queryStorage(key, executionController))
+            {
+                queryContext.partitionsRead++;
+
+                return applyIndexFilter(partition, filterTree, queryContext);
+            }
+        }
+
+        private static UnfilteredRowIterator applyIndexFilter(UnfilteredRowIterator partition, FilterTree tree, QueryContext queryContext)
+        {
+            Row staticRow = partition.staticRow();
+            List<Unfiltered> clusters = new ArrayList<>();
+
+            while (partition.hasNext())
+            {
+                Unfiltered row = partition.next();
+
+                queryContext.rowsFiltered++;
+                if (tree.isSatisfiedBy(partition.partitionKey(), row, staticRow))
+                {
+                    clusters.add(row);
+                }
+            }
+
+            if (clusters.isEmpty())
+            {
+                queryContext.rowsFiltered++;
+                if (tree.isSatisfiedBy(partition.partitionKey(), staticRow, staticRow))
+                {
+                    clusters.add(staticRow);
+                }
+            }
+
+            /*
+             * If {@code clusters} is empty, which means either all clustering row and static row pairs failed,
+             *       or static row and static row pair failed. In both cases, we should not return any partition.
+             * If {@code clusters} is not empty, which means either there are some clustering row and static row pairs match the filters,
+             *       or static row and static row pair matches the filters. In both cases, we should return a partition with static row,
+             *       and remove the static row marker from the {@code clusters} for the latter case.
+             */
+            if (clusters.isEmpty())
+            {
+                return null;
+            }
+
+            return new PartitionIterator(partition, staticRow, Iterators.filter(clusters.iterator(), u -> !((Row)u).isStatic()));
+        }
+
+        private static class PartitionIterator extends AbstractUnfilteredRowIterator
+        {
+            private final Iterator<Unfiltered> rows;
+
+            public PartitionIterator(UnfilteredRowIterator partition, Row staticRow, Iterator<Unfiltered> content)
+            {
+                super(partition.metadata(),
+                      partition.partitionKey(),
+                      partition.partitionLevelDeletion(),
+                      partition.columns(),
+                      staticRow,
+                      partition.isReverseOrder(),
+                      partition.stats());
+
+                rows = content;
+            }
+
+            @Override
+            protected Unfiltered computeNext()
+            {
+                return rows.hasNext() ? rows.next() : endOfData();
+            }
+        }
+
+        @Override
+        public TableMetadata metadata()
+        {
+            return queryController.metadata();
+        }
+
+        @Override
+        public void close()
+        {
+            FileUtils.closeQuietly(resultKeyIterator);
+            queryController.finish();
+        }
+    }
+
+    /**
+     * Used by {@link StorageAttachedIndexSearcher#filterReplicaFilteringProtection} to filter rows for columns that
+     * have transformations so won't get handled correctly by the row filter.
+     */
+    @SuppressWarnings({"resource", "RedundantSuppression"})
+    private static PartitionIterator applyIndexFilter(PartitionIterator response, FilterTree tree, QueryContext queryContext)
+    {
+        return new PartitionIterator()
+        {
+            @Override
+            public void close()
+            {
+                response.close();
+            }
+
+            @Override
+            public boolean hasNext()
+            {
+                return response.hasNext();
+            }
+
+            @Override
+            public RowIterator next()
+            {
+                RowIterator delegate = response.next();
+                Row staticRow = delegate.staticRow();
+
+                return new RowIterator()
+                {
+                    Row next;
+
+                    @Override
+                    public TableMetadata metadata()
+                    {
+                        return delegate.metadata();
+                    }
+
+                    @Override
+                    public boolean isReverseOrder()
+                    {
+                        return delegate.isReverseOrder();
+                    }
+
+                    @Override
+                    public RegularAndStaticColumns columns()
+                    {
+                        return delegate.columns();
+                    }
+
+                    @Override
+                    public DecoratedKey partitionKey()
+                    {
+                        return delegate.partitionKey();
+                    }
+
+                    @Override
+                    public Row staticRow()
+                    {
+                        return staticRow;
+                    }
+
+                    @Override
+                    public void close()
+                    {
+                        delegate.close();
+                    }
+
+                    private Row computeNext()
+                    {
+                        while (delegate.hasNext())
+                        {
+                            Row row = delegate.next();
+                            queryContext.rowsFiltered++;
+                            if (tree.isSatisfiedBy(delegate.partitionKey(), row, staticRow))
+                                return row;
+                        }
+                        return null;
+                    }
+
+                    private Row loadNext()
+                    {
+                        if (next == null)
+                            next = computeNext();
+                        return next;
+                    }
+
+                    @Override
+                    public boolean hasNext()
+                    {
+                        return loadNext() != null;
+                    }
+
+                    @Override
+                    public Row next()
+                    {
+                        Row result = loadNext();
+                        next = null;
+
+                        if (result == null)
+                            throw new NoSuchElementException();
+
+                        return result;
+                    }
+                };
+            }
+        };
+    }
+}

src/java/org/apache/cassandra/index/sai/utils/KeyRangeConcatIterator.java

@@ -0,0 +1,181 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.utils;
+
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import java.util.PriorityQueue;
+
+import com.google.common.annotations.VisibleForTesting;
+
+import org.apache.cassandra.io.util.FileUtils;
+
+/**
+ * {@link KeyRangeConcatIterator} takes a list of sorted range iterator and concatenates them, leaving duplicates in
+ * place, to produce a new stably sorted iterator. Duplicates are eliminated later in
+ * {@link org.apache.cassandra.index.sai.plan.StorageAttachedIndexSearcher}
+ * as results from multiple SSTable indexes and their respective segments are consumed.
+ *
+ * ex. (1, 2, 3) + (3, 3, 4, 5) -> (1, 2, 3, 3, 3, 4, 5)
+ * ex. (1, 2, 2, 3) + (3, 4, 4, 6, 6, 7) -> (1, 2, 2, 3, 3, 4, 4, 6, 6, 7)
+ *
+ * TODO Investigate removing the use of PriorityQueue from this class <a href="https://issues.apache.org/jira/browse/CASSANDRA-18165">CASSANDRA-18165</a>
+ */
+public class KeyRangeConcatIterator extends KeyRangeIterator
+{
+    private final PriorityQueue<KeyRangeIterator> ranges;
+    private final List<KeyRangeIterator> toRelease;
+
+    protected KeyRangeConcatIterator(KeyRangeIterator.Builder.Statistics statistics, PriorityQueue<KeyRangeIterator> ranges)
+    {
+        super(statistics);
+
+        this.ranges = ranges;
+        this.toRelease = new ArrayList<>(ranges);
+    }
+
+    @Override
+    @SuppressWarnings({"resource", "RedundantSuppression"})
+    protected void performSkipTo(PrimaryKey nextKey)
+    {
+        while (!ranges.isEmpty())
+        {
+            if (ranges.peek().getCurrent().compareTo(nextKey) >= 0)
+                break;
+
+            KeyRangeIterator head = ranges.poll();
+
+            if (head.getMaximum().compareTo(nextKey) >= 0)
+            {
+                head.skipTo(nextKey);
+                ranges.add(head);
+                break;
+            }
+        }
+    }
+
+    @Override
+    public void close()
+    {
+        // due to lazy key fetching, we cannot close iterator immediately
+        FileUtils.closeQuietly(toRelease);
+    }
+
+    @Override
+    @SuppressWarnings({"resource", "RedundantSuppression"})
+    protected PrimaryKey computeNext()
+    {
+        while (!ranges.isEmpty())
+        {
+            KeyRangeIterator current = ranges.poll();
+            if (current.hasNext())
+            {
+                PrimaryKey next = current.next();
+                // hasNext will update RangeIterator's current which is used to sort in PQ
+                if (current.hasNext())
+                    ranges.add(current);
+
+                return next;
+            }
+        }
+
+        return endOfData();
+    }
+
+    public static Builder builder(int size)
+    {
+        return new Builder(size);
+    }
+
+    public static KeyRangeIterator build(List<KeyRangeIterator> keys)
+    {
+        return new Builder(keys.size()).add(keys).build();
+    }
+
+    @VisibleForTesting
+    public static class Builder extends KeyRangeIterator.Builder
+    {
+        private final PriorityQueue<KeyRangeIterator> ranges;
+
+        Builder(int size)
+        {
+            super(new ConcatStatistics());
+            ranges = new PriorityQueue<>(size, Comparator.comparing(KeyRangeIterator::getCurrent));
+        }
+
+        @Override
+        public KeyRangeIterator.Builder add(KeyRangeIterator range)
+        {
+            if (range == null)
+                return this;
+
+            if (range.getCount() > 0)
+                ranges.add(range);
+            else
+                FileUtils.closeQuietly(range);
+            statistics.update(range);
+
+            return this;
+        }
+
+        @Override
+        public int rangeCount()
+        {
+            return ranges.size();
+        }
+
+        @Override
+        public void cleanup()
+        {
+            FileUtils.closeQuietly(ranges);
+        }
+
+        @Override
+        protected KeyRangeIterator buildIterator()
+        {
+            if (rangeCount() == 1)
+                return ranges.poll();
+
+            return new KeyRangeConcatIterator(statistics, ranges);
+        }
+    }
+
+    private static class ConcatStatistics extends KeyRangeIterator.Builder.Statistics
+    {
+        @Override
+        public void update(KeyRangeIterator range)
+        {
+            // range iterators should be sorted, but previous max must not be greater than next min.
+            if (range.getCount() > 0)
+            {
+                if (count == 0)
+                {
+                    min = range.getMinimum();
+                }
+                else if (count > 0 && max.compareTo(range.getMinimum()) > 0)
+                {
+                    throw new IllegalArgumentException("RangeIterator must be sorted, previous max: " + max + ", next min: " + range.getMinimum());
+                }
+
+                max = range.getMaximum();
+                count += range.getCount();
+            }
+        }
+    }
+}

src/java/org/apache/cassandra/index/sai/utils/KeyRangeIntersectionIterator.java

@@ -0,0 +1,286 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.utils;
+
+import java.lang.invoke.MethodHandles;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.config.CassandraRelevantProperties;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.tracing.Tracing;
+
+/**
+ * A simple intersection iterator that makes no real attempts at optimising the iteration apart from
+ * initially sorting the ranges. This implementation also supports an intersection limit which limits
+ * the number of ranges that will be included in the intersection. This currently defaults to 2.
+ */
+@SuppressWarnings({"resource", "RedundantSuppression"})
+public class KeyRangeIntersectionIterator extends KeyRangeIterator
+{
+    private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+    // The cassandra.sai.intersection.clause.limit (default: 2) controls the maximum number of range iterator that
+    // will be used in the final intersection of a query operation.
+    private static final int INTERSECTION_CLAUSE_LIMIT = CassandraRelevantProperties.SAI_INTERSECTION_CLAUSE_LIMIT.getInt();
+
+    static
+    {
+        logger.info(String.format("Storage attached index intersection clause limit is %d", INTERSECTION_CLAUSE_LIMIT));
+    }
+
+    private final List<KeyRangeIterator> ranges;
+
+    private KeyRangeIntersectionIterator(Builder.Statistics statistics, List<KeyRangeIterator> ranges)
+    {
+        super(statistics);
+        this.ranges = ranges;
+    }
+
+    @Override
+    protected PrimaryKey computeNext()
+    {
+        // Range iterator that has been advanced in the previous cycle of the outer loop.
+        // Initially there hasn't been the previous cycle, so set to null.
+        int alreadyAvanced = -1;
+
+        // The highest primary key seen on any range iterator so far.
+        // It can become null when we reach the end of the iterator.
+        PrimaryKey highestKey = getCurrent();
+
+        outer:
+        // We need to check if highestKey exceeds the maximum because the maximum is
+        // the lowest value maximum of all the ranges. As a result any value could
+        // potentially exceed it.
+        while (highestKey != null && highestKey.compareTo(getMaximum()) <= 0)
+        {
+            // Try advance all iterators to the highest key seen so far.
+            // Once this inner loop finishes normally, all iterators are guaranteed to be at the same value.
+            for (int index = 0; index < ranges.size(); index++)
+            {
+                if (index != alreadyAvanced)
+                {
+                    KeyRangeIterator range = ranges.get(index);
+                    PrimaryKey nextKey = nextOrNull(range, highestKey);
+                    if (nextKey == null || nextKey.compareTo(highestKey) > 0)
+                    {
+                        // We jumped over the highest key seen so far, so make it the new highest key.
+                        highestKey = nextKey;
+                        // Remember this iterator to avoid advancing it again, because it is already at the highest key
+                        alreadyAvanced = index;
+                        // This iterator jumped over, so the other iterators are lagging behind now,
+                        // including the ones already advanced in the earlier cycles of the inner loop.
+                        // Therefore, restart the inner loop in order to advance
+                        // the other iterators except this one to match the new highest key.
+                        continue outer;
+                    }
+                    assert nextKey.compareTo(highestKey) == 0:
+                    String.format("skipped to an item smaller than the target; " +
+                                  "iterator: %s, target key: %s, returned key: %s", range, highestKey, nextKey);
+                }
+            }
+            // If we reached here, next() has been called at least once on each range iterator and
+            // the last call to next() on each iterator returned a value equal to the highestKey.
+            return highestKey;
+        }
+        return endOfData();
+    }
+
+    @Override
+    protected void performSkipTo(PrimaryKey nextKey)
+    {
+        for (KeyRangeIterator range : ranges)
+            if (range.hasNext())
+                range.skipTo(nextKey);
+    }
+
+    @Override
+    public void close()
+    {
+        FileUtils.closeQuietly(ranges);
+    }
+
+    /**
+     * Fetches the next available item from the iterator, such that the item is not lower than the given key.
+     * If no such items are available, returns null.
+     */
+    private PrimaryKey nextOrNull(KeyRangeIterator iterator, PrimaryKey minKey)
+    {
+        iterator.skipTo(minKey);
+        return iterator.hasNext() ? iterator.next() : null;
+    }
+
+    public static Builder builder(int size)
+    {
+        return builder(size, INTERSECTION_CLAUSE_LIMIT);
+    }
+
+    @VisibleForTesting
+    public static Builder builder(int size, int limit)
+    {
+        return new Builder(size, limit);
+    }
+
+    @VisibleForTesting
+    public static class Builder extends KeyRangeIterator.Builder
+    {
+        private final int limit;
+        // tracks if any of the added ranges are disjoint with the other ranges, which is useful
+        // in case of intersection, as it gives a direct answer whether the iterator is going
+        // to produce any results.
+        private boolean isDisjoint;
+
+        protected final List<KeyRangeIterator> rangeIterators;
+
+        Builder(int size, int limit)
+        {
+            super(new IntersectionStatistics());
+            rangeIterators = new ArrayList<>(size);
+            this.limit = limit;
+        }
+
+        @Override
+        public KeyRangeIterator.Builder add(KeyRangeIterator range)
+        {
+            if (range == null)
+                return this;
+
+            if (range.getCount() > 0)
+                rangeIterators.add(range);
+            else
+                FileUtils.closeQuietly(range);
+
+            updateStatistics(statistics, range);
+
+            return this;
+        }
+
+        @Override
+        public int rangeCount()
+        {
+            return rangeIterators.size();
+        }
+
+        @Override
+        public void cleanup()
+        {
+            FileUtils.closeQuietly(rangeIterators);
+        }
+
+        @Override
+        protected KeyRangeIterator buildIterator()
+        {
+            rangeIterators.sort(Comparator.comparingLong(KeyRangeIterator::getCount));
+            int initialSize = rangeIterators.size();
+            // all ranges will be included
+            if (limit >= rangeIterators.size() || limit <= 0)
+                return buildIterator(statistics, rangeIterators);
+
+            // Apply most selective iterators during intersection, because larger number of iterators will result lots of disk seek.
+            Statistics selectiveStatistics = new IntersectionStatistics();
+            isDisjoint = false;
+            for (int i = rangeIterators.size() - 1; i >= 0 && i >= limit; i--)
+                FileUtils.closeQuietly(rangeIterators.remove(i));
+
+            rangeIterators.forEach(range -> updateStatistics(selectiveStatistics, range));
+
+            if (Tracing.isTracing())
+                Tracing.trace("Selecting {} {} of {} out of {} indexes",
+                              rangeIterators.size(),
+                              rangeIterators.size() > 1 ? "indexes with cardinalities" : "index with cardinality",
+                              rangeIterators.stream().map(KeyRangeIterator::getCount).map(Object::toString).collect(Collectors.joining(", ")),
+                              initialSize);
+
+            return buildIterator(selectiveStatistics, rangeIterators);
+        }
+
+        public boolean isDisjoint()
+        {
+            return isDisjoint;
+        }
+
+        private KeyRangeIterator buildIterator(Statistics statistics, List<KeyRangeIterator> ranges)
+        {
+            // if the ranges are disjoint, or we have an intersection with an empty set,
+            // we can simply return an empty iterator, because it's not going to produce any results.
+            if (isDisjoint)
+            {
+                FileUtils.closeQuietly(ranges);
+                return KeyRangeIterator.empty();
+            }
+
+            if (ranges.size() == 1)
+                return ranges.get(0);
+
+            return new KeyRangeIntersectionIterator(statistics, ranges);
+        }
+
+        private void updateStatistics(Statistics statistics, KeyRangeIterator range)
+        {
+            statistics.update(range);
+            isDisjoint |= isDisjointInternal(statistics.min, statistics.max, range);
+        }
+    }
+
+    private static class IntersectionStatistics extends KeyRangeIterator.Builder.Statistics
+    {
+        @Override
+        public void update(KeyRangeIterator range)
+        {
+            // minimum of the intersection is the biggest minimum of individual iterators
+            min = nullSafeMax(min, range.getMinimum());
+            // maximum of the intersection is the smallest maximum of individual iterators
+            max = nullSafeMin(max, range.getMaximum());
+            count += range.getCount();
+        }
+    }
+
+    @VisibleForTesting
+    protected static boolean isDisjoint(KeyRangeIterator a, KeyRangeIterator b)
+    {
+        return isDisjointInternal(a.getCurrent(), a.getMaximum(), b);
+    }
+
+    /**
+     * Ranges are overlapping the following cases:
+     *
+     *   * When they have a common subrange:
+     *
+     *   min       b.current      max          b.max
+     *   +---------|--------------+------------|
+     *
+     *   b.current      min       max          b.max
+     *   |--------------+---------+------------|
+     *
+     *   min        b.current     b.max        max
+     *   +----------|-------------|------------+
+     *
+     *
+     *  If either range is empty, they're disjoint.
+     */
+    private static boolean isDisjointInternal(PrimaryKey min, PrimaryKey max, KeyRangeIterator b)
+    {
+        return min == null || max == null || b.getCount() == 0 || min.compareTo(b.getMaximum()) > 0 || b.getCurrent().compareTo(max) > 0;
+    }
+}

src/java/org/apache/cassandra/index/sai/utils/KeyRangeIterator.java

@@ -0,0 +1,210 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.utils;
+
+import java.io.Closeable;
+import java.util.List;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+
+import org.apache.cassandra.utils.AbstractGuavaIterator;
+
+/**
+ * An abstract implementation of {@link AbstractGuavaIterator} that supports the building and management of
+ * concatanation, union and intersection iterators.
+ */
+public abstract class KeyRangeIterator extends AbstractGuavaIterator<PrimaryKey> implements Closeable
+{
+    private final PrimaryKey min, max;
+    private final long count;
+    private PrimaryKey current;
+
+    protected KeyRangeIterator(Builder.Statistics statistics)
+    {
+        this(statistics.min, statistics.max, statistics.count);
+    }
+
+    public KeyRangeIterator(PrimaryKey min, PrimaryKey max, long count)
+    {
+        boolean isComplete = min != null && max != null && count != 0;
+        boolean isEmpty = min == null && max == null && (count == 0 || count == -1);
+        Preconditions.checkArgument(isComplete || isEmpty, "Range: [" + min + ',' + max + "], Count: " + count);
+
+        this.min = min;
+        this.current = min;
+        this.max = max;
+        this.count = count;
+    }
+
+    public final PrimaryKey getMinimum()
+    {
+        return min;
+    }
+
+    public final PrimaryKey getCurrent()
+    {
+        return current;
+    }
+
+    public final PrimaryKey getMaximum()
+    {
+        return max;
+    }
+
+    public final long getCount()
+    {
+        return count;
+    }
+
+    /**
+     * When called, this iterators current position should
+     * be skipped forwards until finding either:
+     *   1) an element equal to or bigger than next
+     *   2) the end of the iterator
+     *
+     * @param nextKey value to skip the iterator forward until matching
+     *
+     * @return The next current key after the skip was performed
+     */
+    public final PrimaryKey skipTo(PrimaryKey nextKey)
+    {
+        if (min == null || max == null)
+            return endOfData();
+
+        // In the case of deferred iterators the current value may not accurately
+        // reflect the next value, so we need to check that as well
+        if (current.compareTo(nextKey) >= 0)
+        {
+            next = next == null ? recomputeNext() : next;
+            if (next == null)
+                return endOfData();
+            else if (next.compareTo(nextKey) >= 0)
+                return next;
+        }
+
+        if (max.compareTo(nextKey) < 0)
+            return endOfData();
+
+        performSkipTo(nextKey);
+        return recomputeNext();
+    }
+
+    protected abstract void performSkipTo(PrimaryKey nextKey);
+
+    protected PrimaryKey recomputeNext()
+    {
+        return tryToComputeNext() ? peek() : endOfData();
+    }
+
+    protected boolean tryToComputeNext()
+    {
+        boolean hasNext = super.tryToComputeNext();
+        current = hasNext ? next : getMaximum();
+        return hasNext;
+    }
+
+    public static KeyRangeIterator empty()
+    {
+        return EmptyRangeIterator.instance;
+    }
+
+    private static class EmptyRangeIterator extends KeyRangeIterator
+    {
+        static final KeyRangeIterator instance = new EmptyRangeIterator();
+        EmptyRangeIterator() { super(null, null, 0); }
+        public PrimaryKey computeNext() { return endOfData(); }
+        protected void performSkipTo(PrimaryKey nextKey) { }
+        public void close() { }
+    }
+
+    @VisibleForTesting
+    public static abstract class Builder
+    {
+        protected final Statistics statistics;
+
+        public Builder(Statistics statistics)
+        {
+            this.statistics = statistics;
+        }
+
+        public PrimaryKey getMinimum()
+        {
+            return statistics.min;
+        }
+
+        public PrimaryKey getMaximum()
+        {
+            return statistics.max;
+        }
+
+        public long getCount()
+        {
+            return statistics.count;
+        }
+
+        public Builder add(List<KeyRangeIterator> ranges)
+        {
+            if (ranges == null || ranges.isEmpty())
+                return this;
+
+            ranges.forEach(this::add);
+            return this;
+        }
+
+        public final KeyRangeIterator build()
+        {
+            if (rangeCount() == 0)
+                return empty();
+            else
+                return buildIterator();
+        }
+
+        public abstract Builder add(KeyRangeIterator range);
+
+        public abstract int rangeCount();
+
+        public abstract void cleanup();
+
+        protected abstract KeyRangeIterator buildIterator();
+
+        public static abstract class Statistics
+        {
+            protected PrimaryKey min, max;
+            protected long count;
+
+            public abstract void update(KeyRangeIterator range);
+        }
+    }
+
+    protected static <T extends Comparable<T>> T nullSafeMin(T a, T b)
+    {
+        if (a == null) return b;
+        if (b == null) return a;
+
+        return a.compareTo(b) > 0 ? b : a;
+    }
+
+    protected static <T extends Comparable<T>> T nullSafeMax(T a, T b)
+    {
+        if (a == null) return b;
+        if (b == null) return a;
+
+        return a.compareTo(b) > 0 ? a : b;
+    }
+}

src/java/org/apache/cassandra/index/sai/utils/KeyRangeUnionIterator.java

@@ -0,0 +1,171 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.utils;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import com.google.common.annotations.VisibleForTesting;
+
+import org.apache.cassandra.io.util.FileUtils;
+
+/**
+ * Range Union Iterator is used to return sorted stream of elements from multiple RangeIterator instances.
+ */
+@SuppressWarnings({"resource", "RedundantSuppression"})
+public class KeyRangeUnionIterator extends KeyRangeIterator
+{
+    private final List<KeyRangeIterator> ranges;
+    private final List<KeyRangeIterator> candidates;
+
+    private KeyRangeUnionIterator(Builder.Statistics statistics, List<KeyRangeIterator> ranges)
+    {
+        super(statistics);
+        this.ranges = ranges;
+        this.candidates = new ArrayList<>(ranges.size());
+    }
+
+    @Override
+    public PrimaryKey computeNext()
+    {
+        candidates.clear();
+        PrimaryKey candidate = null;
+        for (KeyRangeIterator range : ranges)
+        {
+            if (range.hasNext())
+            {
+                // Avoid repeated values but only if we have read at least one value
+                while (next != null && range.hasNext() && range.peek().compareTo(getCurrent()) == 0)
+                    range.next();
+                if (!range.hasNext())
+                    continue;
+                if (candidate == null)
+                {
+                    candidate = range.peek();
+                    candidates.add(range);
+                }
+                else
+                {
+                    int cmp = candidate.compareTo(range.peek());
+                    if (cmp == 0)
+                        candidates.add(range);
+                    else if (cmp > 0)
+                    {
+                        candidates.clear();
+                        candidate = range.peek();
+                        candidates.add(range);
+                    }
+                }
+            }
+        }
+        if (candidates.isEmpty())
+            return endOfData();
+        candidates.forEach(KeyRangeIterator::next);
+        return candidate;
+    }
+
+    @Override
+    protected void performSkipTo(PrimaryKey nextKey)
+    {
+        for (KeyRangeIterator range : ranges)
+        {
+            if (range.hasNext())
+                range.skipTo(nextKey);
+        }
+    }
+
+    @Override
+    public void close()
+    {
+        // Due to lazy key fetching, we cannot close iterator immediately
+        FileUtils.closeQuietly(ranges);
+    }
+
+    public static Builder builder(int size)
+    {
+        return new Builder(size);
+    }
+
+    public static KeyRangeIterator build(List<KeyRangeIterator> keys)
+    {
+        return new Builder(keys.size()).add(keys).build();
+    }
+
+    @VisibleForTesting
+    public static class Builder extends KeyRangeIterator.Builder
+    {
+        protected final List<KeyRangeIterator> rangeIterators;
+
+        Builder(int size)
+        {
+            super(new UnionStatistics());
+            this.rangeIterators = new ArrayList<>(size);
+        }
+
+        @Override
+        public KeyRangeIterator.Builder add(KeyRangeIterator range)
+        {
+            if (range == null)
+                return this;
+
+            if (range.getCount() > 0)
+            {
+                rangeIterators.add(range);
+                statistics.update(range);
+            }
+            else
+            {
+                FileUtils.closeQuietly(range);
+            }
+
+            return this;
+        }
+
+        @Override
+        public int rangeCount()
+        {
+            return rangeIterators.size();
+        }
+
+        @Override
+        public void cleanup()
+        {
+            FileUtils.closeQuietly(rangeIterators);
+        }
+
+        @Override
+        protected KeyRangeIterator buildIterator()
+        {
+            if (rangeCount() == 1)
+                return rangeIterators.get(0);
+
+            return new KeyRangeUnionIterator(statistics, rangeIterators);
+        }
+    }
+
+    private static class UnionStatistics extends KeyRangeIterator.Builder.Statistics
+    {
+        @Override
+        public void update(KeyRangeIterator range)
+        {
+            min = nullSafeMin(min, range.getMinimum());
+            max = nullSafeMax(max, range.getMaximum());
+            count += range.getCount();
+        }
+    }
+}

src/java/org/apache/cassandra/index/sai/utils/PrimaryKey.java

@@ -0,0 +1,187 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.utils;
+
+import java.util.Arrays;
+import java.util.Objects;
+import java.util.stream.Collectors;
+
+import org.apache.cassandra.db.Clustering;
+import org.apache.cassandra.db.ClusteringComparator;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+import org.apache.cassandra.utils.bytecomparable.ByteSource;
+
+/**
+ * Representation of the primary key for a row consisting of the {@link DecoratedKey} and
+ * {@link Clustering} associated with a {@link org.apache.cassandra.db.rows.Row}.
+ */
+public class PrimaryKey implements Comparable<PrimaryKey>
+{
+    private final Token token;
+    private final DecoratedKey partitionKey;
+    private final Clustering<?> clustering;
+    private final ClusteringComparator clusteringComparator;
+
+    PrimaryKey(Token token)
+    {
+        this(token, null, null, null);
+    }
+
+    PrimaryKey(Token token,
+               DecoratedKey partitionKey,
+               Clustering<?> clustering,
+               ClusteringComparator clusteringComparator)
+    {
+        this.token = token;
+        this.partitionKey = partitionKey;
+        this.clustering = clustering;
+        this.clusteringComparator = clusteringComparator;
+    }
+
+    /**
+     * Returns a {@link PrimaryKeyFactory} for creating {@link PrimaryKey} instances.
+     *
+     * @param clusteringComparator the {@link ClusteringComparator} used by the
+     *                             {@link PrimaryKeyFactory} for clustering comparisons
+     * @return a {@link PrimaryKeyFactory} for {@link PrimaryKey} creation
+     */
+    public static PrimaryKeyFactory factory(ClusteringComparator clusteringComparator)
+    {
+        return new PrimaryKeyFactory(clusteringComparator);
+    }
+
+    /**
+     * @return the {@link Token} associated with this primary key.
+     */
+    public Token token()
+    {
+        return token;
+    }
+
+    /**
+     * @return the {@link DecoratedKey} associated with this primary key.
+     */
+    public DecoratedKey partitionKey()
+    {
+        return partitionKey;
+    }
+
+    /**
+     * @return the {@link Clustering} associated with this primary key.
+     */
+    public Clustering<?> clustering()
+    {
+        return clustering;
+    }
+
+    /**
+     * Returns the {@link PrimaryKey} as a {@link ByteSource} byte comparable representation.
+     *
+     * It is important that these representations are only ever used with byte comparables using
+     * the same elements. This means that {@code asComparableBytes} responses can only be used
+     * together from the same {@link PrimaryKey} implementation.
+     *
+     * @param version the {@link ByteComparable.Version} to use for the implementation
+     * @return the {@code ByteSource} byte comparable.
+     */
+    public ByteSource asComparableBytes(ByteComparable.Version version)
+    {
+        ByteSource tokenComparable = token.asComparableBytes(version);
+        if (partitionKey == null)
+            return ByteSource.withTerminator(version == ByteComparable.Version.LEGACY ? ByteSource.END_OF_STREAM
+                                                                                      : ByteSource.TERMINATOR,
+                                             tokenComparable,
+                                             null,
+                                             null);
+        ByteSource keyComparable = ByteSource.of(partitionKey.getKey(), version);
+        // It is important that the ClusteringComparator.asBytesComparable method is used
+        // to maintain the correct clustering sort order
+        ByteSource clusteringComparable = clusteringComparator.size() == 0 ||
+                                          clustering == null ||
+                                          clustering.isEmpty() ? null
+                                                               : clusteringComparator.asByteComparable(clustering)
+                                                                                     .asComparableBytes(version);
+        return ByteSource.withTerminator(version == ByteComparable.Version.LEGACY ? ByteSource.END_OF_STREAM
+                                                                                  : ByteSource.TERMINATOR,
+                                         tokenComparable,
+                                         keyComparable,
+                                         clusteringComparable);
+    }
+
+    @Override
+    public int compareTo(PrimaryKey o)
+    {
+        int cmp = token().compareTo(o.token());
+
+        // If the tokens don't match then we don't need to compare any more of the key.
+        // Otherwise, if it's partition key is null or the other partition key is null
+        // then one or both of the keys are token only so we can only compare tokens
+        if ((cmp != 0) || (partitionKey == null) || o.partitionKey() == null)
+            return cmp;
+
+        // Next compare the partition keys. If they are not equal or
+        // this is a single row partition key or there are no
+        // clusterings then we can return the result of this without
+        // needing to compare the clusterings
+        cmp = partitionKey().compareTo(o.partitionKey());
+        if (cmp != 0 || hasEmptyClustering() || o.hasEmptyClustering())
+            return cmp;
+        return clusteringComparator.compare(clustering(), o.clustering());
+    }
+
+    /**
+     * Return whether the primary key has an empty clustering or not.
+     * By default, the clustering is empty if the internal clustering
+     * is null or is empty.
+     *
+     * @return {@code true} if the clustering is empty, otherwise {@code false}
+     */
+    public boolean hasEmptyClustering()
+    {
+        return clustering() == null || clustering().isEmpty();
+    }
+
+    @Override
+    public int hashCode()
+    {
+        return Objects.hash(token, partitionKey, clustering, clusteringComparator);
+    }
+
+    @Override
+    public boolean equals(Object obj)
+    {
+        if (obj instanceof PrimaryKey)
+            return compareTo((PrimaryKey)obj) == 0;
+        return false;
+    }
+
+    @Override
+    public String toString()
+    {
+        return String.format("PrimaryKey: { token: %s, partition: %s, clustering: %s:%s} ",
+                             token,
+                             partitionKey,
+                             clustering == null ? null : clustering.kind(),
+                             clustering == null ? null : Arrays.stream(clustering.getBufferArray())
+                                                               .map(ByteBufferUtil::bytesToHex)
+                                                               .collect(Collectors.joining(",")));
+    }
+}

src/java/org/apache/cassandra/index/sai/utils/PrimaryKeyFactory.java

@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.utils;
+
+import org.apache.cassandra.db.Clustering;
+import org.apache.cassandra.db.ClusteringComparator;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.dht.Token;
+
+/**
+ * A factory for creating instances that are sortable by {@link DecoratedKey} and {@link Clustering}.
+ */
+public class PrimaryKeyFactory
+{
+    private final ClusteringComparator clusteringComparator;
+
+    public PrimaryKeyFactory(ClusteringComparator clusteringComparator)
+    {
+        this.clusteringComparator = clusteringComparator;
+    }
+
+    /**
+     * Creates a {@link PrimaryKey} that is represented by a {@link Token}.
+     *
+     * {@link Token} only primary keys are used for defining the partition range
+     * of a query.
+     */
+    public PrimaryKey createTokenOnly(Token token)
+    {
+        return new PrimaryKey(token);
+    }
+
+    /**
+     * Creates a {@link PrimaryKey} that is fully represented by partition key
+     * and clustering.
+     */
+    public PrimaryKey create(DecoratedKey partitionKey, Clustering<?> clustering)
+    {
+        return new PrimaryKey(partitionKey.getToken(), partitionKey, clustering, clusteringComparator);
+    }
+}

src/java/org/apache/cassandra/index/sai/utils/PrimaryKeys.java

@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.utils;
+
+import java.util.Iterator;
+import java.util.SortedSet;
+import java.util.concurrent.ConcurrentSkipListSet;
+
+import javax.annotation.concurrent.ThreadSafe;
+
+import org.apache.cassandra.utils.ObjectSizes;
+
+/**
+ * A sorted set of {@link PrimaryKey}s.
+ *
+ * The primary keys are sorted first by token, then by partition key value, and then by clustering.
+ */
+@ThreadSafe
+public class PrimaryKeys implements Iterable<PrimaryKey>
+{
+    private static final long EMPTY_SIZE = ObjectSizes.measure(new PrimaryKeys());
+    // from https://github.com/gaul/java-collection-overhead
+    private static final long SET_ENTRY_OVERHEAD = 36;
+
+    private final ConcurrentSkipListSet<PrimaryKey> keys = new ConcurrentSkipListSet<>();
+
+    /**
+     * Adds a {@link PrimaryKey} and returns the on-heap memory used if the key was added
+     */
+    public long add(PrimaryKey key)
+    {
+        return keys.add(key) ? SET_ENTRY_OVERHEAD : 0;
+    }
+
+    public SortedSet<PrimaryKey> keys()
+    {
+        return keys;
+    }
+
+    public int size()
+    {
+        return keys.size();
+    }
+
+    public boolean isEmpty()
+    {
+        return keys.isEmpty();
+    }
+
+    public long unsharedHeapSize()
+    {
+        return EMPTY_SIZE;
+    }
+
+    @Override
+    public Iterator<PrimaryKey> iterator()
+    {
+        return keys.iterator();
+    }
+}

src/java/org/apache/cassandra/index/sai/utils/TypeUtil.java

@@ -0,0 +1,472 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.utils;
+
+import java.math.BigInteger;
+import java.net.InetAddress;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.Iterator;
+import java.util.stream.Stream;
+import java.util.stream.StreamSupport;
+
+import com.googlecode.concurrenttrees.radix.ConcurrentRadixTree;
+import org.apache.cassandra.cql3.statements.schema.IndexTarget;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.AsciiType;
+import org.apache.cassandra.db.marshal.BooleanType;
+import org.apache.cassandra.db.marshal.ByteBufferAccessor;
+import org.apache.cassandra.db.marshal.CollectionType;
+import org.apache.cassandra.db.marshal.CompositeType;
+import org.apache.cassandra.db.marshal.DecimalType;
+import org.apache.cassandra.db.marshal.InetAddressType;
+import org.apache.cassandra.db.marshal.IntegerType;
+import org.apache.cassandra.db.marshal.ReversedType;
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.db.rows.Cell;
+import org.apache.cassandra.db.rows.ComplexColumnData;
+import org.apache.cassandra.index.sai.plan.Expression;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.serializers.MarshalException;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.FastByteOperations;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+import org.apache.cassandra.utils.bytecomparable.ByteSource;
+import org.apache.cassandra.utils.bytecomparable.ByteSourceInverse;
+
+public class TypeUtil
+{
+    private static final byte[] IPV4_PREFIX = new byte[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, -1 };
+
+    /**
+     * DecimalType / BigDecimal values are indexed by truncating their asComparableBytes representation to this size,
+     * padding on the right with zero-value-bytes until this size is reached (if necessary).  This causes
+     * false-positives that must be filtered in a separate step after hitting the index and reading the associated
+     * (full) values.
+     */
+    public static final int DECIMAL_APPROXIMATION_BYTES = 24;
+
+    private TypeUtil() {}
+
+    /**
+     * Returns <code>true</code> if given buffer would pass the {@link AbstractType#validate(ByteBuffer)}
+     * check. False otherwise.
+     */
+    public static boolean isValid(ByteBuffer term, AbstractType<?> validator)
+    {
+        try
+        {
+            validator.validate(term);
+            return true;
+        }
+        catch (MarshalException e)
+        {
+            return false;
+        }
+    }
+
+    /**
+     * Indicates if the type encoding supports rounding of the raw value.
+     *
+     * This is significant in range searches where we have to make all range
+     * queries inclusive when searching the indexes in order to avoid excluding
+     * rounded values. Excluded values are removed by post-filtering.
+     */
+    public static boolean supportsRounding(AbstractType<?> type)
+    {
+        return isBigInteger(type) || isBigDecimal(type);
+    }
+
+    /**
+     * Returns the smaller of two {@code ByteBuffer} values, based on the result of {@link
+     * #compare(ByteBuffer, ByteBuffer, AbstractType)} comparision.
+     */
+    public static ByteBuffer min(ByteBuffer a, ByteBuffer b, AbstractType<?> type)
+    {
+        return a == null ? b : (b == null || compare(b, a, type) > 0) ? a : b;
+    }
+
+    /**
+     * Returns the greater of two {@code ByteBuffer} values, based on the result of {@link
+     * #compare(ByteBuffer, ByteBuffer, AbstractType)} comparision.
+     */
+    public static ByteBuffer max(ByteBuffer a, ByteBuffer b, AbstractType<?> type)
+    {
+        return a == null ? b : (b == null || compare(b, a, type) < 0) ? a : b;
+    }
+
+    /**
+     * Returns the lesser of two {@code ByteComparable} values, based on the result of {@link
+     * ByteComparable#compare(ByteComparable, ByteComparable, ByteComparable.Version)} comparision.
+     */
+    public static ByteComparable min(ByteComparable a, ByteComparable b)
+    {
+        return a == null ? b : (b == null || ByteComparable.compare(b, a, ByteComparable.Version.OSS50) > 0) ? a : b;
+    }
+
+    /**
+     * Returns the greater of two {@code ByteComparable} values, based on the result of {@link
+     * ByteComparable#compare(ByteComparable, ByteComparable, ByteComparable.Version)} comparision.
+     */
+    public static ByteComparable max(ByteComparable a, ByteComparable b)
+    {
+        return a == null ? b : (b == null || ByteComparable.compare(b, a, ByteComparable.Version.OSS50) < 0) ? a : b;
+    }
+
+    public static AbstractType<?> cellValueType(ColumnMetadata columnMetadata, IndexTarget.Type indexType)
+    {
+        AbstractType<?> type = columnMetadata.type;
+        if (isNonFrozenCollection(type))
+        {
+            CollectionType<?> collection = ((CollectionType<?>) type);
+            switch (collection.kind)
+            {
+                case LIST:
+                    return collection.valueComparator();
+                case SET:
+                    return collection.nameComparator();
+                case MAP:
+                    switch (indexType)
+                    {
+                        case KEYS:
+                            return collection.nameComparator();
+                        case VALUES:
+                            return collection.valueComparator();
+                        case KEYS_AND_VALUES:
+                            return CompositeType.getInstance(collection.nameComparator(), collection.valueComparator());
+                    }
+            }
+        }
+        return type;
+    }
+
+    /**
+     * Allows overriding the default getString method for {@link CompositeType}. It is
+     * a requirement of the {@link ConcurrentRadixTree} that the keys are strings but
+     * the getString method of {@link CompositeType} does not return a string that compares
+     * in the same order as the underlying {@link ByteBuffer}. To get round this we convert
+     * the {@link CompositeType} bytes to a hex string.
+     */
+    public static String getString(ByteBuffer value, AbstractType<?> type)
+    {
+        if (isComposite(type))
+            return ByteBufferUtil.bytesToHex(value);
+        return type.getString(value);
+    }
+
+    /**
+     * The inverse of the above method. Overrides the fromString method on {@link CompositeType}
+     * in order to convert the hex string to bytes.
+     */
+    public static ByteBuffer fromString(String value, AbstractType<?> type)
+    {
+        if (isComposite(type))
+            return ByteBufferUtil.hexToBytes(value);
+        return type.fromString(value);
+    }
+
+    public static ByteSource asComparableBytes(ByteBuffer value, AbstractType<?> type, ByteComparable.Version version)
+    {
+        if (type instanceof InetAddressType || type instanceof IntegerType || type instanceof DecimalType)
+            return ByteSource.optionalFixedLength(ByteBufferAccessor.instance, value);
+        return type.asComparableBytes(value, version);
+    }
+
+    /**
+     * Translates the external value of specific types into a format used by the index.
+     */
+    public static ByteBuffer asIndexBytes(ByteBuffer value, AbstractType<?> type)
+    {
+        if (value == null)
+            return null;
+
+        if (isInetAddress(type))
+            return encodeInetAddress(value);
+        else if (isBigInteger(type))
+            return encodeBigInteger(value);
+        else if (type instanceof DecimalType)
+            return encodeDecimal(value);
+        return value;
+    }
+
+    /**
+     * Compare two terms based on their type. This is used in place of {@link AbstractType#compare(ByteBuffer, ByteBuffer)}
+     * so that the default comparison can be overridden for specific types.
+     *
+     * Note: This should be used for all term comparison
+     */
+    public static int compare(ByteBuffer b1, ByteBuffer b2, AbstractType<?> type)
+    {
+        if (isInetAddress(type))
+            return compareInet(b1, b2);
+        // BigInteger values, frozen types and composite types (map entries) use compareUnsigned to maintain
+        // a consistent order between the in-memory index and the on-disk index.
+        else if (isBigInteger(type) || isBigDecimal(type) || isCompositeOrFrozen(type))
+            return FastByteOperations.compareUnsigned(b1, b2);
+
+        return type.compare(b1, b2 );
+    }
+
+    /**
+     * This is used for value comparison in post-filtering - {@link Expression#isSatisfiedBy(ByteBuffer)}.
+     *
+     * This allows types to decide whether they should be compared based on their encoded value or their
+     * raw value. At present only {@link InetAddressType} values are compared by their encoded values to
+     * allow for ipv4 -> ipv6 equivalency in searches.
+     */
+    public static int comparePostFilter(Expression.Value requestedValue, Expression.Value columnValue, AbstractType<?> type)
+    {
+        if (isInetAddress(type))
+            return compareInet(requestedValue.encoded, columnValue.encoded);
+        // Override comparisons for frozen collections and composite types (map entries)
+        else if (isCompositeOrFrozen(type))
+            return FastByteOperations.compareUnsigned(requestedValue.raw, columnValue.raw);
+
+        return type.compare(requestedValue.raw, columnValue.raw);
+    }
+
+    public static Iterator<ByteBuffer> collectionIterator(AbstractType<?> validator,
+                                                          ComplexColumnData cellData,
+                                                          ColumnMetadata columnMetadata,
+                                                          IndexTarget.Type indexType,
+                                                          long nowInSecs)
+    {
+        if (cellData == null)
+            return null;
+
+        Stream<ByteBuffer> stream = StreamSupport.stream(cellData.spliterator(), false).filter(cell -> cell != null && cell.isLive(nowInSecs))
+                                                 .map(cell -> cellValue(columnMetadata, indexType, cell));
+
+        if (isInetAddress(validator))
+            stream = stream.sorted((c1, c2) -> compareInet(encodeInetAddress(c1), encodeInetAddress(c2)));
+
+        return stream.iterator();
+    }
+
+    public static Comparator<ByteBuffer> comparator(AbstractType<?> type)
+    {
+        // Override the comparator for BigInteger, frozen collections and composite types
+        if (isBigInteger(type) || isBigDecimal(type) || isCompositeOrFrozen(type))
+            return FastByteOperations::compareUnsigned;
+
+        return type;
+    }
+
+    private static ByteBuffer cellValue(ColumnMetadata columnMetadata, IndexTarget.Type indexType, Cell<?> cell)
+    {
+        if (columnMetadata.type.isCollection() && columnMetadata.type.isMultiCell())
+        {
+            switch (((CollectionType<?>) columnMetadata.type).kind)
+            {
+                case LIST:
+                    return cell.buffer();
+                case SET:
+                    return cell.path().get(0);
+                case MAP:
+                    switch (indexType)
+                    {
+                        case KEYS:
+                            return cell.path().get(0);
+                        case VALUES:
+                            return cell.buffer();
+                        case KEYS_AND_VALUES:
+                            return CompositeType.build(ByteBufferAccessor.instance, cell.path().get(0), cell.buffer());
+                    }
+            }
+        }
+        return cell.buffer();
+    }
+
+    /**
+     * Compares 2 InetAddress terms by ensuring that both addresses are represented as
+     * ipv6 addresses.
+     */
+    private static int compareInet(ByteBuffer b1, ByteBuffer b2)
+    {
+        assert isIPv6(b1) && isIPv6(b2);
+
+        return FastByteOperations.compareUnsigned(b1, b2);
+    }
+
+    private static boolean isIPv6(ByteBuffer address)
+    {
+        return address.remaining() == 16;
+    }
+
+    /**
+     * Encode a {@link InetAddress} into a fixed width 16 byte encoded value.
+     *
+     * The encoded value is byte comparable and prefix compressible.
+     *
+     * The encoding is done by converting ipv4 addresses to their ipv6 equivalent.
+     */
+    private static ByteBuffer encodeInetAddress(ByteBuffer value)
+    {
+        if (value.remaining() == 4)
+        {
+            int position = value.hasArray() ? value.arrayOffset() + value.position() : value.position();
+            ByteBuffer mapped = ByteBuffer.allocate(16);
+            System.arraycopy(IPV4_PREFIX, 0, mapped.array(), 0, IPV4_PREFIX.length);
+            ByteBufferUtil.copyBytes(value, position, mapped, IPV4_PREFIX.length, value.remaining());
+            return mapped;
+        }
+        return value;
+    }
+
+    /**
+     * Encode a {@link BigInteger} into a fixed width 20 byte encoded value.
+     *
+     * The encoded value is byte comparable and prefix compressible.
+     *
+     * The format of the encoding is:
+     *
+     *  The first 4 bytes contain the length of the {@link BigInteger} byte array
+     *  with the top bit flipped for positive values.
+     *
+     *  The remaining 16 bytes contain the 16 most significant bytes of the
+     *  {@link BigInteger} byte array.
+     *
+     *  For {@link BigInteger} values whose underlying byte array is less than
+     *  16 bytes, the encoded value is sign extended.
+     */
+    public static ByteBuffer encodeBigInteger(ByteBuffer value)
+    {
+        int size = value.remaining();
+        int position = value.hasArray() ? value.arrayOffset() + value.position() : value.position();
+        byte[] bytes = new byte[20];
+        if (size < 16)
+        {
+            ByteBufferUtil.copyBytes(value, position, bytes, bytes.length - size, size);
+            if ((bytes[bytes.length - size] & 0x80) != 0)
+                Arrays.fill(bytes, 4, bytes.length - size, (byte)0xff);
+            else
+                Arrays.fill(bytes, 4, bytes.length - size, (byte)0x00);
+        }
+        else
+        {
+            ByteBufferUtil.copyBytes(value, position, bytes, 4, 16);
+        }
+        if ((bytes[4] & 0x80) != 0)
+        {
+            size = -size;
+        }
+        bytes[0] = (byte)(size >> 24 & 0xff);
+        bytes[1] = (byte)(size >> 16 & 0xff);
+        bytes[2] = (byte)(size >> 8 & 0xff);
+        bytes[3] = (byte)(size & 0xff);
+        bytes[0] ^= 0x80;
+        return ByteBuffer.wrap(bytes);
+    }
+
+    /**
+     * Returns <code>true</code> if values of the given {@link AbstractType} should be indexed as literals.
+     */
+    public static boolean isLiteral(AbstractType<?> type)
+    {
+        return isUTF8OrAscii(type) || isCompositeOrFrozen(type) || baseType(type) instanceof BooleanType;
+    }
+
+    /**
+     * Returns <code>true</code> if given {@link AbstractType} is UTF8 or Ascii
+     */
+    public static boolean isUTF8OrAscii(AbstractType<?> type)
+    {
+        type = baseType(type);
+        return type instanceof UTF8Type || type instanceof AsciiType;
+    }
+
+    /**
+     * Returns <code>true</code> if given {@link AbstractType} is a Composite(map entry) or frozen.
+     */
+    public static boolean isCompositeOrFrozen(AbstractType<?> type)
+    {
+        type = baseType(type);
+        return type instanceof CompositeType || isFrozen(type);
+    }
+
+    /**
+     * Returns <code>true</code> if given {@link AbstractType} is frozen.
+     */
+    public static boolean isFrozen(AbstractType<?> type)
+    {
+        type = baseType(type);
+        return !type.subTypes().isEmpty() && !type.isMultiCell();
+    }
+
+    /**
+     * Returns <code>true</code> if given {@link AbstractType} is a non-frozen collection.
+     */
+    public static boolean isNonFrozenCollection(AbstractType<?> type)
+    {
+        type = baseType(type);
+        return type.isCollection() && type.isMultiCell();
+    }
+
+    /**
+     * Returns <code>true</code> if given {@link AbstractType} is {@link InetAddressType}
+     */
+    private static boolean isInetAddress(AbstractType<?> type)
+    {
+        type = baseType(type);
+        return type instanceof InetAddressType;
+    }
+
+    /**
+     * Returns <code>true</code> if given {@link AbstractType} is {@link IntegerType}
+     */
+    private static boolean isBigInteger(AbstractType<?> type)
+    {
+        type = baseType(type);
+        return type instanceof IntegerType;
+    }
+
+    /**
+     * Returns <code>true</code> if given {@link AbstractType} is {@link DecimalType}
+     */
+    private static boolean isBigDecimal(AbstractType<?> type)
+    {
+        type = baseType(type);
+        return type instanceof DecimalType;
+    }
+
+    /**
+     * Returns <code>true</code> if given {@link AbstractType} is {@link CompositeType}
+     */
+    public static boolean isComposite(AbstractType<?> type)
+    {
+        type = baseType(type);
+        return type instanceof CompositeType;
+    }
+
+    /**
+     * @return base type if given type is reversed, otherwise return itself
+     */
+    private static AbstractType<?> baseType(AbstractType<?> type)
+    {
+        return type.isReversed() ? ((ReversedType<?>) type).baseType : type;
+    }
+
+    public static ByteBuffer encodeDecimal(ByteBuffer value)
+    {
+        ByteSource bs = DecimalType.instance.asComparableBytes(value, ByteComparable.Version.OSS50);
+        bs = ByteSource.cutOrRightPad(bs, DECIMAL_APPROXIMATION_BYTES, 0);
+        return ByteBuffer.wrap(ByteSourceInverse.readBytes(bs, DECIMAL_APPROXIMATION_BYTES));
+    }
+}

src/java/org/apache/cassandra/index/sai/virtual/IndexesSystemView.java

@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.virtual;
+
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.marshal.BooleanType;
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.db.partitions.PartitionUpdate;
+import org.apache.cassandra.db.virtual.AbstractVirtualTable;
+import org.apache.cassandra.db.virtual.SimpleDataSet;
+import org.apache.cassandra.db.virtual.VirtualTable;
+import org.apache.cassandra.dht.LocalPartitioner;
+import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.index.Index;
+import org.apache.cassandra.index.SecondaryIndexManager;
+import org.apache.cassandra.index.sai.IndexContext;
+import org.apache.cassandra.index.sai.StorageAttachedIndex;
+import org.apache.cassandra.index.sai.StorageAttachedIndexGroup;
+import org.apache.cassandra.schema.KeyspaceMetadata;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.TableMetadata;
+
+/**
+ * A {@link VirtualTable} providing a system view of per-column storage-attached index metadata.
+ */
+public class IndexesSystemView extends AbstractVirtualTable
+{
+    static final String NAME = "indexes";
+
+    static final String KEYSPACE_NAME = "keyspace_name";
+    static final String INDEX_NAME = "index_name";
+    static final String TABLE_NAME = "table_name";
+    static final String COLUMN_NAME = "column_name";
+    static final String IS_QUERYABLE = "is_queryable";
+    static final String IS_BUILDING = "is_building";
+    static final String IS_STRING = "is_string";
+    static final String ANALYZER = "analyzer";
+
+    public IndexesSystemView(String keyspace)
+    {
+        super(TableMetadata.builder(keyspace, NAME)
+                           .partitioner(new LocalPartitioner(UTF8Type.instance))
+                           .comment("Storage-attached column index metadata")
+                           .kind(TableMetadata.Kind.VIRTUAL)
+                           .addPartitionKeyColumn(KEYSPACE_NAME, UTF8Type.instance)
+                           .addClusteringColumn(INDEX_NAME, UTF8Type.instance)
+                           .addRegularColumn(TABLE_NAME, UTF8Type.instance)
+                           .addRegularColumn(COLUMN_NAME, UTF8Type.instance)
+                           .addRegularColumn(IS_QUERYABLE, BooleanType.instance)
+                           .addRegularColumn(IS_BUILDING, BooleanType.instance)
+                           .addRegularColumn(IS_STRING, BooleanType.instance)
+                           .addRegularColumn(ANALYZER, UTF8Type.instance)
+                           .build());
+    }
+
+    @Override
+    public void apply(PartitionUpdate update)
+    {
+        throw new InvalidRequestException("Modification is not supported by table " + metadata);
+    }
+
+    @Override
+    public DataSet data()
+    {
+        SimpleDataSet dataset = new SimpleDataSet(metadata());
+
+        for (KeyspaceMetadata ks : Schema.instance.getUserKeyspaces())
+        {
+            Keyspace keyspace = Schema.instance.getKeyspaceInstance(ks.name);
+            if (keyspace == null)
+                throw new IllegalArgumentException("Unknown keyspace " + ks.name);
+
+            for (ColumnFamilyStore cfs : keyspace.getColumnFamilyStores())
+            {
+                SecondaryIndexManager manager = cfs.indexManager;
+                StorageAttachedIndexGroup group = StorageAttachedIndexGroup.getIndexGroup(cfs);
+
+                if (group != null)
+                {
+                    for (Index index : group.getIndexes())
+                    {
+                        IndexContext context = ((StorageAttachedIndex) index).getIndexContext();
+                        String indexName = context.getIndexName();
+
+                        dataset.row(ks.name, indexName)
+                               .column(TABLE_NAME, cfs.name)
+                               .column(COLUMN_NAME, context.getColumnName())
+                               .column(IS_QUERYABLE, manager.isIndexQueryable(index))
+                               .column(IS_BUILDING, manager.isIndexBuilding(indexName))
+                               .column(IS_STRING, context.isLiteral())
+                               .column(ANALYZER, context.getIndexAnalyzerFactory().toString());
+                    }
+                }
+            }
+        }
+
+        return dataset;
+    }
+}

src/java/org/apache/cassandra/index/sai/virtual/StorageAttachedIndexTables.java

@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.virtual;
+
+import java.util.Collection;
+import java.util.Collections;
+
+import org.apache.cassandra.db.virtual.VirtualTable;
+
+public class StorageAttachedIndexTables
+{
+    private StorageAttachedIndexTables()
+    {}
+
+    public static Collection<VirtualTable> getAll(String keyspace)
+    {
+        return Collections.singletonList(new IndexesSystemView(keyspace));
+    }
+}

src/java/org/apache/cassandra/index/sasi/disk/OnDiskIndex.java

@@ -40,15 +40,15 @@ import org.apache.cassandra.db.marshal.AbstractType;
 import org.apache.cassandra.index.sasi.Term;
 import org.apache.cassandra.index.sasi.plan.Expression;
 import org.apache.cassandra.index.sasi.plan.Expression.Op;
-import org.apache.cassandra.index.sasi.utils.AbstractIterator;
 import org.apache.cassandra.index.sasi.utils.MappedBuffer;
-import org.apache.cassandra.index.sasi.utils.RangeIterator;
 import org.apache.cassandra.index.sasi.utils.RangeUnionIterator;
+import org.apache.cassandra.index.sasi.utils.RangeIterator;
 import org.apache.cassandra.io.FSReadError;
 import org.apache.cassandra.io.util.ChannelProxy;
 import org.apache.cassandra.io.util.File;
 import org.apache.cassandra.io.util.FileInputStreamPlus;
 import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.utils.AbstractGuavaIterator;
 import org.apache.cassandra.utils.ByteBufferUtil;
 import org.apache.cassandra.utils.FBUtilities;
 
@@ -731,7 +731,7 @@ public class OnDiskIndex implements Iterable<OnDiskIndex.DataTerm>, Closeable
         return indexPath;
     }
 
-    private class TermIterator extends AbstractIterator<DataTerm>
+    private class TermIterator extends AbstractGuavaIterator<DataTerm>
     {
         private final Expression e;
         private final IteratorOrder order;

src/java/org/apache/cassandra/index/sasi/disk/TokenTree.java

@@ -21,10 +21,10 @@ import java.io.IOException;
 import java.util.*;
 
 import org.apache.cassandra.db.DecoratedKey;
-import org.apache.cassandra.index.sasi.utils.AbstractIterator;
 import org.apache.cassandra.index.sasi.utils.CombinedValue;
 import org.apache.cassandra.index.sasi.utils.MappedBuffer;
 import org.apache.cassandra.index.sasi.utils.RangeIterator;
+import org.apache.cassandra.utils.AbstractGuavaIterator;
 import org.apache.cassandra.utils.MergeIterator;
 
 import com.carrotsearch.hppc.LongHashSet;
@@ -503,7 +503,7 @@ public class TokenTree
         }
     }
 
-    private static class KeyIterator extends AbstractIterator<DecoratedKey>
+    private static class KeyIterator extends AbstractGuavaIterator<DecoratedKey>
     {
         private final Function<Long, DecoratedKey> keyFetcher;
         private final long[] offsets;

src/java/org/apache/cassandra/index/sasi/memory/KeyRangeIterator.java

@@ -25,12 +25,13 @@ import java.util.concurrent.ConcurrentSkipListSet;
 
 import org.apache.cassandra.db.DecoratedKey;
 import org.apache.cassandra.index.sasi.disk.Token;
-import org.apache.cassandra.index.sasi.utils.AbstractIterator;
 import org.apache.cassandra.index.sasi.utils.CombinedValue;
 import org.apache.cassandra.index.sasi.utils.RangeIterator;
 
 import com.carrotsearch.hppc.LongHashSet;
 import com.carrotsearch.hppc.LongSet;
+import org.apache.cassandra.utils.AbstractGuavaIterator;
+
 import com.google.common.collect.PeekingIterator;
 
 public class KeyRangeIterator extends RangeIterator<Long, Token>
@@ -64,7 +65,7 @@ public class KeyRangeIterator extends RangeIterator<Long, Token>
     public void close() throws IOException
     {}
 
-    private static class DKIterator extends AbstractIterator<DecoratedKey> implements PeekingIterator<DecoratedKey>
+    private static class DKIterator extends AbstractGuavaIterator<DecoratedKey> implements PeekingIterator<DecoratedKey>
     {
         private final Iterator<DecoratedKey> keys;
 

src/java/org/apache/cassandra/index/sasi/utils/RangeIterator.java

@@ -24,7 +24,9 @@ import java.util.PriorityQueue;
 
 import com.google.common.annotations.VisibleForTesting;
 
-public abstract class RangeIterator<K extends Comparable<K>, T extends CombinedValue<K>> extends AbstractIterator<T> implements Closeable
+import org.apache.cassandra.utils.AbstractGuavaIterator;
+
+public abstract class RangeIterator<K extends Comparable<K>, T extends CombinedValue<K>> extends AbstractGuavaIterator<T> implements Closeable
 {
     private final K min, max;
     private final long count;

src/java/org/apache/cassandra/locator/ReplicaPlans.java

@@ -37,6 +37,8 @@ import org.apache.cassandra.dht.AbstractBounds;
 import org.apache.cassandra.dht.Token;
 import org.apache.cassandra.exceptions.UnavailableException;
 import org.apache.cassandra.gms.FailureDetector;
+import org.apache.cassandra.index.Index;
+import org.apache.cassandra.index.IndexStatusManager;
 import org.apache.cassandra.schema.SchemaConstants;
 import org.apache.cassandra.service.StorageService;
 import org.apache.cassandra.service.reads.AlwaysSpeculativeRetryPolicy;
@@ -56,6 +58,8 @@ import java.util.function.Consumer;
 import java.util.function.Function;
 import java.util.function.Predicate;
 
+import javax.annotation.Nullable;
+
 import static com.google.common.collect.Iterables.any;
 import static com.google.common.collect.Iterables.filter;
 import static org.apache.cassandra.db.ConsistencyLevel.EACH_QUORUM;
@@ -522,12 +526,14 @@ public class ReplicaPlans
         return new ReplicaPlan.ForPaxosWrite(keyspace, consistencyForPaxos, liveAndDown.pending(), liveAndDown.all(), live.all(), contacts, requiredParticipants);
     }
 
-
-    private static <E extends Endpoints<E>> E candidatesForRead(ConsistencyLevel consistencyLevel, E liveNaturalReplicas)
+    private static <E extends Endpoints<E>> E candidatesForRead(Keyspace keyspace,
+                                                                @Nullable Index.QueryPlan indexQueryPlan,
+                                                                ConsistencyLevel consistencyLevel,
+                                                                E liveNaturalReplicas)
     {
-        return consistencyLevel.isDatacenterLocal()
-                ? liveNaturalReplicas.filter(InOurDc.replicas())
-                : liveNaturalReplicas;
+        E replicas = consistencyLevel.isDatacenterLocal() ? liveNaturalReplicas.filter(InOurDc.replicas()) : liveNaturalReplicas;
+
+        return indexQueryPlan != null ? IndexStatusManager.instance.filterForQuery(replicas, keyspace, indexQueryPlan, consistencyLevel) : replicas;
     }
 
     private static <E extends Endpoints<E>> E contactForEachQuorumRead(NetworkTopologyStrategy replicationStrategy, E candidates)
@@ -587,10 +593,14 @@ public class ReplicaPlans
      * The candidate collection can be used for speculation, although at present
      * it would break EACH_QUORUM to do so without further filtering
      */
-    public static ReplicaPlan.ForTokenRead forRead(Keyspace keyspace, Token token, ConsistencyLevel consistencyLevel, SpeculativeRetryPolicy retry)
+    public static ReplicaPlan.ForTokenRead forRead(Keyspace keyspace,
+                                                   Token token,
+                                                   @Nullable Index.QueryPlan indexQueryPlan,
+                                                   ConsistencyLevel consistencyLevel,
+                                                   SpeculativeRetryPolicy retry)
     {
         AbstractReplicationStrategy replicationStrategy = keyspace.getReplicationStrategy();
-        EndpointsForToken candidates = candidatesForRead(consistencyLevel, ReplicaLayout.forTokenReadLiveSorted(replicationStrategy, token).natural());
+        EndpointsForToken candidates = candidatesForRead(keyspace, indexQueryPlan, consistencyLevel, ReplicaLayout.forTokenReadLiveSorted(replicationStrategy, token).natural());
         EndpointsForToken contacts = contactForRead(replicationStrategy, consistencyLevel, retry.equals(AlwaysSpeculativeRetryPolicy.INSTANCE), candidates);
 
         assureSufficientLiveReplicasForRead(replicationStrategy, consistencyLevel, contacts);
@@ -604,10 +614,14 @@ public class ReplicaPlans
      *
      * There is no speculation for range read queries at present, so we never 'always speculate' here, and a failed response fails the query.
      */
-    public static ReplicaPlan.ForRangeRead forRangeRead(Keyspace keyspace, ConsistencyLevel consistencyLevel, AbstractBounds<PartitionPosition> range, int vnodeCount)
+    public static ReplicaPlan.ForRangeRead forRangeRead(Keyspace keyspace,
+                                                        @Nullable Index.QueryPlan indexQueryPlan,
+                                                        ConsistencyLevel consistencyLevel,
+                                                        AbstractBounds<PartitionPosition> range,
+                                                        int vnodeCount)
     {
         AbstractReplicationStrategy replicationStrategy = keyspace.getReplicationStrategy();
-        EndpointsForRange candidates = candidatesForRead(consistencyLevel, ReplicaLayout.forRangeReadLiveSorted(replicationStrategy, range).natural());
+        EndpointsForRange candidates = candidatesForRead(keyspace, indexQueryPlan, consistencyLevel, ReplicaLayout.forRangeReadLiveSorted(replicationStrategy, range).natural());
         EndpointsForRange contacts = contactForRead(replicationStrategy, consistencyLevel, false, candidates);
 
         assureSufficientLiveReplicasForRead(replicationStrategy, consistencyLevel, contacts);

src/java/org/apache/cassandra/schema/IndexMetadata.java

@@ -36,8 +36,11 @@ import org.apache.cassandra.cql3.ColumnIdentifier;
 import org.apache.cassandra.cql3.CqlBuilder;
 import org.apache.cassandra.cql3.statements.schema.IndexTarget;
 import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.exceptions.InvalidRequestException;
 import org.apache.cassandra.exceptions.UnknownIndexException;
 import org.apache.cassandra.index.Index;
+import org.apache.cassandra.index.internal.CassandraIndex;
+import org.apache.cassandra.index.sai.StorageAttachedIndex;
 import org.apache.cassandra.index.sasi.SASIIndex;
 import org.apache.cassandra.io.util.DataInputPlus;
 import org.apache.cassandra.io.util.DataOutputPlus;
@@ -64,6 +67,7 @@ public final class IndexMetadata
 
     static
     {
+        indexNameAliases.put(StorageAttachedIndex.class.getSimpleName(), StorageAttachedIndex.class.getCanonicalName());
         indexNameAliases.put(SASIIndex.class.getSimpleName(), SASIIndex.class.getCanonicalName());
     }
 
@@ -135,8 +139,8 @@ public final class IndexMetadata
                 throw new ConfigurationException(String.format("Required option missing for index %s : %s",
                                                                name, IndexTarget.CUSTOM_INDEX_OPTION_NAME));
 
-            // Find any aliases to the fully qualified index class name:
-            String className = expandAliases(options.get(IndexTarget.CUSTOM_INDEX_OPTION_NAME));
+            // Get the fully qualified class name:
+            String className = getIndexClassName();
 
             Class<Index> indexerClass = FBUtilities.classForName(className, "custom indexer");
             if (!Index.class.isAssignableFrom(indexerClass))
@@ -145,9 +149,14 @@ public final class IndexMetadata
         }
     }
 
-    public static String expandAliases(String className)
+    public String getIndexClassName()
     {
-        return indexNameAliases.getOrDefault(className, className);
+        if (isCustom())
+        {
+            String className = options.get(IndexTarget.CUSTOM_INDEX_OPTION_NAME);
+            return indexNameAliases.getOrDefault(className, className);
+        }
+        return CassandraIndex.class.getName();
     }
 
     private void validateCustomIndexOptions(TableMetadata table, Class<? extends Index> indexerClass, Map<String, String> options)
@@ -179,6 +188,8 @@ public final class IndexMetadata
         }
         catch (InvocationTargetException e)
         {
+            if (e.getTargetException() instanceof InvalidRequestException)
+                throw (InvalidRequestException) e.getTargetException();
             if (e.getTargetException() instanceof ConfigurationException)
                 throw (ConfigurationException) e.getTargetException();
             throw new ConfigurationException("Failed to validate custom indexer options: " + options);

src/java/org/apache/cassandra/service/StorageService.java

@@ -148,6 +148,7 @@ import org.apache.cassandra.gms.IFailureDetector;
 import org.apache.cassandra.gms.TokenSerializer;
 import org.apache.cassandra.gms.VersionedValue;
 import org.apache.cassandra.hints.HintsService;
+import org.apache.cassandra.index.IndexStatusManager;
 import org.apache.cassandra.io.sstable.IScrubber;
 import org.apache.cassandra.io.sstable.IVerifier;
 import org.apache.cassandra.io.sstable.SSTableLoader;
@@ -630,7 +631,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
      * they get the Gossip shutdown message, so even if
      * we don't get time to broadcast this, it is not a problem.
      *
-     * See Gossiper.markAsShutdown(InetAddressAndPort)
+     * See {@code Gossiper#markAsShutdown(InetAddressAndPort)}
      */
     private void shutdownClientServers()
     {
@@ -2718,6 +2719,12 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
         }
         else
         {
+            if (state == ApplicationState.INDEX_STATUS)
+            {
+                updateIndexStatus(endpoint, value);
+                return;
+            }
+
             EndpointState epState = Gossiper.instance.getEndpointStateForEndpoint(endpoint);
             if (epState == null || Gossiper.instance.isDeadState(epState))
             {
@@ -2788,6 +2795,11 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
         return value.value.split(VersionedValue.DELIMITER_STR, -1);
     }
 
+    private void updateIndexStatus(InetAddressAndPort endpoint, VersionedValue versionedValue)
+    {
+        IndexStatusManager.instance.receivePeerIndexStatus(endpoint, versionedValue);
+    }
+
     private void updateNetVersion(InetAddressAndPort endpoint, VersionedValue value)
     {
         try
@@ -2860,6 +2872,12 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
                 case HOST_ID:
                     SystemKeyspace.updatePeerInfo(endpoint, "host_id", UUID.fromString(entry.getValue().value));
                     break;
+                case INDEX_STATUS:
+                    // Need to set the peer index status in SIM here
+                    // to ensure the status is correct before the node
+                    // fully joins the ring
+                    updateIndexStatus(endpoint, entry.getValue());
+                    break;
             }
         }
 

src/java/org/apache/cassandra/service/reads/AbstractReadExecutor.java

@@ -181,13 +181,19 @@ public abstract class AbstractReadExecutor
     /**
      * @return an executor appropriate for the configured speculative read policy
      */
-    public static AbstractReadExecutor getReadExecutor(SinglePartitionReadCommand command, ConsistencyLevel consistencyLevel, long queryStartNanoTime) throws UnavailableException
+    public static AbstractReadExecutor getReadExecutor(SinglePartitionReadCommand command,
+                                                       ConsistencyLevel consistencyLevel,
+                                                       long queryStartNanoTime) throws UnavailableException
     {
         Keyspace keyspace = Keyspace.open(command.metadata().keyspace);
         ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(command.metadata().id);
         SpeculativeRetryPolicy retry = cfs.metadata().params.speculativeRetry;
 
-        ReplicaPlan.ForTokenRead replicaPlan = ReplicaPlans.forRead(keyspace, command.partitionKey().getToken(), consistencyLevel, retry);
+        ReplicaPlan.ForTokenRead replicaPlan = ReplicaPlans.forRead(keyspace,
+                                                                    command.partitionKey().getToken(),
+                                                                    command.indexQueryPlan(),
+                                                                    consistencyLevel,
+                                                                    retry);
 
         // Speculative retry is disabled *OR*
         // 11980: Disable speculative retry if using EACH_QUORUM in order to prevent miscounting DC responses

src/java/org/apache/cassandra/service/reads/ReplicaFilteringProtection.java

@@ -523,7 +523,7 @@ public class ReplicaFilteringProtection<E extends Endpoints<E>>
             SinglePartitionReadCommand cmd = SinglePartitionReadCommand.create(command.metadata(),
                                                                                command.nowInSec(),
                                                                                command.columnFilter(),
-                                                                               RowFilter.NONE,
+                                                                               RowFilter.none(),
                                                                                limits,
                                                                                key,
                                                                                filter);

src/java/org/apache/cassandra/service/reads/range/RangeCommands.java

@@ -74,7 +74,10 @@ public class RangeCommands
         Tracing.trace("Computing ranges to query");
 
         Keyspace keyspace = Keyspace.open(command.metadata().keyspace);
-        ReplicaPlanIterator replicaPlans = new ReplicaPlanIterator(command.dataRange().keyRange(), keyspace, consistencyLevel);
+        ReplicaPlanIterator replicaPlans = new ReplicaPlanIterator(command.dataRange().keyRange(),
+                                                                   command.indexQueryPlan(),
+                                                                   keyspace,
+                                                                   consistencyLevel);
 
         // our estimate of how many result rows there will be per-range
         float resultsPerRange = estimateResultsPerRange(command, keyspace);
@@ -128,11 +131,13 @@ public class RangeCommands
         {
             Keyspace keyspace = Keyspace.open(metadata.keyspace);
             ReplicaPlanIterator rangeIterator = new ReplicaPlanIterator(DataRange.allData(metadata.partitioner).keyRange(),
-                                                                        keyspace, consistency);
+                                                                        null,
+                                                                        keyspace,
+                                                                        consistency);
 
             // Called for the side effect of running assureSufficientLiveReplicasForRead.
             // Deliberately called with an invalid vnode count in case it is used elsewhere in the future..
-            rangeIterator.forEachRemaining(r ->  ReplicaPlans.forRangeRead(keyspace, consistency, r.range(), -1));
+            rangeIterator.forEachRemaining(r ->  ReplicaPlans.forRangeRead(keyspace, null, consistency, r.range(), -1));
             return true;
         }
         catch (UnavailableException e)

src/java/org/apache/cassandra/service/reads/range/ReplicaPlanIterator.java

@@ -23,6 +23,8 @@ import java.util.Collections;
 import java.util.Iterator;
 import java.util.List;
 
+import javax.annotation.Nullable;
+
 import com.google.common.annotations.VisibleForTesting;
 
 import org.apache.cassandra.db.ConsistencyLevel;
@@ -31,6 +33,7 @@ import org.apache.cassandra.db.PartitionPosition;
 import org.apache.cassandra.dht.AbstractBounds;
 import org.apache.cassandra.dht.Bounds;
 import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.index.Index;
 import org.apache.cassandra.locator.LocalStrategy;
 import org.apache.cassandra.locator.ReplicaPlan;
 import org.apache.cassandra.locator.ReplicaPlans;
@@ -43,12 +46,17 @@ class ReplicaPlanIterator extends AbstractIterator<ReplicaPlan.ForRangeRead>
 {
     private final Keyspace keyspace;
     private final ConsistencyLevel consistency;
+    private final Index.QueryPlan indexQueryPlan;
     @VisibleForTesting
     final Iterator<? extends AbstractBounds<PartitionPosition>> ranges;
     private final int rangeCount;
 
-    ReplicaPlanIterator(AbstractBounds<PartitionPosition> keyRange, Keyspace keyspace, ConsistencyLevel consistency)
+    ReplicaPlanIterator(AbstractBounds<PartitionPosition> keyRange,
+                        @Nullable Index.QueryPlan indexQueryPlan,
+                        Keyspace keyspace,
+                        ConsistencyLevel consistency)
     {
+        this.indexQueryPlan = indexQueryPlan;
         this.keyspace = keyspace;
         this.consistency = consistency;
 
@@ -73,7 +81,7 @@ class ReplicaPlanIterator extends AbstractIterator<ReplicaPlan.ForRangeRead>
         if (!ranges.hasNext())
             return endOfData();
 
-        return ReplicaPlans.forRangeRead(keyspace, consistency, ranges.next(), 1);
+        return ReplicaPlans.forRangeRead(keyspace, indexQueryPlan, consistency, ranges.next(), 1);
     }
 
     /**

src/java/org/apache/cassandra/utils/AbstractGuavaIterator.java

@@ -1,11 +1,15 @@
 /*
  * Copyright (C) 2007 The Guava Authors
  *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
  *
- * http://www.apache.org/licenses/LICENSE-2.0
+ *     http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
@@ -14,7 +18,7 @@
  * limitations under the License.
  */
 
-package org.apache.cassandra.index.sasi.utils;
+package org.apache.cassandra.utils;
 
 import java.util.NoSuchElementException;
 
@@ -22,15 +26,17 @@ import com.google.common.collect.PeekingIterator;
 
 import static com.google.common.base.Preconditions.checkState;
 
-// This is fork of the Guava AbstractIterator, the only difference
-// is that state & next variables are now protected, this was required
-// for SkippableIterator.skipTo(..) to void all previous state.
-public abstract class AbstractIterator<T> implements PeekingIterator<T>
+/**
+ * This is fork of the Guava AbstractIterator, the only difference
+ * is that the next variable is now protected so that the KeyRangeIterator.skipTo
+ * method can avoid early state changed.
+ */
+public abstract class AbstractGuavaIterator<T> implements PeekingIterator<T>
 {
-    protected State state = State.NOT_READY;
+    private State state = State.NOT_READY;
 
     /** Constructor for use by subclasses. */
-    protected AbstractIterator() {}
+    protected AbstractGuavaIterator() {}
 
     protected enum State
     {

src/java/org/apache/cassandra/utils/ByteBufferUtil.java

@@ -280,6 +280,15 @@ public class ByteBufferUtil
         return clone;
     }
 
+    /**
+     * Transfer bytes from a ByteBuffer to byte array.
+     *
+     * @param src the source ByteBuffer
+     * @param srcPos starting position in the source ByteBuffer
+     * @param dst the destination byte array
+     * @param dstPos starting position in the destination byte array
+     * @param length the number of bytes to copy
+     */
     public static void copyBytes(ByteBuffer src, int srcPos, byte[] dst, int dstPos, int length)
     {
         FastByteOperations.copy(src, srcPos, dst, dstPos, length);

test/distributed/org/apache/cassandra/distributed/test/sai/IndexAvailabilityTest.java

@@ -0,0 +1,288 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.test.sai;
+
+import java.net.InetAddress;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.TimeUnit;
+
+import com.google.common.base.Objects;
+import org.junit.Test;
+
+import org.apache.cassandra.distributed.Cluster;
+import org.apache.cassandra.distributed.api.IInvokableInstance;
+import org.apache.cassandra.distributed.test.TestBaseImpl;
+import org.apache.cassandra.index.Index;
+import org.apache.cassandra.index.IndexStatusManager;
+import org.apache.cassandra.index.SecondaryIndexManager;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.schema.KeyspaceMetadata;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.utils.FBUtilities;
+
+import static org.apache.cassandra.distributed.api.Feature.GOSSIP;
+import static org.apache.cassandra.distributed.api.Feature.NETWORK;
+import static org.apache.cassandra.distributed.test.sai.SAIUtil.waitForIndexQueryable;
+import static org.awaitility.Awaitility.await;
+import static org.junit.Assert.assertEquals;
+
+public class IndexAvailabilityTest extends TestBaseImpl
+{
+    private static final String CREATE_KEYSPACE = "CREATE KEYSPACE %s WITH replication = {'class': 'SimpleStrategy', 'replication_factor': %d}";
+    private static final String CREATE_TABLE = "CREATE TABLE %s.%s (pk text primary key, v1 int, v2 text) " +
+                                               "WITH compaction = {'class' : 'SizeTieredCompactionStrategy', 'enabled' : false }";
+    private static final String CREATE_INDEX = "CREATE CUSTOM INDEX %s ON %s.%s(%s) USING 'StorageAttachedIndex'";
+    
+    private static final Map<NodeIndex, Index.Status> expectedNodeIndexQueryability = new ConcurrentHashMap<>();
+    private List<String> keyspaces;
+    private List<String> indexesPerKs;
+
+    @Test
+    public void verifyIndexStatusPropagation() throws Exception
+    {
+        try (Cluster cluster = init(Cluster.build(2)
+                                           .withConfig(config -> config.with(GOSSIP)
+                                                                       .with(NETWORK))
+                                           .start()))
+        {
+            String ks1 = "ks1";
+            String ks2 = "ks2";
+            String ks3 = "ks3";
+            String cf1 = "cf1";
+            String index1 = "cf1_idx1";
+            String index2 = "cf1_idx2";
+
+            keyspaces = Arrays.asList(ks1, ks2, ks3);
+            indexesPerKs = Arrays.asList(index1, index2);
+
+            // create 1 tables per keyspace, 2 indexes per table. all indexes are queryable
+            for (String ks : keyspaces)
+            {
+                cluster.schemaChange(String.format(CREATE_KEYSPACE, ks, 2));
+                cluster.schemaChange(String.format(CREATE_TABLE, ks, cf1));
+                cluster.schemaChange(String.format(CREATE_INDEX, index1, ks, cf1, "v1"));
+                cluster.schemaChange(String.format(CREATE_INDEX, index2, ks, cf1, "v2"));
+                waitForIndexQueryable(cluster, ks);
+                cluster.forEach(node -> {
+                    expectedNodeIndexQueryability.put(NodeIndex.create(ks, index1, node), Index.Status.BUILD_SUCCEEDED);
+                    expectedNodeIndexQueryability.put(NodeIndex.create(ks, index2, node), Index.Status.BUILD_SUCCEEDED);
+                });
+            }
+
+            // mark ks1 index1 as non-queryable on node1
+            markIndexNonQueryable(cluster.get(1), ks1, cf1, index1);
+            // on node2, it observes that node1 ks1.index1 is not queryable
+            waitForIndexingStatus(cluster.get(2), ks1, index1, cluster.get(1), Index.Status.BUILD_FAILED);
+            // other indexes or keyspaces should not be affected
+            assertIndexingStatus(cluster);
+
+            // mark ks2 index2 as non-queryable on node2
+            markIndexNonQueryable(cluster.get(2), ks2, cf1, index2);
+            // on node1, it observes that node2 ks2.index2 is not queryable
+            waitForIndexingStatus(cluster.get(1), ks2, index2, cluster.get(2), Index.Status.BUILD_FAILED);
+            // other indexes or keyspaces should not be affected
+            assertIndexingStatus(cluster);
+
+            // mark ks1 index1 as queryable on node1
+            markIndexQueryable(cluster.get(1), ks1, cf1, index1);
+            // on node2, it observes that node1 ks1.index1 is queryable
+            waitForIndexingStatus(cluster.get(2), ks1, index1, cluster.get(1), Index.Status.BUILD_SUCCEEDED);
+            // other indexes or keyspaces should not be affected
+            assertIndexingStatus(cluster);
+
+            // mark ks2 index2 as indexing on node1
+            markIndexBuilding(cluster.get(1), ks2, cf1, index2);
+            // on node2, it observes that node1 ks2.index2 is not queryable
+            waitForIndexingStatus(cluster.get(2), ks2, index2, cluster.get(1), Index.Status.FULL_REBUILD_STARTED);
+            // other indexes or keyspaces should not be affected
+            assertIndexingStatus(cluster);
+
+            // drop ks1, ks1 index1/index2 should be non queryable on all nodes
+            cluster.schemaChange("DROP KEYSPACE " + ks1);
+            expectedNodeIndexQueryability.keySet().forEach(k -> {
+                if (k.keyspace.equals(ks1))
+                    expectedNodeIndexQueryability.put(k, Index.Status.UNKNOWN);
+            });
+            assertIndexingStatus(cluster);
+
+            // drop ks2 index2, there should be no ks2 index2 status on all node
+            cluster.schemaChange("DROP INDEX " + ks2 + "." + index2);
+            expectedNodeIndexQueryability.keySet().forEach(k -> {
+                if (k.keyspace.equals(ks2) && k.index.equals(index2))
+                    expectedNodeIndexQueryability.put(k, Index.Status.UNKNOWN);
+            });
+            assertIndexingStatus(cluster);
+
+            // drop ks3 cf1, there should be no ks3 index1/index2 status
+            cluster.schemaChange("DROP TABLE " + ks3 + "." + cf1);
+            expectedNodeIndexQueryability.keySet().forEach(k -> {
+                if (k.keyspace.equals(ks3))
+                    expectedNodeIndexQueryability.put(k, Index.Status.UNKNOWN);
+            });
+            assertIndexingStatus(cluster);
+        }
+    }
+
+    private void markIndexNonQueryable(IInvokableInstance node, String keyspace, String table, String indexName)
+    {
+        expectedNodeIndexQueryability.put(NodeIndex.create(keyspace, indexName, node), Index.Status.BUILD_FAILED);
+
+        node.runOnInstance(() -> {
+            SecondaryIndexManager sim = Schema.instance.getKeyspaceInstance(keyspace).getColumnFamilyStore(table).indexManager;
+            Index index = sim.getIndexByName(indexName);
+            sim.makeIndexNonQueryable(index, Index.Status.BUILD_FAILED);
+        });
+    }
+
+    private void markIndexQueryable(IInvokableInstance node, String keyspace, String table, String indexName)
+    {
+        expectedNodeIndexQueryability.put(NodeIndex.create(keyspace, indexName, node), Index.Status.BUILD_SUCCEEDED);
+
+        node.runOnInstance(() -> {
+            SecondaryIndexManager sim = Schema.instance.getKeyspaceInstance(keyspace).getColumnFamilyStore(table).indexManager;
+            Index index = sim.getIndexByName(indexName);
+            sim.makeIndexQueryable(index, Index.Status.BUILD_SUCCEEDED);
+        });
+    }
+
+    private void markIndexBuilding(IInvokableInstance node, String keyspace, String table, String indexName)
+    {
+        expectedNodeIndexQueryability.put(NodeIndex.create(keyspace, indexName, node), Index.Status.FULL_REBUILD_STARTED);
+
+        node.runOnInstance(() -> {
+            SecondaryIndexManager sim = Schema.instance.getKeyspaceInstance(keyspace).getColumnFamilyStore(table).indexManager;
+            Index index = sim.getIndexByName(indexName);
+            sim.markIndexesBuilding(Collections.singleton(index), true, false);
+        });
+    }
+
+    private void assertIndexingStatus(Cluster cluster)
+    {
+        for (String ks : keyspaces)
+        {
+            for (String indexName : indexesPerKs)
+            {
+                assertIndexingStatus(cluster, ks, indexName);
+            }
+        }
+    }
+
+    private static void assertIndexingStatus(Cluster cluster, String keyspace, String indexName)
+    {
+        for (int nodeId = 1; nodeId <= cluster.size(); nodeId++)
+        {
+            for (int replica = 1; replica <= cluster.size(); replica++)
+            {
+                NodeIndex nodeIndex = NodeIndex.create(keyspace, indexName, cluster.get(replica));
+                Index.Status expected = expectedNodeIndexQueryability.get(nodeIndex);
+
+                assertIndexingStatus(cluster.get(nodeId), keyspace, indexName, cluster.get(replica), expected);
+            }
+        }
+    }
+
+    private static void assertIndexingStatus(IInvokableInstance node, String keyspaceName, String indexName, IInvokableInstance replica, Index.Status expected)
+    {
+        InetAddressAndPort replicaAddressAndPort = getFullAddress(replica);
+        try
+        {
+            Index.Status actual = getNodeIndexStatus(node, keyspaceName, indexName, replicaAddressAndPort);
+            String errorMessage = String.format("Failed to verify %s.%s status for replica %s on node %s, expected %s, but got %s.",
+                                                keyspaceName, indexName, replica.broadcastAddress(), node.broadcastAddress(), expected, actual);
+            assertEquals(errorMessage, expected, actual);
+        }
+        catch (Exception e)
+        {
+            throw new RuntimeException(e);
+        }
+    }
+
+    private static void waitForIndexingStatus(IInvokableInstance node, String keyspace, String index, IInvokableInstance replica, Index.Status status)
+    {
+        InetAddressAndPort replicaAddressAndPort = getFullAddress(replica);
+        await().atMost(5, TimeUnit.SECONDS)
+               .until(() -> node.callOnInstance(() -> getIndexStatus(keyspace, index, replicaAddressAndPort) == status));
+    }
+
+    private static Index.Status getNodeIndexStatus(IInvokableInstance node, String keyspaceName, String indexName, InetAddressAndPort replica)
+    {
+        return Index.Status.values()[node.callsOnInstance(() -> getIndexStatus(keyspaceName, indexName, replica).ordinal()).call()];
+    }
+    
+    private static Index.Status getIndexStatus(String keyspaceName, String indexName, InetAddressAndPort replica)
+    {
+        KeyspaceMetadata keyspace = Schema.instance.getKeyspaceMetadata(keyspaceName);
+        if (keyspace == null)
+            return Index.Status.UNKNOWN;
+
+        TableMetadata table = keyspace.findIndexedTable(indexName).orElse(null);
+        if (table == null)
+            return Index.Status.UNKNOWN;
+
+        return IndexStatusManager.instance.getIndexStatus(replica, keyspaceName, indexName);
+    }
+
+    private static InetAddressAndPort getFullAddress(IInvokableInstance node)
+    {
+        InetAddress address = node.broadcastAddress().getAddress();
+        int port = node.callOnInstance(() -> FBUtilities.getBroadcastAddressAndPort().getPort());
+        return InetAddressAndPort.getByAddressOverrideDefaults(address, port);
+    }
+    
+    private static class NodeIndex
+    {
+        private final String keyspace;
+        private final String index;
+        private final IInvokableInstance node;
+
+        NodeIndex(String keyspace, String index, IInvokableInstance node)
+        {
+            this.keyspace = keyspace;
+            this.index = index;
+            this.node = node;
+        }
+
+        public static NodeIndex create(String keyspace, String index, IInvokableInstance node)
+        {
+            return new NodeIndex(keyspace, index, node);
+        }
+
+        @Override
+        public boolean equals(Object o)
+        {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            NodeIndex that = (NodeIndex) o;
+            return node.equals(that.node) &&
+                   Objects.equal(keyspace, that.keyspace) &&
+                   Objects.equal(index, that.index);
+        }
+
+        @Override
+        public int hashCode()
+        {
+            return Objects.hashCode(keyspace, index, node);
+        }
+    }
+}

test/distributed/org/apache/cassandra/distributed/test/sai/SAIUtil.java

@@ -0,0 +1,141 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.test.sai;
+
+import java.net.InetSocketAddress;
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+
+import org.apache.cassandra.distributed.Cluster;
+import org.apache.cassandra.distributed.api.Feature;
+import org.apache.cassandra.distributed.api.IInstance;
+import org.apache.cassandra.distributed.api.IInvokableInstance;
+import org.apache.cassandra.distributed.api.SimpleQueryResult;
+import org.apache.cassandra.index.Index;
+import org.apache.cassandra.index.IndexStatusManager;
+import org.apache.cassandra.index.SecondaryIndexManager;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.assertj.core.util.Streams;
+
+import static org.awaitility.Awaitility.await;
+
+public class SAIUtil
+{
+    /**
+     * Waits until all indexes in the given keyspace become queryable.
+     */
+    public static void waitForIndexQueryable(Cluster cluster, String keyspace)
+    {
+        assertGossipEnabled(cluster);
+        final List<String> indexes = getIndexes(cluster, keyspace);
+        await().atMost(60, TimeUnit.SECONDS)
+               .untilAsserted(() -> assertIndexesQueryable(cluster, keyspace, indexes));
+    }
+
+    /**
+     * Waits until given index becomes queryable.
+     */
+    public static void waitForIndexQueryable(Cluster cluster, String keyspace, String index)
+    {
+        assertGossipEnabled(cluster);
+        await().atMost(60, TimeUnit.SECONDS)
+               .untilAsserted(() -> assertIndexQueryable(cluster, keyspace, index));
+    }
+
+    private static void assertGossipEnabled(Cluster cluster)
+    {
+        cluster.stream().forEach(node -> {
+            assert node.config().has(Feature.NETWORK) : "Network not enabled on this cluster";
+            assert node.config().has(Feature.GOSSIP) : "Gossip not enabled on this cluster";
+        });
+    }
+
+    /**
+     * Checks if index is known to be queryable, by pulling index state from {{@link SecondaryIndexManager}}.
+     * Requires gossip.
+     */
+    public static void assertIndexQueryable(Cluster cluster, String keyspace, String index)
+    {
+        assertIndexesQueryable(cluster, keyspace, Collections.singleton(index));
+    }
+
+    /**
+     * Checks if all indexes are known to be queryable, by pulling index state from local {{@link SecondaryIndexManager}}.
+     * Requires gossip.
+     */
+    private static void assertIndexesQueryable(Cluster cluster, String keyspace, final Iterable<String> indexes)
+    {
+        IInvokableInstance localNode = cluster.get(1);
+        final List<InetAddressAndPort> nodes =
+            cluster.stream()
+                   .map(node -> nodeAddress(node.broadcastAddress()))
+                   .collect(Collectors.toList());
+
+        localNode.runOnInstance(() -> {
+            for (String index : indexes)
+            {
+                for (InetAddressAndPort node : nodes)
+                {
+                    Index.Status status = IndexStatusManager.instance.getIndexStatus(node, keyspace, index);
+                    assert status == Index.Status.BUILD_SUCCEEDED
+                        : "Index " + index + " not queryable on node " + node + " (status = " + status + ')';
+                }
+            }
+        });
+    }
+
+    private static InetAddressAndPort nodeAddress(InetSocketAddress address)
+    {
+        return InetAddressAndPort.getByAddressOverrideDefaults(address.getAddress(), address.getPort());
+    }
+
+    /**
+     * Returns names of the indexes in the keyspace, found on the first node of the cluster.
+     */
+    public static List<String> getIndexes(Cluster cluster, String keyspace)
+    {
+        waitForSchemaAgreement(cluster);
+        String query = String.format("SELECT index_name FROM system_views.indexes WHERE keyspace_name = '%s' ALLOW FILTERING", keyspace);
+        SimpleQueryResult result = cluster.get(1).executeInternalWithResult(query);
+        return Streams.stream(result)
+                      .map(row -> (String) row.get("index_name"))
+                      .collect(Collectors.toList());
+    }
+
+    public static void waitForSchemaAgreement(Cluster cluster)
+    {
+        await().atMost(60, TimeUnit.SECONDS)
+               .until(() -> schemaAgrees(cluster));
+    }
+
+    /**
+     * Returns true if schema agrees on all nodes of the cluster
+     */
+    public static boolean schemaAgrees(Cluster cluster)
+    {
+        Set<UUID> versions = cluster.stream()
+                                    .map(IInstance::schemaVersion)
+                                    .collect(Collectors.toSet());
+        return versions.size() == 1;
+    }
+}

test/unit/org/apache/cassandra/cql3/CQLTester.java

@@ -2570,6 +2570,21 @@ public abstract class CQLTester
         {
             return "TupleValue" + toCQLString();
         }
+
+        @Override
+        public boolean equals(Object o)
+        {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            TupleValue that = (TupleValue) o;
+            return Arrays.equals(values, that.values);
+        }
+
+        @Override
+        public int hashCode()
+        {
+            return Objects.hashCode(values);
+        }
     }
 
     private static class UserTypeValue extends TupleValue

test/unit/org/apache/cassandra/db/KeyspaceTest.java

@@ -297,7 +297,7 @@ public class KeyspaceTest extends CQLTester
                          ? DataLimits.NONE
                          : DataLimits.cqlLimits(rowLimit);
         return SinglePartitionReadCommand.create(
-                cfs.metadata(), FBUtilities.nowInSeconds(), ColumnFilter.all(cfs.metadata()), RowFilter.NONE, limit, Util.dk(key), filter);
+                cfs.metadata(), FBUtilities.nowInSeconds(), ColumnFilter.all(cfs.metadata()), RowFilter.none(), limit, Util.dk(key), filter);
     }
 
     @Test
@@ -483,17 +483,17 @@ public class KeyspaceTest extends CQLTester
     {
         ClusteringIndexSliceFilter filter = slices(cfs, 1000, null, false);
         SinglePartitionReadCommand command = SinglePartitionReadCommand.create(
-                cfs.metadata(), FBUtilities.nowInSeconds(), ColumnFilter.all(cfs.metadata()), RowFilter.NONE, DataLimits.cqlLimits(3), Util.dk("0"), filter);
+                cfs.metadata(), FBUtilities.nowInSeconds(), ColumnFilter.all(cfs.metadata()), RowFilter.none(), DataLimits.cqlLimits(3), Util.dk("0"), filter);
         assertRowsInResult(cfs, command, 1000, 1001, 1002);
 
         filter = slices(cfs, 1195, null, false);
         command = SinglePartitionReadCommand.create(
-                cfs.metadata(), FBUtilities.nowInSeconds(), ColumnFilter.all(cfs.metadata()), RowFilter.NONE, DataLimits.cqlLimits(3), Util.dk("0"), filter);
+                cfs.metadata(), FBUtilities.nowInSeconds(), ColumnFilter.all(cfs.metadata()), RowFilter.none(), DataLimits.cqlLimits(3), Util.dk("0"), filter);
         assertRowsInResult(cfs, command, 1195, 1196, 1197);
 
         filter = slices(cfs, null, 1996, true);
         command = SinglePartitionReadCommand.create(
-                cfs.metadata(), FBUtilities.nowInSeconds(), ColumnFilter.all(cfs.metadata()), RowFilter.NONE, DataLimits.cqlLimits(1000), Util.dk("0"), filter);
+                cfs.metadata(), FBUtilities.nowInSeconds(), ColumnFilter.all(cfs.metadata()), RowFilter.none(), DataLimits.cqlLimits(1000), Util.dk("0"), filter);
         int[] expectedValues = new int[997];
         for (int i = 0, v = 1996; v >= 1000; i++, v--)
             expectedValues[i] = v;
@@ -501,22 +501,22 @@ public class KeyspaceTest extends CQLTester
 
         filter = slices(cfs, 1990, null, false);
         command = SinglePartitionReadCommand.create(
-                cfs.metadata(), FBUtilities.nowInSeconds(), ColumnFilter.all(cfs.metadata()), RowFilter.NONE, DataLimits.cqlLimits(3), Util.dk("0"), filter);
+                cfs.metadata(), FBUtilities.nowInSeconds(), ColumnFilter.all(cfs.metadata()), RowFilter.none(), DataLimits.cqlLimits(3), Util.dk("0"), filter);
         assertRowsInResult(cfs, command, 1990, 1991, 1992);
 
         filter = slices(cfs, null, null, true);
         command = SinglePartitionReadCommand.create(
-                cfs.metadata(), FBUtilities.nowInSeconds(), ColumnFilter.all(cfs.metadata()), RowFilter.NONE, DataLimits.cqlLimits(3), Util.dk("0"), filter);
+                cfs.metadata(), FBUtilities.nowInSeconds(), ColumnFilter.all(cfs.metadata()), RowFilter.none(), DataLimits.cqlLimits(3), Util.dk("0"), filter);
         assertRowsInResult(cfs, command, 1999, 1998, 1997);
 
         filter = slices(cfs, null, 9000, true);
         command = SinglePartitionReadCommand.create(
-                cfs.metadata(), FBUtilities.nowInSeconds(), ColumnFilter.all(cfs.metadata()), RowFilter.NONE, DataLimits.cqlLimits(3), Util.dk("0"), filter);
+                cfs.metadata(), FBUtilities.nowInSeconds(), ColumnFilter.all(cfs.metadata()), RowFilter.none(), DataLimits.cqlLimits(3), Util.dk("0"), filter);
         assertRowsInResult(cfs, command, 1999, 1998, 1997);
 
         filter = slices(cfs, 9000, null, false);
         command = SinglePartitionReadCommand.create(
-                cfs.metadata(), FBUtilities.nowInSeconds(), ColumnFilter.all(cfs.metadata()), RowFilter.NONE, DataLimits.cqlLimits(3), Util.dk("0"), filter);
+                cfs.metadata(), FBUtilities.nowInSeconds(), ColumnFilter.all(cfs.metadata()), RowFilter.none(), DataLimits.cqlLimits(3), Util.dk("0"), filter);
         assertRowsInResult(cfs, command);
     }
 

test/unit/org/apache/cassandra/db/ReadCommandVerbHandlerTest.java

@@ -166,7 +166,7 @@ public class ReadCommandVerbHandlerTest
                   metadata,
                   FBUtilities.nowInSeconds(),
                   ColumnFilter.all(metadata),
-                  RowFilter.NONE,
+                  RowFilter.none(),
                   DataLimits.NONE,
                   KEY,
                   new ClusteringIndexSliceFilter(Slices.ALL, false),

test/unit/org/apache/cassandra/db/ReadResponseTest.java

@@ -258,7 +258,7 @@ public class ReadResponseTest
                   metadata,
                   FBUtilities.nowInSeconds(),
                   ColumnFilter.all(metadata),
-                  RowFilter.NONE,
+                  RowFilter.none(),
                   DataLimits.NONE,
                   metadata.partitioner.decorateKey(ByteBufferUtil.bytes(key)),
                   null,

test/unit/org/apache/cassandra/db/SinglePartitionSliceCommandTest.java

@@ -186,7 +186,7 @@ public class SinglePartitionSliceCommandTest
         ReadCommand cmd = SinglePartitionReadCommand.create(CFM_SLICES,
                                                             FBUtilities.nowInSeconds(),
                                                             ColumnFilter.all(CFM_SLICES),
-                                                            RowFilter.NONE,
+                                                            RowFilter.none(),
                                                             DataLimits.NONE,
                                                             key,
                                                             clusteringFilter);
@@ -247,7 +247,7 @@ public class SinglePartitionSliceCommandTest
         ReadCommand cmd = SinglePartitionReadCommand.create(metadata,
                                                             FBUtilities.nowInSeconds(),
                                                             columnFilter,
-                                                            RowFilter.NONE,
+                                                            RowFilter.none(),
                                                             DataLimits.NONE,
                                                             key,
                                                             sliceFilter);
@@ -475,7 +475,7 @@ public class SinglePartitionSliceCommandTest
         ReadCommand cmd = SinglePartitionReadCommand.create(metadata,
                                                             FBUtilities.nowInSeconds(),
                                                             columnFilter,
-                                                            RowFilter.NONE,
+                                                            RowFilter.none(),
                                                             DataLimits.NONE,
                                                             key,
                                                             sliceFilter);

test/unit/org/apache/cassandra/db/compaction/CompactionsTest.java

@@ -352,7 +352,7 @@ public class CompactionsTest
         for (FilteredPartition p : Util.getAll(Util.cmd(cfs).build()))
         {
             k.add(p.partitionKey());
-            final SinglePartitionReadCommand command = SinglePartitionReadCommand.create(cfs.metadata(), FBUtilities.nowInSeconds(), ColumnFilter.all(cfs.metadata()), RowFilter.NONE, DataLimits.NONE, p.partitionKey(), new ClusteringIndexSliceFilter(Slices.ALL, false));
+            final SinglePartitionReadCommand command = SinglePartitionReadCommand.create(cfs.metadata(), FBUtilities.nowInSeconds(), ColumnFilter.all(cfs.metadata()), RowFilter.none(), DataLimits.NONE, p.partitionKey(), new ClusteringIndexSliceFilter(Slices.ALL, false));
             try (ReadExecutionController executionController = command.executionController();
                  PartitionIterator iterator = command.executeInternal(executionController))
             {

test/unit/org/apache/cassandra/db/filter/RowFilterTest.java

@@ -63,7 +63,7 @@ public class RowFilterTest
         ColumnMetadata r = metadata.getColumn(new ColumnIdentifier("r", true));
 
         ByteBuffer one = Int32Type.instance.decompose(1);
-        RowFilter filter = RowFilter.NONE.withNewExpressions(new ArrayList<>());
+        RowFilter filter = RowFilter.none().withNewExpressions(new ArrayList<>());
         filter.add(s, Operator.NEQ, one);
         AtomicBoolean closed = new AtomicBoolean();
         UnfilteredPartitionIterator iter = filter.filter(new SingletonUnfilteredPartitionIterator(new UnfilteredRowIterator()
@@ -91,7 +91,7 @@ public class RowFilterTest
         Assert.assertFalse(iter.hasNext());
         Assert.assertTrue(closed.get());
 
-        filter = RowFilter.NONE.withNewExpressions(new ArrayList<>());
+        filter = RowFilter.none().withNewExpressions(new ArrayList<>());
         filter.add(r, Operator.NEQ, one);
         closed.set(false);
         iter = filter.filter(new SingletonUnfilteredPartitionIterator(new UnfilteredRowIterator()

test/unit/org/apache/cassandra/db/rows/UnfilteredRowIteratorWithLowerBoundTest.java

@@ -245,7 +245,7 @@ public class UnfilteredRowIteratorWithLowerBoundTest
         SinglePartitionReadCommand cmd = SinglePartitionReadCommand.create(metadata,
                                                                            FBUtilities.nowInSeconds(),
                                                                            ColumnFilter.all(metadata),
-                                                                           RowFilter.NONE,
+                                                                           RowFilter.none(),
                                                                            DataLimits.NONE,
                                                                            key,
                                                                            filter);
@@ -260,4 +260,4 @@ public class UnfilteredRowIteratorWithLowerBoundTest
             return iter.lowerBound() != null;
         }
     }
-}
+}

test/unit/org/apache/cassandra/db/virtual/GossipInfoTableTest.java

@@ -77,7 +77,7 @@ public class GossipInfoTableTest extends CQLTester
 
             assertThat(resultSet.size()).isEqualTo(1);
             UntypedResultSet.Row row = resultSet.one();
-            assertThat(row.getColumns().size()).isEqualTo(64);
+            assertThat(row.getColumns().size()).isEqualTo(66);
 
             assertThat(endpoint).isNotNull();
             assertThat(localState).isNotNull();

test/unit/org/apache/cassandra/dht/KeyCollisionTest.java

@@ -128,6 +128,12 @@ public class KeyCollisionTest
             return 0;
         }
 
+        @Override
+        public long getLongValue()
+        {
+            return token.longValue();
+        }
+
         @Override
         public ByteSource asComparableBytes(ByteComparable.Version version)
         {

test/unit/org/apache/cassandra/index/SecondaryIndexManagerTest.java

@@ -50,7 +50,6 @@ import static org.junit.Assert.fail;
 
 public class SecondaryIndexManagerTest extends CQLTester
 {
-
     @After
     public void after()
     {

test/unit/org/apache/cassandra/index/sai/IndexingSchemaLoader.java

@@ -0,0 +1,173 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai;
+
+import java.util.HashMap;
+
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.cql3.statements.schema.IndexTarget;
+import org.apache.cassandra.db.marshal.DoubleType;
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.db.marshal.LongType;
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.schema.IndexMetadata;
+import org.apache.cassandra.schema.Indexes;
+import org.apache.cassandra.schema.TableMetadata;
+
+public class IndexingSchemaLoader extends SchemaLoader
+{
+    public static TableMetadata.Builder skinnySAITableMetadata(String ksName, String cfName)
+    {
+        TableMetadata.Builder builder =
+                TableMetadata.builder(ksName, cfName)
+                             .addPartitionKeyColumn("id", UTF8Type.instance)
+                             .addRegularColumn("first_name", UTF8Type.instance)
+                             .addRegularColumn("last_name", UTF8Type.instance)
+                             .addRegularColumn("age", Int32Type.instance)
+                             .addRegularColumn("height", Int32Type.instance)
+                             .addRegularColumn("timestamp", LongType.instance)
+                             .addRegularColumn("address", UTF8Type.instance)
+                             .addRegularColumn("score", DoubleType.instance)
+                             .addRegularColumn("comment", UTF8Type.instance)
+                             .addRegularColumn("comment_suffix_split", UTF8Type.instance)
+                             .addRegularColumn("/output/full-name/", UTF8Type.instance)
+                             .addRegularColumn("/data/output/id", UTF8Type.instance)
+                             .addRegularColumn("first_name_prefix", UTF8Type.instance);
+
+        Indexes.Builder indexes = Indexes.builder();
+
+        indexes.add(IndexMetadata.fromSchemaMetadata(cfName + "_first_name", IndexMetadata.Kind.CUSTOM, new HashMap<String, String>()
+        {{
+            put(IndexTarget.CUSTOM_INDEX_OPTION_NAME, StorageAttachedIndex.class.getName());
+            put(IndexTarget.TARGET_OPTION_NAME, "first_name");
+        }}))
+                .add(IndexMetadata.fromSchemaMetadata(cfName + "_last_name", IndexMetadata.Kind.CUSTOM, new HashMap<String, String>()
+                {{
+                    put(IndexTarget.CUSTOM_INDEX_OPTION_NAME, StorageAttachedIndex.class.getName());
+                    put(IndexTarget.TARGET_OPTION_NAME, "last_name");
+                }}))
+                .add(IndexMetadata.fromSchemaMetadata(cfName + "_age", IndexMetadata.Kind.CUSTOM, new HashMap<String, String>()
+                {{
+                    put(IndexTarget.CUSTOM_INDEX_OPTION_NAME, StorageAttachedIndex.class.getName());
+                    put(IndexTarget.TARGET_OPTION_NAME, "age");
+                }}))
+                .add(IndexMetadata.fromSchemaMetadata(cfName + "_timestamp", IndexMetadata.Kind.CUSTOM, new HashMap<String, String>()
+                {{
+                    put(IndexTarget.CUSTOM_INDEX_OPTION_NAME, StorageAttachedIndex.class.getName());
+                    put(IndexTarget.TARGET_OPTION_NAME, "timestamp");
+
+                }}))
+                .add(IndexMetadata.fromSchemaMetadata(cfName + "_address", IndexMetadata.Kind.CUSTOM, new HashMap<String, String>()
+                {{
+                    put(IndexTarget.CUSTOM_INDEX_OPTION_NAME, StorageAttachedIndex.class.getName());
+                    put(IndexTarget.TARGET_OPTION_NAME, "address");
+                }}))
+                .add(IndexMetadata.fromSchemaMetadata(cfName + "_score", IndexMetadata.Kind.CUSTOM, new HashMap<String, String>()
+                {{
+                    put(IndexTarget.CUSTOM_INDEX_OPTION_NAME, StorageAttachedIndex.class.getName());
+                    put(IndexTarget.TARGET_OPTION_NAME, "score");
+                }}))
+                .add(IndexMetadata.fromSchemaMetadata(cfName + "_comment", IndexMetadata.Kind.CUSTOM, new HashMap<String, String>()
+                {{
+                    put(IndexTarget.CUSTOM_INDEX_OPTION_NAME, StorageAttachedIndex.class.getName());
+                    put(IndexTarget.TARGET_OPTION_NAME, "comment");
+                }}))
+                .add(IndexMetadata.fromSchemaMetadata(cfName + "_comment_suffix_split", IndexMetadata.Kind.CUSTOM, new HashMap<String, String>()
+                {{
+                    put(IndexTarget.CUSTOM_INDEX_OPTION_NAME, StorageAttachedIndex.class.getName());
+                    put(IndexTarget.TARGET_OPTION_NAME, "comment_suffix_split");
+                }}))
+                .add(IndexMetadata.fromSchemaMetadata(cfName + "_output_full_name", IndexMetadata.Kind.CUSTOM, new HashMap<String, String>()
+                {{
+                    put(IndexTarget.CUSTOM_INDEX_OPTION_NAME, StorageAttachedIndex.class.getName());
+                    put(IndexTarget.TARGET_OPTION_NAME, "/output/full-name/");
+                }}))
+                .add(IndexMetadata.fromSchemaMetadata(cfName + "_data_output_id", IndexMetadata.Kind.CUSTOM, new HashMap<String, String>()
+                {{
+                    put(IndexTarget.CUSTOM_INDEX_OPTION_NAME, StorageAttachedIndex.class.getName());
+                    put(IndexTarget.TARGET_OPTION_NAME, "/data/output/id");
+                }}))
+                .add(IndexMetadata.fromSchemaMetadata(cfName + "_first_name_prefix", IndexMetadata.Kind.CUSTOM, new HashMap<String, String>()
+                {{
+                    put(IndexTarget.CUSTOM_INDEX_OPTION_NAME, StorageAttachedIndex.class.getName());
+                    put(IndexTarget.TARGET_OPTION_NAME, "first_name_prefix");
+                }}));
+
+        return builder.indexes(indexes.build());
+    }
+
+    public static TableMetadata.Builder clusteringSAITableMetadata(String ksName, String cfName)
+    {
+        return clusteringSAITableMetadata(ksName, cfName, "location", "age", "height", "score");
+    }
+
+    public static TableMetadata.Builder clusteringSAITableMetadata(String ksName, String cfName, String...indexedColumns)
+    {
+        Indexes.Builder indexes = Indexes.builder();
+        for (String indexedColumn : indexedColumns)
+        {
+            indexes.add(IndexMetadata.fromSchemaMetadata(cfName + "_" + indexedColumn, IndexMetadata.Kind.CUSTOM, new HashMap<String, String>()
+            {{
+                put(IndexTarget.CUSTOM_INDEX_OPTION_NAME, StorageAttachedIndex.class.getName());
+                put(IndexTarget.TARGET_OPTION_NAME, indexedColumn);
+            }}));
+        }
+
+        return TableMetadata.builder(ksName, cfName)
+                            .addPartitionKeyColumn("name", UTF8Type.instance)
+                            .addClusteringColumn("location", UTF8Type.instance)
+                            .addClusteringColumn("age", Int32Type.instance)
+                            .addRegularColumn("height", Int32Type.instance)
+                            .addRegularColumn("score", DoubleType.instance)
+                            .addStaticColumn("nickname", UTF8Type.instance)
+                            .indexes(indexes.build());
+    }
+
+    public static TableMetadata.Builder staticSAITableMetadata(String ksName, String cfName)
+    {
+        TableMetadata.Builder builder =
+                TableMetadata.builder(ksName, cfName)
+                             .addPartitionKeyColumn("sensor_id", Int32Type.instance)
+                             .addStaticColumn("sensor_type", UTF8Type.instance)
+                             .addClusteringColumn("date", LongType.instance)
+                             .addRegularColumn("value", DoubleType.instance)
+                             .addRegularColumn("variance", Int32Type.instance);
+
+        Indexes.Builder indexes = Indexes.builder();
+
+        indexes.add(IndexMetadata.fromSchemaMetadata(cfName + "_sensor_type", IndexMetadata.Kind.CUSTOM, new HashMap<String, String>()
+        {{
+            put(IndexTarget.CUSTOM_INDEX_OPTION_NAME, StorageAttachedIndex.class.getName());
+            put(IndexTarget.TARGET_OPTION_NAME, "sensor_type");
+        }}));
+
+        indexes.add(IndexMetadata.fromSchemaMetadata(cfName + "_value", IndexMetadata.Kind.CUSTOM, new HashMap<String, String>()
+        {{
+            put(IndexTarget.CUSTOM_INDEX_OPTION_NAME, StorageAttachedIndex.class.getName());
+            put(IndexTarget.TARGET_OPTION_NAME, "value");
+        }}));
+
+        indexes.add(IndexMetadata.fromSchemaMetadata(cfName + "_variance", IndexMetadata.Kind.CUSTOM, new HashMap<String, String>()
+        {{
+            put(IndexTarget.CUSTOM_INDEX_OPTION_NAME, StorageAttachedIndex.class.getName());
+            put(IndexTarget.TARGET_OPTION_NAME, "variance");
+        }}));
+
+        return builder.indexes(indexes.build());
+    }
+}

test/unit/org/apache/cassandra/index/sai/SAITester.java

@@ -0,0 +1,363 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+package org.apache.cassandra.index.sai;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+import javax.management.AttributeNotFoundException;
+import javax.management.ObjectName;
+
+import org.junit.After;
+import org.junit.Rule;
+import org.junit.rules.TestRule;
+import org.junit.rules.TestWatcher;
+import org.junit.runner.Description;
+
+import com.carrotsearch.randomizedtesting.generators.RandomInts;
+import com.carrotsearch.randomizedtesting.generators.RandomStrings;
+import com.datastax.driver.core.Session;
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.cql3.UntypedResultSet;
+import org.apache.cassandra.cql3.statements.schema.IndexTarget;
+import org.apache.cassandra.db.ClusteringComparator;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Directories;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.compaction.CompactionManager;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.index.Index;
+import org.apache.cassandra.index.sai.utils.PrimaryKey;
+import org.apache.cassandra.index.sai.utils.PrimaryKeyFactory;
+import org.apache.cassandra.index.sai.utils.ResourceLeakDetector;
+import org.apache.cassandra.inject.Injections;
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.IndexMetadata;
+import org.apache.cassandra.service.snapshot.TableSnapshot;
+import org.apache.cassandra.utils.Throwables;
+import org.awaitility.Awaitility;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public abstract class SAITester extends CQLTester
+{
+    protected static final String CREATE_KEYSPACE_TEMPLATE = "CREATE KEYSPACE IF NOT EXISTS %s WITH replication = " +
+                                                             "{'class': 'SimpleStrategy', 'replication_factor': '1'}";
+
+    protected static final String CREATE_TABLE_TEMPLATE = "CREATE TABLE %s (id1 TEXT PRIMARY KEY, v1 INT, v2 TEXT) WITH compaction = " +
+                                                          "{'class' : 'SizeTieredCompactionStrategy', 'enabled' : false }";
+    protected static final String CREATE_INDEX_TEMPLATE = "CREATE CUSTOM INDEX IF NOT EXISTS ON %%s(%s) USING 'StorageAttachedIndex'";
+
+    private static Randomization random;
+
+    public static final ClusteringComparator EMPTY_COMPARATOR = new ClusteringComparator();
+
+    public static final PrimaryKeyFactory TEST_FACTORY = PrimaryKey.factory(EMPTY_COMPARATOR);
+
+    @Rule
+    public TestRule testRules = new ResourceLeakDetector();
+
+    @Rule
+    public FailureWatcher failureRule = new FailureWatcher();
+
+    @After
+    public void removeAllInjections()
+    {
+        Injections.deleteAll();
+    }
+
+    public static Randomization getRandom()
+    {
+        if (random == null)
+            random = new Randomization();
+        return random;
+    }
+
+    public static IndexContext createIndexContext(String name, AbstractType<?> validator)
+    {
+        return new IndexContext("test_ks",
+                                "test_cf",
+                                UTF8Type.instance,
+                                new ClusteringComparator(),
+                                ColumnMetadata.regularColumn("sai", "internal", name, validator),
+                                IndexTarget.Type.SIMPLE,
+                                IndexMetadata.fromSchemaMetadata(name, IndexMetadata.Kind.CUSTOM, null));
+    }
+
+    protected void waitForAssert(Runnable runnableAssert, long timeout, TimeUnit unit)
+    {
+        Awaitility.await().dontCatchUncaughtExceptions().atMost(timeout, unit).untilAsserted(runnableAssert::run);
+    }
+
+    protected boolean isIndexQueryable(String keyspace, String table)
+    {
+        ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
+        for (Index index : cfs.indexManager.listIndexes())
+        {
+            if (!cfs.indexManager.isIndexQueryable(index))
+                return false;
+        }
+        return true;
+    }
+
+    protected Object getMBeanAttribute(ObjectName name, String attribute) throws Exception
+    {
+        return jmxConnection.getAttribute(name, attribute);
+    }
+
+    protected Object getMetricValue(ObjectName metricObjectName)
+    {
+        // lets workaround the fact that gauges have Value, but counters have Count
+        Object metricValue;
+        try
+        {
+            try
+            {
+                metricValue = getMBeanAttribute(metricObjectName, "Value");
+            }
+            catch (AttributeNotFoundException ignored)
+            {
+                metricValue = getMBeanAttribute(metricObjectName, "Count");
+            }
+        }
+        catch (Exception e)
+        {
+            throw new RuntimeException(e);
+        }
+        return metricValue;
+    }
+
+    public void waitForIndexQueryable()
+    {
+        waitForIndexQueryable(KEYSPACE, currentTable());
+    }
+
+    public void waitForIndexQueryable(String keyspace, String table)
+    {
+        waitForAssert(() -> assertTrue(isIndexQueryable(keyspace, table)), 60, TimeUnit.SECONDS);
+    }
+
+    protected void waitForCompactionsFinished()
+    {
+        waitForAssert(() -> assertEquals(0, getCompactionTasks()), 10, TimeUnit.SECONDS);
+    }
+
+    protected void waitForEquals(ObjectName name, long value)
+    {
+        waitForAssert(() -> assertEquals(value, ((Number) getMetricValue(name)).longValue()), 10, TimeUnit.SECONDS);
+    }
+
+    protected ObjectName objectName(String name, String keyspace, String table, String index, String type)
+    {
+        try
+        {
+            return new ObjectName(String.format("org.apache.cassandra.metrics:type=StorageAttachedIndex," +
+                                                "keyspace=%s,table=%s,index=%s,scope=%s,name=%s",
+                                                keyspace, table, index, type, name));
+        }
+        catch (Throwable ex)
+        {
+            throw Throwables.unchecked(ex);
+        }
+    }
+
+    protected ObjectName objectNameNoIndex(String name, String keyspace, String table, String type)
+    {
+        try
+        {
+            return new ObjectName(String.format("org.apache.cassandra.metrics:type=StorageAttachedIndex," +
+                                                "keyspace=%s,table=%s,scope=%s,name=%s",
+                                                keyspace, table, type, name));
+        }
+        catch (Throwable ex)
+        {
+            throw Throwables.unchecked(ex);
+        }
+    }
+
+    @Override
+    public String createTable(String query)
+    {
+        return createTable(KEYSPACE, query);
+    }
+
+    @Override
+    public UntypedResultSet execute(String query, Object... values)
+    {
+        return executeFormattedQuery(formatQuery(query), values);
+    }
+
+    @Override
+    public Session sessionNet()
+    {
+        return sessionNet(getDefaultVersion());
+    }
+
+    public void flush(String keyspace, String table)
+    {
+        ColumnFamilyStore store = Keyspace.open(keyspace).getColumnFamilyStore(table);
+        if (store != null)
+            store.forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS);
+    }
+
+    public void compact(String keyspace, String table)
+    {
+        ColumnFamilyStore store = Keyspace.open(keyspace).getColumnFamilyStore(table);
+        if (store != null)
+            store.forceMajorCompaction();
+    }
+
+    protected void truncate(boolean snapshot)
+    {
+        ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(currentTable());
+        if (snapshot)
+            cfs.truncateBlocking();
+        else
+            cfs.truncateBlockingWithoutSnapshot();
+    }
+
+    protected int getCompactionTasks()
+    {
+        return CompactionManager.instance.getActiveCompactions() + CompactionManager.instance.getPendingTasks();
+    }
+
+    protected int snapshot(String snapshotName)
+    {
+        ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(currentTable());
+        TableSnapshot snapshot = cfs.snapshot(snapshotName);
+        return snapshot.getDirectories().size();
+    }
+
+    protected List<String> restore(ColumnFamilyStore cfs, Directories.SSTableLister lister)
+    {
+        File dataDirectory = cfs.getDirectories().getDirectoryForNewSSTables();
+
+        List<String> fileNames = new ArrayList<>();
+        for (File file : lister.listFiles())
+        {
+            if (file.tryMove(new File(dataDirectory.absolutePath() + File.pathSeparator() + file.name())))
+            {
+                fileNames.add(file.name());
+            }
+        }
+        cfs.loadNewSSTables();
+        return fileNames;
+    }
+
+    public static class Randomization
+    {
+        private final long seed;
+        private final Random random;
+
+        Randomization()
+        {
+            seed = Long.getLong("cassandra.test.random.seed", System.nanoTime());
+            random = new Random(seed);
+        }
+
+        public void printSeedOnFailure()
+        {
+            System.err.println("Randomized test failed. To rerun test use -Dcassandra.test.random.seed=" + seed);
+        }
+
+        public int nextInt()
+        {
+            return random.nextInt();
+        }
+
+        public int nextIntBetween(int minValue, int maxValue)
+        {
+            return RandomInts.randomIntBetween(random, minValue, maxValue);
+        }
+
+        public long nextLong()
+        {
+            return random.nextLong();
+        }
+
+        public short nextShort()
+        {
+            return (short)random.nextInt(Short.MAX_VALUE + 1);
+        }
+
+        public byte nextByte()
+        {
+            return (byte)random.nextInt(Byte.MAX_VALUE + 1);
+        }
+
+        public BigInteger nextBigInteger(int minNumBits, int maxNumBits)
+        {
+            return new BigInteger(RandomInts.randomIntBetween(random, minNumBits, maxNumBits), random);
+        }
+
+        public BigDecimal nextBigDecimal(int minUnscaledValue, int maxUnscaledValue, int minScale, int maxScale)
+        {
+            return BigDecimal.valueOf(RandomInts.randomIntBetween(random, minUnscaledValue, maxUnscaledValue),
+                                      RandomInts.randomIntBetween(random, minScale, maxScale));
+        }
+
+        public float nextFloat()
+        {
+            return random.nextFloat();
+        }
+
+        public double nextDouble()
+        {
+            return random.nextDouble();
+        }
+
+        public String nextAsciiString(int minLength, int maxLength)
+        {
+            return RandomStrings.randomAsciiOfLengthBetween(random, minLength, maxLength);
+        }
+
+        public String nextTextString(int minLength, int maxLength)
+        {
+            return RandomStrings.randomRealisticUnicodeOfLengthBetween(random, minLength, maxLength);
+        }
+
+        public boolean nextBoolean()
+        {
+            return random.nextBoolean();
+        }
+
+        public void nextBytes(byte[] bytes)
+        {
+            random.nextBytes(bytes);
+        }
+    }
+
+    public static class FailureWatcher extends TestWatcher
+    {
+        @Override
+        protected void failed(Throwable e, Description description)
+        {
+            if (random != null)
+                random.printSeedOnFailure();
+        }
+    }
+}

test/unit/org/apache/cassandra/index/sai/cql/AbstractQueryTester.java

@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.cql;
+
+import java.util.LinkedList;
+import java.util.List;
+
+import com.google.common.collect.ImmutableList;
+import org.junit.Before;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import org.apache.cassandra.index.sai.SAITester;
+import org.apache.cassandra.index.sai.plan.StorageAttachedIndexSearcher;
+import org.apache.cassandra.inject.Injections;
+
+import static org.apache.cassandra.inject.InvokePointBuilder.newInvokePoint;
+
+@RunWith(Parameterized.class)
+public class AbstractQueryTester extends SAITester
+{
+    public static final Injections.Counter INDEX_QUERY_COUNTER = Injections.newCounter("IndexQueryCounter")
+                                                                              .add(newInvokePoint().onClass(StorageAttachedIndexSearcher.class).onMethod("search"))
+                                                                              .build();
+
+    @Parameterized.Parameter(0)
+    public BaseDataModel dataModel;
+    @Parameterized.Parameter(1)
+    public List<IndexQuerySupport.BaseQuerySet> sets;
+
+    protected BaseDataModel.Executor executor;
+
+    @Before
+    public void setup() throws Throwable
+    {
+        requireNetwork();
+
+        schemaChange(String.format("CREATE KEYSPACE IF NOT EXISTS %s WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '1'}", BaseDataModel.KEYSPACE));
+
+        Injections.inject(INDEX_QUERY_COUNTER);
+
+        executor = new SingleNodeExecutor(this, INDEX_QUERY_COUNTER);
+    }
+
+    @SuppressWarnings("unused")
+    @Parameterized.Parameters(name = "{0}")
+    public static List<Object[]> params() throws Throwable
+    {
+        List<Object[]> scenarios = new LinkedList<>();
+
+        scenarios.add(new Object[]{ new BaseDataModel(BaseDataModel.NORMAL_COLUMNS, BaseDataModel.NORMAL_COLUMN_DATA), IndexQuerySupport.BASE_QUERY_SETS });
+
+        scenarios.add(new Object[]{ new BaseDataModel.CompoundKeyDataModel(BaseDataModel.NORMAL_COLUMNS, BaseDataModel.NORMAL_COLUMN_DATA), IndexQuerySupport.BASE_QUERY_SETS });
+
+        scenarios.add(new Object[]{ new BaseDataModel.CompoundKeyWithStaticsDataModel(BaseDataModel.STATIC_COLUMNS, BaseDataModel.STATIC_COLUMN_DATA), IndexQuerySupport.STATIC_QUERY_SETS });
+
+        scenarios.add(new Object[]{ new BaseDataModel.CompositePartitionKeyDataModel(BaseDataModel.NORMAL_COLUMNS, BaseDataModel.NORMAL_COLUMN_DATA),
+                                    ImmutableList.builder().addAll(IndexQuerySupport.BASE_QUERY_SETS).addAll(IndexQuerySupport.COMPOSITE_PARTITION_QUERY_SETS).build()});
+
+        return scenarios;
+    }
+}

test/unit/org/apache/cassandra/index/sai/cql/BaseDataModel.java

@@ -0,0 +1,397 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+package org.apache.cassandra.index.sai.cql;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+import com.google.common.base.MoreObjects;
+import com.google.common.collect.ForwardingList;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Sets;
+
+import org.apache.cassandra.cql3.CQL3Type;
+import org.apache.cassandra.utils.Pair;
+
+class BaseDataModel
+{
+    public static String KEYSPACE = "sai_query_keyspace";
+
+    public static String SIMPLE_SELECT_TEMPLATE = "SELECT %s FROM %%s WHERE %s %s ? LIMIT ?";
+    public static String SIMPLE_SELECT_WITH_FILTERING_TEMPLATE = "SELECT %s FROM %%s WHERE %s %s ? LIMIT ? ALLOW FILTERING";
+    public static String TWO_CLAUSE_AND_QUERY_TEMPLATE = "SELECT %s FROM %%s WHERE %s %s ? AND %s %s ? LIMIT ? ALLOW FILTERING";
+    public static String TWO_CLAUSE_AND_QUERY_FILTERING_TEMPLATE = "SELECT %s FROM %%s WHERE %s %s ? AND %s %s ? LIMIT ? ALLOW FILTERING";
+    public static String THREE_CLAUSE_AND_QUERY_FILTERING_TEMPLATE = "SELECT %s FROM %%s WHERE %s %s ? AND %s %s ? AND %s %s ? LIMIT ? ALLOW FILTERING";
+
+    public static String ASCII_COLUMN = "abbreviation";
+    public static String BIGINT_COLUMN = "gdp";
+    public static String BOOLEAN_COLUMN = "active";
+    public static String DATE_COLUMN = "visited";
+    public static String DOUBLE_COLUMN = "area_sq_miles";
+    public static String FLOAT_COLUMN = "murder_rate";
+    public static String INET_COLUMN = "ip";
+    public static String INT_COLUMN = "population";
+    public static String SMALLINT_COLUMN = "murders_per_year";
+    public static String TINYINT_COLUMN = "tiny_murders_per_year";
+    public static String TEXT_COLUMN = "name";
+    public static String TIME_COLUMN = "avg_dmv_wait";
+    public static String TIMESTAMP_COLUMN = "visited_timestamp";
+    public static String UUID_COLUMN = "id";
+    public static String TIMEUUID_COLUMN = "temporal_id";
+    public static String NON_INDEXED_COLUMN = "non_indexed";
+
+    public static int DEFAULT_TTL_SECONDS = 10;
+
+    public static List<Pair<String, String>> NORMAL_COLUMNS =
+            ImmutableList.<Pair<String, String>>builder()
+                    .add(Pair.create(ASCII_COLUMN, CQL3Type.Native.ASCII.toString()))
+                    .add(Pair.create(BIGINT_COLUMN, CQL3Type.Native.BIGINT.toString()))
+                    .add(Pair.create(BOOLEAN_COLUMN, CQL3Type.Native.BOOLEAN.toString()))
+                    .add(Pair.create(DATE_COLUMN, CQL3Type.Native.DATE.toString()))
+                    .add(Pair.create(DOUBLE_COLUMN, CQL3Type.Native.DOUBLE.toString()))
+                    .add(Pair.create(FLOAT_COLUMN, CQL3Type.Native.FLOAT.toString()))
+                    .add(Pair.create(INET_COLUMN, CQL3Type.Native.INET.toString()))
+                    .add(Pair.create(INT_COLUMN, CQL3Type.Native.INT.toString()))
+                    .add(Pair.create(SMALLINT_COLUMN, CQL3Type.Native.SMALLINT.toString()))
+                    .add(Pair.create(TINYINT_COLUMN, CQL3Type.Native.TINYINT.toString()))
+                    .add(Pair.create(TEXT_COLUMN, CQL3Type.Native.TEXT.toString()))
+                    .add(Pair.create(TIME_COLUMN, CQL3Type.Native.TIME.toString()))
+                    .add(Pair.create(TIMESTAMP_COLUMN, CQL3Type.Native.TIMESTAMP.toString()))
+                    .add(Pair.create(UUID_COLUMN, CQL3Type.Native.UUID.toString()))
+                    .add(Pair.create(TIMEUUID_COLUMN, CQL3Type.Native.TIMEUUID.toString()))
+                    .add(Pair.create(NON_INDEXED_COLUMN, CQL3Type.Native.INT.toString()))
+                    .build();
+
+    public static List<String> NORMAL_COLUMN_DATA =
+            ImmutableList.<String>builder()
+                    .add("'AK',   500000000,  true, '2009-07-15', 570640.95,  7.7,   '158.145.20.64',   737709,  164,  16,        'Alaska', '00:18:20', '2009-07-15T00:00:00', e37394dc-d17b-11e8-a8d5-f2801f1b9fd1, acfe5ada-d17c-11e8-a8d5-f2801f1b9fd1, 1")
+                    .add("'AL',  1000000000,  true, '2011-09-13',  50645.33,  7.0,   '206.16.212.91',  4853875,   57,   5,       'Alabama', '01:04:00', '2011-09-13T00:00:00', b7373af6-d7c1-45ae-b145-5bf4b5cdd00c, c592c37e-d17c-11e8-a8d5-f2801f1b9fd1, 1")
+                    .add("'AR',  2000000000, false, '2013-06-17', 113594.08,  5.5,  '170.94.194.134',  2977853,   99,   9,      'Arkansas', '00:55:23', '2013-06-17T00:00:00', a0daaeb4-c8a2-4c68-9899-e32d08238550, cfaae67a-d17c-11e8-a8d5-f2801f1b9fd1, 1")
+                    .add("'CA',  3000000000,  true, '2012-06-17', 155779.22,  4.8,    '67.157.98.46', 38993940, 1861, 117,    'California', '01:30:45', '2012-06-17T00:00:00', 96232af0-0af7-438b-9049-c5a5a944ff93, d7e80692-d17c-11e8-a8d5-f2801f1b9fd1, 1")
+                    .add("'DE',  4000000000, false, '2013-06-17',   1948.54,  6.7,   '167.21.128.20',   944076,   63,   6,      'Delaware', '00:23:45', '2013-06-17T00:00:00', b2a0a879-5223-40d2-9671-775ee209b6f2, dd10a5b6-d17c-11e8-a8d5-f2801f1b9fd1, 1")
+                    .add("'ID',  4500000000, false, '2015-06-18',  82643.12,  1.8,   '164.165.67.10',  1652828,   30,   3,         'Idaho', '00:18:45', '2015-06-18T00:00:00', c6eec0b0-0eef-40e8-ac38-3a82110443e4, e2788780-d17c-11e8-a8d5-f2801f1b9fd1, 1")
+                    .add("'KY',  4750000000, false, '2018-03-12',  39486.34,  4.7,  '205.204.196.64',  4424611,  209,  20,      'Kentucky', '00:45:00', '2018-03-12T00:00:00', 752355f8-405b-4d94-88f3-9992cda30f1e, e7c4e1d4-d17c-11e8-a8d5-f2801f1b9fd1, 1")
+                    .add("'LA',  4800000000,  true, '2013-06-10',  43203.90, 10.2,  '204.196.242.71',  4668960,  474,  47,     'Louisiana', '00:56:07', '2013-06-10T00:00:00', 17be691a-c1a4-4467-a4ad-64605c74fb1c, ee6136d2-d17c-11e8-a8d5-f2801f1b9fd1, 1")
+                    .add("'MA',  5000000000,  true,  '2010-07-04',   7800.06,  1.9,   '170.63.206.57',  6784240,  126,  12, 'Massachusetts', '01:01:34', '2010-07-04T00:00:00', e8a3c287-78cf-46b5-b554-42562e7dcfb3, f57a3b62-d17c-11e8-a8d5-f2801f1b9fd1, 2")
+                    .add("'MI',  6000000000, false, '2011-09-13',  56538.90,  5.8,    '23.72.184.64',  9917715,  571,  57,      'Michigan', '00:43:09', '2011-09-13T00:00:00', a0daaeb4-c8a2-4c68-9899-e32d08238550, 0497b886-d17d-11e8-a8d5-f2801f1b9fd1, 2")
+                    .add("'MS',  7000000000,  true, '2013-06-17',  46923.27,  5.3,   '192.251.58.38',  2989390,  159,  15,   'Mississippi', '01:04:23', '2013-06-17T00:00:00', 96232af0-0af7-438b-9049-c5a5a944ff93, 0b0205e6-d17d-11e8-a8d5-f2801f1b9fd1, 2")
+                    .add("'TN',  8000000000, false, '2018-03-10',  41234.90,  6.1, '170.141.221.177',  6595056,  402,  40,     'Tennessee', '00:39:45', '2018-03-10T00:00:00', b2a0a879-5223-40d2-9671-775ee209b6f2, 105dc746-d17d-11e8-a8d5-f2801f1b9fd1, 2")
+                    .add("'TX',  9000000000,  true, '2014-06-17', 261231.71,  4.7,   '204.66.40.181', 27429639, 1276, 107,         'Texas', '00:38:13', '2014-06-17T00:00:00', c6eec0b0-0eef-40e8-ac38-3a82110443e4, 155b6bcc-d17d-11e8-a8d5-f2801f1b9fd1, 2")
+                    .add("'UT',  9250000000,  true, '2014-06-20',  82169.62,  1.8,   '204.113.13.48',  2990632,   54,   5,          'Utah', '00:25:00', '2014-06-20T00:00:00', 752355f8-405b-4d94-88f3-9992cda30f1e, 1a267c50-d17d-11e8-a8d5-f2801f1b9fd1, 2")
+                    .add("'VA',  9500000000,  true, '2018-06-19',  39490.09,  4.6,  '152.130.96.221',  8367587,  383,  38,      'Virginia', '00:43:07', '2018-06-19T00:00:00', 17be691a-c1a4-4467-a4ad-64605c74fb1c, 1fc81a4c-d17d-11e8-a8d5-f2801f1b9fd1, 2")
+                    .add("'WY', 10000000000, false, '2015-06-17',  97093.14,  2.7,  '192.146.215.91',   586107,   57,   5,       'Wyoming', '00:15:50', '2015-06-17T00:00:00', e8a3c287-78cf-46b5-b554-42562e7dcfb3, 2576612e-d17d-11e8-a8d5-f2801f1b9fd1, 2")
+                    .build();
+
+    public static String STATIC_INT_COLUMN = "entered";
+
+    public static List<Pair<String, String>> STATIC_COLUMNS =
+            ImmutableList.<Pair<String, String>>builder().add(Pair.create(STATIC_INT_COLUMN, CQL3Type.Native.INT.toString() + " static"))
+                                                         .addAll(NORMAL_COLUMNS).build();
+
+    public static List<String> STATIC_COLUMN_DATA = ImmutableList.of("1819, " + NORMAL_COLUMN_DATA.get(0),
+                                                       "1819, " + NORMAL_COLUMN_DATA.get(1),
+                                                       "1850, " + NORMAL_COLUMN_DATA.get(2),
+                                                       "1850, " + NORMAL_COLUMN_DATA.get(3),
+                                                       "1910, " + NORMAL_COLUMN_DATA.get(4),
+                                                       "1910, " + NORMAL_COLUMN_DATA.get(5),
+                                                       "1792, " + NORMAL_COLUMN_DATA.get(6),
+                                                       "1792, " + NORMAL_COLUMN_DATA.get(7),
+                                                       "1788, " + NORMAL_COLUMN_DATA.get(8),
+                                                       "1788, " + NORMAL_COLUMN_DATA.get(9),
+                                                       "1817, " + NORMAL_COLUMN_DATA.get(10),
+                                                       "1817, " + NORMAL_COLUMN_DATA.get(11),
+                                                       "1896, " + NORMAL_COLUMN_DATA.get(12),
+                                                       "1896, " + NORMAL_COLUMN_DATA.get(13),
+                                                       "1845, " + NORMAL_COLUMN_DATA.get(14),
+                                                       "1845, " + NORMAL_COLUMN_DATA.get(15));
+
+    private static final AtomicInteger seq = new AtomicInteger();
+
+    private final String columnNames;
+    private final List<String> rows;
+
+    protected final Set<String> skipColumns = Sets.newHashSet(NON_INDEXED_COLUMN, BOOLEAN_COLUMN);
+    protected final List<Pair<String, String>> columns;
+    protected final String indexedTable = "table_" + seq.getAndIncrement();
+    protected final String nonIndexedTable = "table_" + seq.getAndIncrement();
+    protected String tableOptions = "";
+    protected List<Pair<String, String>> keyColumns;
+    protected String primaryKey;
+    protected List<String> keys;
+
+    public BaseDataModel(List<Pair<String, String>> columns, List<String> rows)
+    {
+        this.keyColumns = ImmutableList.of(Pair.create("p", "int"));
+        this.primaryKey = keyColumns.stream().map(pair -> pair.left).collect(Collectors.joining(", "));
+
+        this.columns = columns;
+        this.columnNames = columns.stream().map(pair -> pair.left).collect(Collectors.joining(", "));
+        this.rows = rows;
+
+        this.keys = new SimplePrimaryKeyList(rows.size());
+    }
+
+    public List<Pair<String, String>> keyColumns()
+    {
+        return keyColumns;
+    }
+
+    public void createTables(Executor tester)
+    {
+        String keyColumnDefs = keyColumns.stream().map(column -> column.left + ' ' + column.right).collect(Collectors.joining(", "));
+        String normalColumnDefs = columns.stream().map(column -> column.left + ' ' + column.right).collect(Collectors.joining(", "));
+
+        String template = "CREATE TABLE %s (%s, %s, PRIMARY KEY (%s))" + tableOptions;
+        tester.createTable(String.format(template, KEYSPACE + "." + indexedTable, keyColumnDefs, normalColumnDefs, primaryKey));
+        tester.createTable(String.format(template, KEYSPACE + "." + nonIndexedTable, keyColumnDefs, normalColumnDefs, primaryKey));
+    }
+
+    public void createIndexes(Executor tester) throws Throwable
+    {
+        String template = "CREATE CUSTOM INDEX ndi_%s_index_%s ON %%s (%s) USING 'StorageAttachedIndex'";
+
+        for (Pair<String, String> column : columns)
+        {
+            if (!skipColumns.contains(column.left))
+            {
+                executeLocalIndexed(tester, String.format(template, column.left, indexedTable, column.left));
+                tester.waitForIndexQueryable(KEYSPACE, indexedTable);
+            }
+        }
+    }
+
+    public void insertRows(Executor tester) throws Throwable
+    {
+        String template = "INSERT INTO %%s (%s, %s) VALUES (%s, %s)";
+
+        for (int i = 0; i < keys.size(); i++)
+        {
+            executeLocal(tester, String.format(template, primaryKey, columnNames, keys.get(i), rows.get(i)));
+        }
+    }
+
+    public void insertRowsWithTTL(Executor tester) throws Throwable
+    {
+        String template = "INSERT INTO %%s (%s, %s) VALUES (%s, %s)%s";
+
+        for (int i = 0; i < keys.size(); i++)
+        {
+            String ttl = deletable().contains(i) ? " USING TTL " + DEFAULT_TTL_SECONDS : "";
+            executeLocal(tester, String.format(template, primaryKey, columnNames, keys.get(i), rows.get(i), ttl));
+        }
+    }
+
+    public void executeLocal(Executor tester, String query, Object... values) throws Throwable
+    {
+        tester.executeLocal(formatIndexedQuery(query), values);
+        tester.executeLocal(formatNonIndexedQuery(query), values);
+    }
+
+    public void executeLocalIndexed(Executor tester, String query, Object... values) throws Throwable
+    {
+        tester.executeLocal(formatIndexedQuery(query), values);
+    }
+
+    public List<Object> executeIndexed(Executor tester, String query, int fetchSize, Object... values) throws Throwable
+    {
+        return tester.executeRemote(formatIndexedQuery(query), fetchSize, values);
+    }
+
+    public List<Object> executeNonIndexed(Executor tester, String query, int fetchSize, Object... values) throws Throwable
+    {
+        return tester.executeRemote(formatNonIndexedQuery(query), fetchSize, values);
+    }
+
+    protected Set<Integer> deletable()
+    {
+        return Sets.newHashSet(3, 7, 9, 12);
+    }
+
+    private String formatIndexedQuery(String query)
+    {
+        return String.format(query, KEYSPACE + "." + indexedTable);
+    }
+
+    private String formatNonIndexedQuery(String query)
+    {
+        return String.format(query, KEYSPACE + "." + nonIndexedTable);
+    }
+
+    @Override
+    public String toString()
+    {
+        return MoreObjects.toStringHelper(this).add("primaryKey", primaryKey).toString();
+    }
+
+    static class CompoundKeyWithStaticsDataModel extends CompoundKeyDataModel
+    {
+        public CompoundKeyWithStaticsDataModel(List<Pair<String, String>> columns, List<String> rows)
+        {
+            super(columns, rows);
+
+            this.keys = new CompoundPrimaryKeyList(rows.size(), 2);
+        }
+
+        @Override
+        public void insertRows(Executor tester) throws Throwable
+        {
+            super.insertRows(tester);
+
+            executeLocal(tester, String.format("INSERT INTO %%s (p, %s) VALUES(100, 2019)", STATIC_INT_COLUMN)); // static only
+        }
+
+        @Override
+        protected Set<Integer> deletable()
+        {
+            return Sets.newHashSet(4, 8, 12, 13, 100);
+        }
+    }
+
+    static class CompositePartitionKeyDataModel extends BaseDataModel
+    {
+        public CompositePartitionKeyDataModel(List<Pair<String, String>> columns, List<String> rows)
+        {
+            super(columns, rows);
+
+            this.keyColumns = ImmutableList.of(Pair.create("p1", "int"), Pair.create("p2", "int"));
+            this.primaryKey = keyColumns.stream().map(pair -> pair.left).collect(Collectors.joining(", "));
+            this.keys = new CompoundPrimaryKeyList(rows.size(), 2);
+        }
+
+        @Override
+        public void createTables(Executor tester)
+        {
+            String keyColumnDefs = keyColumns.stream().map(column -> column.left + ' ' + column.right).collect(Collectors.joining(", "));
+            String normalColumnDefs = columns.stream().map(column -> column.left + ' ' + column.right).collect(Collectors.joining(", "));
+
+            String template = "CREATE TABLE %s (%s, %s, PRIMARY KEY ((%s)))" + tableOptions;
+            tester.createTable(String.format(template, KEYSPACE + "." + indexedTable, keyColumnDefs, normalColumnDefs, primaryKey));
+            tester.createTable(String.format(template, KEYSPACE + "." + nonIndexedTable, keyColumnDefs, normalColumnDefs, primaryKey));
+        }
+
+        @Override
+        public void createIndexes(Executor tester) throws Throwable
+        {
+            super.createIndexes(tester);
+            String template = "CREATE CUSTOM INDEX ndi_%s_index_%s ON %%s (%s) USING 'StorageAttachedIndex'";
+
+            for (Pair<String, String> column : keyColumns)
+            {
+                if (!skipColumns.contains(column.left))
+                {
+                    executeLocalIndexed(tester, String.format(template, column.left, indexedTable, column.left));
+                    tester.waitForIndexQueryable(KEYSPACE, indexedTable);
+                }
+            }
+        }
+
+        @Override
+        public void insertRows(Executor tester) throws Throwable
+        {
+            super.insertRows(tester);
+        }
+
+        @Override
+        protected Set<Integer> deletable()
+        {
+            // already overwrites {@code deleteRows()}
+            return Collections.emptySet();
+        }
+    }
+
+    static class CompoundKeyDataModel extends BaseDataModel
+    {
+        public CompoundKeyDataModel(List<Pair<String, String>> columns, List<String> rows)
+        {
+            super(columns, rows);
+
+            this.keyColumns = ImmutableList.of(Pair.create("p", "int"), Pair.create("c", "int"));
+            this.primaryKey = keyColumns.stream().map(pair -> pair.left).collect(Collectors.joining(", "));
+            this.keys = new CompoundPrimaryKeyList(rows.size(), 1);
+        }
+
+    }
+
+    static class SimplePrimaryKeyList extends ForwardingList<String>
+    {
+        private final List<String> primaryKeys;
+
+        SimplePrimaryKeyList(int rows)
+        {
+            this.primaryKeys = IntStream.range(0, rows).mapToObj(String::valueOf).collect(Collectors.toList());
+        }
+
+        @Override
+        protected List<String> delegate()
+        {
+            return primaryKeys;
+        }
+
+        @Override
+        public String toString()
+        {
+            return String.format("SimplePrimaryKeyList[rows: %d]", primaryKeys.size());
+        }
+    }
+
+    static class CompoundPrimaryKeyList extends ForwardingList<String>
+    {
+        private final List<String> primaryKeys;
+        private final int rowsPerPartition;
+
+        CompoundPrimaryKeyList(int rows, int rowsPerPartition)
+        {
+            this.primaryKeys = IntStream.range(0, rows).mapToObj(v -> v / rowsPerPartition + ", " + v % rowsPerPartition).collect(Collectors.toList());
+            this.rowsPerPartition = rowsPerPartition;
+        }
+
+        @Override
+        protected List<String> delegate()
+        {
+            return primaryKeys;
+        }
+
+        @Override
+        public String toString()
+        {
+            return String.format("CompoundPrimaryKeyList[rows: %d, partition size: %d]", primaryKeys.size(), rowsPerPartition);
+        }
+    }
+
+    interface Executor
+    {
+        void createTable(String statement);
+
+        void waitForIndexQueryable(String keyspace, String table);
+
+        void executeLocal(String query, Object...values) throws Throwable;
+
+        List<Object> executeRemote(String query, int fetchSize, Object...values) throws Throwable;
+
+        void counterReset();
+
+        long getCounter();
+    }
+}

test/unit/org/apache/cassandra/index/sai/cql/BooleanTypeTest.java

@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.cql;
+
+import org.junit.Test;
+
+import org.apache.cassandra.index.sai.SAITester;
+
+import static org.junit.Assert.assertEquals;
+
+public class BooleanTypeTest extends SAITester
+{
+    @Test
+    public void test() throws Throwable
+    {
+        createTable("CREATE TABLE %s (id text PRIMARY KEY, val boolean)");
+
+        createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex'");
+        waitForIndexQueryable();
+
+        execute("INSERT INTO %s (id, val) VALUES ('0', false)");
+        execute("INSERT INTO %s (id, val) VALUES ('1', true)");
+        execute("INSERT INTO %s (id, val) VALUES ('2', true)");
+
+        assertEquals(2, execute("SELECT id FROM %s WHERE val = true").size());
+        assertEquals(1, execute("SELECT id FROM %s WHERE val = false").size());
+    }
+}

test/unit/org/apache/cassandra/index/sai/cql/ClusteringKeyIndexTest.java

@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.cql;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import org.apache.cassandra.index.sai.SAITester;
+
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+public class ClusteringKeyIndexTest extends SAITester
+{
+    @Before
+    public void createTableAndIndex()
+    {
+        createTable("CREATE TABLE %s (pk1 int, pk2 text, val int, PRIMARY KEY((pk1), pk2)) WITH CLUSTERING ORDER BY (pk2 DESC)");
+        createIndex("CREATE CUSTOM INDEX pk2_idx ON %s(pk2) USING 'StorageAttachedIndex'");
+
+        disableCompaction();
+    }
+
+    private void insertData1() throws Throwable
+    {
+        execute("INSERT INTO %s (pk1, pk2, val) VALUES (1, '1', 1)");
+        execute("INSERT INTO %s (pk1, pk2, val) VALUES (2, '2', 2)");
+        execute("INSERT INTO %s (pk1, pk2, val) VALUES (3, '3', 3)");
+    }
+
+    private void insertData2() throws Throwable
+    {
+        execute("INSERT INTO %s (pk1, pk2, val) VALUES (4, '4', 4)");
+        execute("INSERT INTO %s (pk1, pk2, val) VALUES (5, '5', 5)");
+        execute("INSERT INTO %s (pk1, pk2, val) VALUES (6, '6', 6)");
+    }
+
+    @Test
+    public void queryFromMemtable() throws Throwable
+    {
+        insertData1();
+        insertData2();
+        runQueries();
+    }
+
+    private Object[] expectedRow(int index)
+    {
+        return row(index, Integer.toString(index), index);
+    }
+
+    private void runQueries() throws Throwable
+    {
+        assertRowsIgnoringOrder(execute("SELECT * FROM %s WHERE pk1 = 2"), expectedRow(2));
+
+        assertRowsIgnoringOrder(execute("SELECT * FROM %s WHERE pk2 = '2'"), expectedRow(2));
+
+        assertRowsIgnoringOrder(execute("SELECT * FROM %s WHERE pk1 = -1 AND pk2 = '2'"));
+
+        assertThatThrownBy(()->execute("SELECT * FROM %s WHERE pk1 = -1 AND val = 2")).hasMessageContaining("use ALLOW FILTERING");
+
+    }
+}

test/unit/org/apache/cassandra/index/sai/cql/CollectionIndexingTest.java

@@ -0,0 +1,233 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.cql;
+
+import java.util.Arrays;
+import java.util.HashMap;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import org.apache.cassandra.index.sai.SAITester;
+
+import static org.junit.Assert.assertEquals;
+
+// This test is primarily handling edge conditions, error conditions
+// and basic functionality. Comprehensive type testing of collections
+// is in the cql/types/collections package
+public class CollectionIndexingTest extends SAITester
+{
+    private static final String ALLOW_FILTERING_ERROR = "Cannot execute this query as it might involve data filtering " +
+                                                        "and thus may have unpredictable performance. If you want to " +
+                                                        "execute this query despite the performance unpredictability, " +
+                                                        "use ALLOW FILTERING";
+    @Before
+    public void setup() throws Throwable
+    {
+        requireNetwork();
+    }
+
+    @Test
+    public void indexMap() throws Throwable
+    {
+        createPopulatedMap("CREATE CUSTOM INDEX ON %s(value) USING 'StorageAttachedIndex'");
+        assertEquals(2, execute("SELECT * FROM %s WHERE value CONTAINS 'v1'").size());
+    }
+
+    @Test
+    public void indexMapKeys() throws Throwable
+    {
+        createPopulatedMap("CREATE CUSTOM INDEX ON %s(KEYS(value)) USING 'StorageAttachedIndex'");
+        assertEquals(2, execute("SELECT * FROM %s WHERE value CONTAINS KEY 1").size());
+    }
+
+    @Test
+    public void indexMapValues() throws Throwable
+    {
+        createPopulatedMap("CREATE CUSTOM INDEX ON %s(VALUES(value)) USING 'StorageAttachedIndex'");
+        assertEquals(2, execute("SELECT * FROM %s WHERE value CONTAINS 'v1'").size());
+    }
+
+    @Test
+    public void indexMapEntries() throws Throwable
+    {
+        createPopulatedMap("CREATE CUSTOM INDEX ON %s(ENTRIES(value)) USING 'StorageAttachedIndex'");
+        assertEquals(2, execute("SELECT * FROM %s WHERE value[1] = 'v1'").size());
+        assertEquals(1, execute("SELECT * FROM %s WHERE value[1] = 'v1' AND value[2] = 'v2' ALLOW FILTERING").size());
+    }
+
+    @Test
+    public void indexFrozenList() throws Throwable
+    {
+        createPopulatedFrozenList("CREATE CUSTOM INDEX ON %s(FULL(value)) USING 'StorageAttachedIndex'");
+        assertEquals(2, execute("SELECT * FROM %s WHERE value = ?", Arrays.asList(1, 2, 3)).size());
+    }
+
+    @Test
+    public void indexFrozenMap() throws Throwable
+    {
+        createPopulatedFrozenMap("CREATE CUSTOM INDEX ON %s(FULL(value)) USING 'StorageAttachedIndex'");
+        assertEquals(1, execute("SELECT * FROM %s WHERE value = ?", new HashMap<Integer, String>() {{
+            put(1, "v1");
+            put(2, "v2");
+        }}).size());
+
+    }
+
+    @Test
+    public void indexFrozenMapQueryKeys() throws Throwable
+    {
+        createPopulatedFrozenMap("CREATE CUSTOM INDEX ON %s(FULL(value)) USING 'StorageAttachedIndex'");
+        assertUnsupportedIndexOperator("SELECT * FROM %s WHERE value contains key 1");
+        assertEquals(2, execute("SELECT * FROM %s WHERE value contains key 1 ALLOW FILTERING").size());
+    }
+
+    @Test
+    public void indexFrozenMapQueryValues() throws Throwable
+    {
+        createPopulatedFrozenMap("CREATE CUSTOM INDEX ON %s(FULL(value)) USING 'StorageAttachedIndex'");
+        assertUnsupportedIndexOperator("SELECT * FROM %s WHERE value contains 'v1'");
+        assertEquals(2, execute("SELECT * FROM %s WHERE value contains 'v1' ALLOW FILTERING").size());
+    }
+
+    @Test
+    public void indexFrozenMapQueryEntries() throws Throwable
+    {
+        createPopulatedFrozenMap("CREATE CUSTOM INDEX ON %s(FULL(value)) USING 'StorageAttachedIndex'");
+        assertInvalidMessage("Map-entry equality predicates on frozen map column value are not supported",
+                "SELECT * FROM %s WHERE value[1] = 'v1'");
+    }
+
+    @Test
+    public void indexMapEntriesQueryEq() throws Throwable
+    {
+        createPopulatedMap("CREATE CUSTOM INDEX ON %s(ENTRIES(value)) USING 'StorageAttachedIndex'");
+        assertInvalidMessage("Collection column 'value' (map<int, text>) cannot be restricted by a '=' relation",
+                "SELECT * FROM %s WHERE value = ?", Arrays.asList(1, 2));
+    }
+
+    @Test
+    public void indexMapEntriesQueryKeys() throws Throwable
+    {
+        createPopulatedMap("CREATE CUSTOM INDEX ON %s(ENTRIES(value)) USING 'StorageAttachedIndex'");
+        assertUnsupportedIndexOperator("SELECT * FROM %s WHERE value contains key 1");
+        assertEquals(2, execute("SELECT * FROM %s WHERE value contains key 1 ALLOW FILTERING").size());
+    }
+
+    @Test
+    public void indexMapEntriesQueryValues() throws Throwable
+    {
+        createPopulatedMap("CREATE CUSTOM INDEX ON %s(ENTRIES(value)) USING 'StorageAttachedIndex'");
+        assertUnsupportedIndexOperator("SELECT * FROM %s WHERE value contains 'v1'");
+        assertEquals(2, execute("SELECT * FROM %s WHERE value contains 'v1' ALLOW FILTERING").size());
+    }
+
+    @Test
+    public void indexMapKeysQueryEq() throws Throwable
+    {
+        createPopulatedMap("CREATE CUSTOM INDEX ON %s(KEYS(value)) USING 'StorageAttachedIndex'");
+        assertInvalidMessage("Collection column 'value' (map<int, text>) cannot be restricted by a '=' relation",
+                "SELECT * FROM %s WHERE value = ?", Arrays.asList(1, 2));
+    }
+
+    @Test
+    public void indexMapKeysQueryValues() throws Throwable
+    {
+        createPopulatedMap("CREATE CUSTOM INDEX ON %s(KEYS(value)) USING 'StorageAttachedIndex'");
+        assertUnsupportedIndexOperator("SELECT * FROM %s WHERE value contains 'v1'");
+        assertEquals(2, execute("SELECT * FROM %s WHERE value contains 'v1' ALLOW FILTERING").size());
+    }
+
+    @Test
+    public void indexMapKeysQueryEntries() throws Throwable
+    {
+        createPopulatedMap("CREATE CUSTOM INDEX ON %s(KEYS(value)) USING 'StorageAttachedIndex'");
+        assertUnsupportedIndexOperator("SELECT * FROM %s WHERE value[1] = 'v1'");
+        assertEquals(2, execute("SELECT * FROM %s WHERE value[1] = 'v1' ALLOW FILTERING").size());
+    }
+
+    @Test
+    public void indexMapValuesQueryEq() throws Throwable
+    {
+        createPopulatedMap("CREATE CUSTOM INDEX ON %s(VALUES(value)) USING 'StorageAttachedIndex'");
+        assertInvalidMessage("Collection column 'value' (map<int, text>) cannot be restricted by a '=' relation",
+                "SELECT * FROM %s WHERE value = ?", Arrays.asList(1, 2));
+    }
+
+    @Test
+    public void indexMapValuesQueryKeys() throws Throwable
+    {
+        createPopulatedMap("CREATE CUSTOM INDEX ON %s(VALUES(value)) USING 'StorageAttachedIndex'");
+        assertUnsupportedIndexOperator("SELECT * FROM %s WHERE value contains key 1");
+        assertEquals(2, execute("SELECT * FROM %s WHERE value contains key 1 ALLOW FILTERING").size());
+    }
+
+    @Test
+    public void indexMapValuesQueryEntries() throws Throwable
+    {
+        createPopulatedMap("CREATE CUSTOM INDEX ON %s(VALUES(value)) USING 'StorageAttachedIndex'");
+        assertUnsupportedIndexOperator("SELECT * FROM %s WHERE value[1] = 'v1'");
+        assertEquals(2, execute("SELECT * FROM %s WHERE value[1] = 'v1' ALLOW FILTERING").size());
+    }
+
+    private void createPopulatedMap(String createIndex) throws Throwable
+    {
+        createTable("CREATE TABLE %s (pk int primary key, value map<int, text>)");
+        createIndex(createIndex);
+        waitForIndexQueryable();
+        execute("INSERT INTO %s (pk, value) VALUES (?, ?)", 1, new HashMap<Integer, String>() {{
+            put(1, "v1");
+            put(2, "v2");
+        }});
+        execute("INSERT INTO %s (pk, value) VALUES (?, ?)", 2, new HashMap<Integer, String>() {{
+            put(1, "v1");
+            put(2, "v3");
+        }});
+    }
+
+    private void createPopulatedFrozenMap(String createIndex) throws Throwable
+    {
+        createTable("CREATE TABLE %s (pk int primary key, value frozen<map<int, text>>)");
+        createIndex(createIndex);
+        waitForIndexQueryable();
+        execute("INSERT INTO %s (pk, value) VALUES (?, ?)", 1, new HashMap<Integer, String>() {{
+            put(1, "v1");
+            put(2, "v2");
+        }});
+        execute("INSERT INTO %s (pk, value) VALUES (?, ?)", 2, new HashMap<Integer, String>() {{
+            put(1, "v1");
+            put(2, "v3");
+        }});
+    }
+
+    private void createPopulatedFrozenList(String createIndex) throws Throwable
+    {
+        createTable("CREATE TABLE %s (pk int primary key, value frozen<list<int>>)");
+        createIndex(createIndex);
+        waitForIndexQueryable();
+        execute("INSERT INTO %s (pk, value) VALUES (?, ?)", 1, Arrays.asList(1, 2, 3));
+        execute("INSERT INTO %s (pk, value) VALUES (?, ?)", 2, Arrays.asList(1, 2, 3));
+        execute("INSERT INTO %s (pk, value) VALUES (?, ?)", 3, Arrays.asList(4, 5, 6));
+        execute("INSERT INTO %s (pk, value) VALUES (?, ?)", 4, Arrays.asList(1, 2, 7));
+    }
+
+    private void assertUnsupportedIndexOperator(String query, Object... values) throws Throwable
+    {
+        assertInvalidMessage(ALLOW_FILTERING_ERROR, query, values);
+    }
+}

test/unit/org/apache/cassandra/index/sai/cql/CompositePartitionKeyIndexTest.java

@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.cql;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import org.apache.cassandra.index.sai.SAITester;
+
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+public class CompositePartitionKeyIndexTest extends SAITester
+{
+    @Before
+    public void createTableAndIndex()
+    {
+        createTable("CREATE TABLE %s (pk1 int, pk2 text, val int, PRIMARY KEY((pk1, pk2)))");
+        createIndex("CREATE CUSTOM INDEX ON %s(pk1) USING 'StorageAttachedIndex'");
+        createIndex("CREATE CUSTOM INDEX ON %s(pk2) USING 'StorageAttachedIndex'");
+
+        disableCompaction();
+    }
+
+    private void insertData1() throws Throwable
+    {
+        execute("INSERT INTO %s (pk1, pk2, val) VALUES (1, '1', 1)");
+        execute("INSERT INTO %s (pk1, pk2, val) VALUES (2, '2', 2)");
+        execute("INSERT INTO %s (pk1, pk2, val) VALUES (3, '3', 3)");
+    }
+
+    private void insertData2() throws Throwable
+    {
+        execute("INSERT INTO %s (pk1, pk2, val) VALUES (4, '4', 4)");
+        execute("INSERT INTO %s (pk1, pk2, val) VALUES (5, '5', 5)");
+        execute("INSERT INTO %s (pk1, pk2, val) VALUES (6, '6', 6)");
+    }
+
+    @Test
+    public void queryFromMemtable() throws Throwable
+    {
+        insertData1();
+        insertData2();
+        runQueries();
+    }
+
+    private Object[] expectedRow(int index) {
+        return row(index, Integer.toString(index), index);
+    }
+
+    private void runQueries() throws Throwable
+    {
+        assertRowsIgnoringOrder(execute("SELECT * FROM %s WHERE pk1 = 2"),
+                expectedRow(2));
+
+        assertRowsIgnoringOrder(execute("SELECT * FROM %s WHERE pk1 > 1"),
+                expectedRow(2),
+                expectedRow(3),
+                expectedRow(4),
+                expectedRow(5),
+                expectedRow(6));
+
+        assertRowsIgnoringOrder(execute("SELECT * FROM %s WHERE pk1 >= 3"),
+                expectedRow(3),
+                expectedRow(4),
+                expectedRow(5),
+                expectedRow(6));
+
+        assertRowsIgnoringOrder(execute("SELECT * FROM %s WHERE pk1 < 3"),
+                expectedRow(1),
+                expectedRow(2));
+
+        assertRowsIgnoringOrder(execute("SELECT * FROM %s WHERE pk1 <= 3"),
+                expectedRow(1),
+                expectedRow(2),
+                expectedRow(3));
+
+        assertRowsIgnoringOrder(execute("SELECT * FROM %s WHERE pk2 = '2'"),
+                expectedRow(2));
+
+        assertRowsIgnoringOrder(execute("SELECT * FROM %s WHERE pk1 > 1 AND pk2 = '2' ALLOW FILTERING"),
+                expectedRow(2));
+
+        assertRowsIgnoringOrder(execute("SELECT * FROM %s WHERE pk1 = -1 AND pk2 = '2' ALLOW FILTERING"));
+
+        assertThatThrownBy(()->execute("SELECT * FROM %s WHERE pk1 = -1 AND val = 2"))
+                .hasMessageContaining("use ALLOW FILTERING");
+
+    }
+}

test/unit/org/apache/cassandra/index/sai/cql/DecimalLargeValueTest.java

@@ -0,0 +1,151 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.cql;
+
+import java.math.BigDecimal;
+
+import com.google.common.base.Preconditions;
+import org.apache.commons.lang3.StringUtils;
+import org.junit.Before;
+import org.junit.Test;
+
+import org.apache.cassandra.index.sai.SAITester;
+
+public class DecimalLargeValueTest extends SAITester
+{
+    @Before
+    public void createTableAndIndex()
+    {
+        requireNetwork();
+
+        createTable("CREATE TABLE %s (pk int, ck int, dec decimal, PRIMARY KEY (pk, ck))");
+
+        createIndex("CREATE CUSTOM INDEX ON %s(dec) USING 'StorageAttachedIndex'");
+
+        disableCompaction();
+    }
+
+    /**
+     * This test tries to induce rounding errors involving decimal values with wide significands.
+     *
+     * Two values are indexed:
+     * <ul>
+     * <li>1.0</li>
+     * <li>1.(510 zeros)1</li>
+     * </ul>
+     */
+    @Test
+    public void runQueriesWithDecimalValueCollision() throws Throwable
+    {
+        final int significandSizeInDecimalDigits = 512;
+        // String.repeat(int) exists in JDK 11 and later, but this line was introduced on JDK 8
+        String wideDecimalString = "1." + StringUtils.repeat('0', significandSizeInDecimalDigits - 2) + "1";
+        BigDecimal wideDecimal = new BigDecimal(wideDecimalString);
+        // Sanity checks that this value was actually constructed as intended
+        Preconditions.checkState(wideDecimal.precision() == significandSizeInDecimalDigits,
+                                 "expected precision %s, but got %s; string representation is \"%s\"",
+                                 significandSizeInDecimalDigits, wideDecimal.precision(), wideDecimalString);
+        Preconditions.checkState(wideDecimalString.equals(wideDecimal.toPlainString()),
+                                 "expected: %s; actual: %s", wideDecimalString, wideDecimal.toPlainString());
+
+        execute("INSERT INTO %s (pk, ck, dec) VALUES (0, 1, 1.0)");
+        execute("INSERT INTO %s (pk, ck, dec) VALUES (2, 0, " + wideDecimalString + ")");
+
+        // EQ queries
+        assertRows(execute("SELECT * FROM %s WHERE dec = 1.0"),
+                row(0, 1, BigDecimal.valueOf(1.0D)));
+
+        assertRows(execute("SELECT * FROM %s WHERE dec = " + wideDecimalString),
+                row(2, 0, wideDecimal));
+
+        // LT/LTE queries
+        assertRows(execute("SELECT * FROM %s WHERE dec < " + wideDecimalString),
+                row(0, 1, BigDecimal.valueOf(1.0D)));
+
+        assertRowsIgnoringOrder(execute("SELECT * FROM %s WHERE dec <= " + wideDecimalString),
+                row(0, 1, BigDecimal.valueOf(1.0D)),
+                row(2, 0, wideDecimal));
+
+        assertEmpty(execute("SELECT * FROM %s WHERE dec < 1.0"));
+
+        assertRows(execute("SELECT * FROM %s WHERE dec <= 1.0"),
+                row(0, 1, BigDecimal.valueOf(1.0D)));
+
+        // GT/GTE queries
+        assertRows(execute("SELECT * FROM %s WHERE dec > 1.0"),
+                row(2, 0, wideDecimal));
+
+        assertRowsIgnoringOrder(execute("SELECT * FROM %s WHERE dec >= " + wideDecimalString),
+                row(2, 0, wideDecimal));
+
+        assertEmpty(execute("SELECT * FROM %s WHERE dec > " + wideDecimalString));
+
+        assertRowsIgnoringOrder(execute("SELECT * FROM %s WHERE dec >= 1.0"),
+                row(0, 1, BigDecimal.valueOf(1.0D)),
+                row(2, 0, wideDecimal));
+    }
+    /**
+     * This is a control method with small (two-significant-digit) values.
+     */
+    @Test
+    public void runQueriesWithoutCollisions() throws Throwable
+    {
+        execute("INSERT INTO %s (pk, ck, dec) VALUES (-2, 1, 2.2)");
+        execute("INSERT INTO %s (pk, ck, dec) VALUES (-2, 2, 2.2)");
+        execute("INSERT INTO %s (pk, ck, dec) VALUES (-1, 1, 1.1)");
+        execute("INSERT INTO %s (pk, ck, dec) VALUES (0, 1, 0)");
+        execute("INSERT INTO %s (pk, ck, dec) VALUES (1, 1, 1.1)");
+        execute("INSERT INTO %s (pk, ck, dec) VALUES (2, 1, 2.2)");
+        execute("INSERT INTO %s (pk, ck, dec) VALUES (2, 2, 2.2)");
+
+        // EQ queries
+        assertRowsIgnoringOrder(execute("SELECT * FROM %s WHERE dec = 1.1"),
+                row(-1, 1, BigDecimal.valueOf(1.1D)),
+                row(1, 1, BigDecimal.valueOf(1.1D)));
+
+        assertRowsIgnoringOrder(execute("SELECT * FROM %s WHERE dec = 2.2"),
+                row(-2, 1, BigDecimal.valueOf(2.2D)),
+                row(-2, 2, BigDecimal.valueOf(2.2D)),
+                row(2, 1, BigDecimal.valueOf(2.2D)),
+                row(2, 2, BigDecimal.valueOf(2.2D)));
+
+        // LT/LTE queries
+        assertRowsIgnoringOrder(execute("SELECT * FROM %s WHERE dec < 1.1"),
+                row(0, 1, BigDecimal.valueOf(0)));
+
+        assertRowsIgnoringOrder(execute("SELECT * FROM %s WHERE dec <= 1.1"),
+                row(-1, 1, BigDecimal.valueOf(1.1D)),
+                row(0, 1, BigDecimal.valueOf(0)),
+                row(1, 1, BigDecimal.valueOf(1.1D)));
+
+        // GT/GTE queries
+        assertRowsIgnoringOrder(execute("SELECT * FROM %s WHERE dec > 1.1"),
+                row(-2, 1, BigDecimal.valueOf(2.2D)),
+                row(-2, 2, BigDecimal.valueOf(2.2D)),
+                row(2, 1, BigDecimal.valueOf(2.2D)),
+                row(2, 2, BigDecimal.valueOf(2.2D)));
+
+        assertRowsIgnoringOrder(execute("SELECT * FROM %s WHERE dec >= 1.1"),
+                row(-2, 1, BigDecimal.valueOf(2.2D)),
+                row(-2, 2, BigDecimal.valueOf(2.2D)),
+                row(-1, 1, BigDecimal.valueOf(1.1D)),
+                row(1, 1, BigDecimal.valueOf(1.1D)),
+                row(2, 1, BigDecimal.valueOf(2.2D)),
+                row(2, 2, BigDecimal.valueOf(2.2D)));
+    }
+}

test/unit/org/apache/cassandra/index/sai/cql/DuplicateRowIDTest.java

@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.cql;
+
+import java.util.List;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import com.datastax.driver.core.Row;
+import org.apache.cassandra.index.sai.SAITester;
+
+import static org.junit.Assert.assertEquals;
+
+public class DuplicateRowIDTest extends SAITester
+{
+    @BeforeClass
+    public static void setupCluster()
+    {
+        requireNetwork();
+    }
+
+    @Test
+    public void shouldTolerateDuplicatedRowIDsAfterMemtableUpdates() throws Throwable
+    {
+        createTable("CREATE TABLE %s (id1 TEXT PRIMARY KEY, v1 INT)");
+        createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1"));
+        waitForIndexQueryable();
+
+        for (int i = 0; i < 2048; ++i)
+        {
+            execute("INSERT INTO %s (id1, v1) VALUES (?, ?)", Integer.toString(i % 10), i);
+        }
+
+        // tolerate query duplicates from memtable
+        List<Row> rows = executeNet("SELECT * FROM %s WHERE v1 > 0").all();
+        assertEquals(10, rows.size());
+
+        for (int i = 0; i < 2048; ++i)
+        {
+            execute("INSERT INTO %s (id1, v1) VALUES (?, ?)", Integer.toString(i % 10), i);
+        }
+
+        // tolerate duplicates from memtable and sstable
+        rows = executeNet("SELECT * FROM %s WHERE v1 > 0").all();
+        assertEquals(10, rows.size());
+    }
+}

test/unit/org/apache/cassandra/index/sai/cql/IndexQuerySupport.java

@@ -0,0 +1,482 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+package org.apache.cassandra.index.sai.cql;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+
+import com.google.common.base.MoreObjects;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
+
+import org.apache.cassandra.cql3.Operator;
+import org.apache.cassandra.db.marshal.InetAddressType;
+import org.apache.cassandra.db.marshal.SimpleDateType;
+import org.apache.cassandra.db.marshal.TimeType;
+import org.apache.cassandra.db.marshal.TimestampType;
+import org.apache.cassandra.db.marshal.UUIDType;
+import org.apache.cassandra.index.sai.plan.StorageAttachedIndexSearcher;
+import org.apache.cassandra.utils.Pair;
+import org.hamcrest.Matchers;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThat;
+
+/**
+ * A CQL-based test framework for simulating queries across as much of the index state space as possible.
+ *
+ * This includes, but need not be limited to...
+ *
+ * 1.) ...queries on the same data as it migrates through the write path and storage engine.
+ * 2.) ...queries across all supported native data types.
+ * 3.) ...queries for all supported operators and value boundaries.
+ * 4.) ...queries for varying write, update, delete, and TTL workloads.
+ * 5.) ...queries across varying primary key and table structures.
+ * 6.) ...queries across static, normal, and clustering column types.
+ * 7.) ...queries across various paging and limit settings.
+ *
+ * IMPORTANT: This class is shared between the single-node SAITester based classes and the
+ * multi-node distributed classes. It must not reference SAITester or CQLTester directly
+ * to avoid static loading and initialisation.
+ */
+public class IndexQuerySupport
+{
+    public static List<BaseQuerySet> BASE_QUERY_SETS = ImmutableList.of(new BaseQuerySet(10, 5),
+                                                                        new BaseQuerySet(10, 9),
+                                                                        new BaseQuerySet(10, 10),
+                                                                        new BaseQuerySet(10, Integer.MAX_VALUE),
+                                                                        new BaseQuerySet(24, 10),
+                                                                        new BaseQuerySet(24, 100),
+                                                                        new BaseQuerySet(24, Integer.MAX_VALUE));
+
+    public static List<BaseQuerySet> COMPOSITE_PARTITION_QUERY_SETS = ImmutableList.of(new CompositePartitionQuerySet(10, 5),
+                                                                                       new CompositePartitionQuerySet(10, 10),
+                                                                                       new CompositePartitionQuerySet(10, Integer.MAX_VALUE),
+                                                                                       new CompositePartitionQuerySet(24, 10),
+                                                                                       new CompositePartitionQuerySet(24, 100),
+                                                                                       new CompositePartitionQuerySet(24, Integer.MAX_VALUE));
+
+    public static List<BaseQuerySet> STATIC_QUERY_SETS = ImmutableList.of(new StaticColumnQuerySet(10, 5),
+                                                                          new StaticColumnQuerySet(10, 10),
+                                                                          new StaticColumnQuerySet(10, Integer.MAX_VALUE),
+                                                                          new StaticColumnQuerySet(24, 10),
+                                                                          new StaticColumnQuerySet(24, 100),
+                                                                          new StaticColumnQuerySet(24, Integer.MAX_VALUE));
+
+    public static void writeLifecycle(BaseDataModel.Executor executor, BaseDataModel dataModel, List<BaseQuerySet> sets) throws Throwable
+    {
+        dataModel.createTables(executor);
+
+        dataModel.createIndexes(executor);
+
+        // queries against Memtable adjacent in-memory indexes
+        dataModel.insertRows(executor);
+        executeQueries(dataModel, executor, sets);
+    }
+
+    public static void timeToLive(BaseDataModel.Executor executor, BaseDataModel dataModel, List<BaseQuerySet> sets) throws Throwable
+    {
+        dataModel.createTables(executor);
+
+        dataModel.createIndexes(executor);
+        dataModel.insertRowsWithTTL(executor);
+
+        // Wait for the TTL to become effective:
+        TimeUnit.SECONDS.sleep(BaseDataModel.DEFAULT_TTL_SECONDS);
+
+        // Make sure TTLs are reflected in our query results from the Memtable:
+        executeQueries(dataModel, executor, sets);
+    }
+
+    private static void executeQueries(BaseDataModel dataModel, BaseDataModel.Executor executor, List<BaseQuerySet> sets) throws Throwable
+    {
+        for (BaseQuerySet set : sets)
+        {
+            set.execute(executor, dataModel);
+        }
+    }
+
+    static class StaticColumnQuerySet extends BaseQuerySet
+    {
+        StaticColumnQuerySet(int limit, int fetchSize)
+        {
+            super(limit, fetchSize);
+        }
+
+        public void execute(BaseDataModel.Executor tester, BaseDataModel model) throws Throwable
+        {
+            super.execute(tester, model);
+
+            query(tester, model, BaseDataModel.STATIC_INT_COLUMN, Operator.EQ, 1845);
+            query(tester, model, BaseDataModel.STATIC_INT_COLUMN, Operator.LT, 1845);
+            query(tester, model, BaseDataModel.STATIC_INT_COLUMN, Operator.LTE, 1845);
+            query(tester, model, BaseDataModel.STATIC_INT_COLUMN, Operator.GT, 1845);
+            query(tester, model, BaseDataModel.STATIC_INT_COLUMN, Operator.GTE, 1845);
+            query(tester, model, BaseDataModel.STATIC_INT_COLUMN, Operator.EQ, 1909);
+            query(tester, model, BaseDataModel.STATIC_INT_COLUMN, Operator.LT, 1787);
+            query(tester, model, BaseDataModel.STATIC_INT_COLUMN, Operator.GT, 1910);
+
+            rangeQuery(tester, model, BaseDataModel.STATIC_INT_COLUMN, 1845, 1909);
+        }
+    }
+
+    static class CompositePartitionQuerySet extends BaseQuerySet
+    {
+        CompositePartitionQuerySet(int limit, int fetchSize)
+        {
+            super(limit, fetchSize);
+        }
+
+        public void execute(BaseDataModel.Executor tester, BaseDataModel model) throws Throwable
+        {
+            super.execute(tester, model);
+
+            for(Pair<String, String> partitionKeyComponent: model.keyColumns)
+            {
+                String partitionKeyComponentName = partitionKeyComponent.left;
+                query(tester, model, partitionKeyComponentName, Operator.EQ, 0);
+                query(tester, model, partitionKeyComponentName, Operator.GT, 0);
+                query(tester, model, partitionKeyComponentName, Operator.LTE, 2);
+                query(tester, model, partitionKeyComponentName, Operator.GTE, -1);
+                query(tester, model, partitionKeyComponentName, Operator.LT, 50);
+                query(tester, model, partitionKeyComponentName, Operator.GT, 0);
+            }
+
+            String firstPartitionKey = model.keyColumns.get(0).left;
+            String secondPartitionKey = model.keyColumns.get(1).left;
+            List<Operator> numericOperators = Arrays.asList(Operator.EQ, Operator.GT, Operator.LT, Operator.GTE, Operator.LTE);
+            List<List<Operator>> combinations = Lists.cartesianProduct(numericOperators, numericOperators).stream()
+                                                     .filter(p-> p.get(0) != Operator.EQ || p.get(1) != Operator.EQ) //If both are EQ the entire partition is specified
+                                                     .collect(Collectors.toList());
+            for(List<Operator> operators : combinations)
+            {
+                andQuery(tester,
+                         model,
+                         firstPartitionKey, operators.get(0), 2,
+                         secondPartitionKey, operators.get(1), 2,
+                         false);
+            }
+        }
+    }
+
+    public static class BaseQuerySet
+    {
+        final int limit;
+        final int fetchSize;
+
+        BaseQuerySet(int limit, int fetchSize)
+        {
+            this.limit = limit;
+            this.fetchSize = fetchSize;
+        }
+
+        void execute(BaseDataModel.Executor tester, BaseDataModel model) throws Throwable
+        {
+            query(tester, model, BaseDataModel.ASCII_COLUMN, Operator.EQ, "MA");
+            query(tester, model, BaseDataModel.ASCII_COLUMN, Operator.EQ, "LA");
+            query(tester, model, BaseDataModel.ASCII_COLUMN, Operator.EQ, "XX");
+
+            query(tester, model, BaseDataModel.BIGINT_COLUMN, Operator.EQ, 4800000000L);
+            query(tester, model, BaseDataModel.BIGINT_COLUMN, Operator.EQ, 5000000000L);
+            query(tester, model, BaseDataModel.BIGINT_COLUMN, Operator.LT, 5000000000L);
+            query(tester, model, BaseDataModel.BIGINT_COLUMN, Operator.LTE, 5000000000L);
+            query(tester, model, BaseDataModel.BIGINT_COLUMN, Operator.GT, 5000000000L);
+            query(tester, model, BaseDataModel.BIGINT_COLUMN, Operator.GTE, 5000000000L);
+            query(tester, model, BaseDataModel.BIGINT_COLUMN, Operator.EQ, 22L);
+            query(tester, model, BaseDataModel.BIGINT_COLUMN, Operator.LT, 400000000L);
+            query(tester, model, BaseDataModel.BIGINT_COLUMN, Operator.GT, 10000000000L);
+
+            rangeQuery(tester, model, BaseDataModel.BIGINT_COLUMN, 3000000000L, 7000000000L);
+
+            query(tester, model, BaseDataModel.DATE_COLUMN, Operator.EQ, SimpleDateType.instance.fromString("2013-06-10"));
+            query(tester, model, BaseDataModel.DATE_COLUMN, Operator.EQ, SimpleDateType.instance.fromString("2013-06-17"));
+            query(tester, model, BaseDataModel.DATE_COLUMN, Operator.LT, SimpleDateType.instance.fromString("2013-06-17"));
+            query(tester, model, BaseDataModel.DATE_COLUMN, Operator.LTE, SimpleDateType.instance.fromString("2013-06-17"));
+            query(tester, model, BaseDataModel.DATE_COLUMN, Operator.GT, SimpleDateType.instance.fromString("2013-06-17"));
+            query(tester, model, BaseDataModel.DATE_COLUMN, Operator.GTE, SimpleDateType.instance.fromString("2013-06-17"));
+            query(tester, model, BaseDataModel.DATE_COLUMN, Operator.EQ, SimpleDateType.instance.fromString("2017-01-01"));
+            query(tester, model, BaseDataModel.DATE_COLUMN, Operator.LT, SimpleDateType.instance.fromString("2000-01-01"));
+            query(tester, model, BaseDataModel.DATE_COLUMN, Operator.GT, SimpleDateType.instance.fromString("2020-01-01"));
+
+            rangeQuery(tester, model, BaseDataModel.DATE_COLUMN, SimpleDateType.instance.fromString("2013-06-17"), SimpleDateType.instance.fromString("2018-06-19"));
+
+            query(tester, model, BaseDataModel.DOUBLE_COLUMN, Operator.EQ, 43203.90);
+            query(tester, model, BaseDataModel.DOUBLE_COLUMN, Operator.EQ, 7800.06);
+            query(tester, model, BaseDataModel.DOUBLE_COLUMN, Operator.LT, 82169.62);
+            query(tester, model, BaseDataModel.DOUBLE_COLUMN, Operator.LTE, 82169.62);
+            query(tester, model, BaseDataModel.DOUBLE_COLUMN, Operator.GT, 82169.62);
+            query(tester, model, BaseDataModel.DOUBLE_COLUMN, Operator.GTE, 82169.62);
+            query(tester, model, BaseDataModel.DOUBLE_COLUMN, Operator.EQ, 82169.60);
+            query(tester, model, BaseDataModel.DOUBLE_COLUMN, Operator.LT, 1948.54);
+            query(tester, model, BaseDataModel.DOUBLE_COLUMN, Operator.GT, 570640.95);
+
+            rangeQuery(tester, model, BaseDataModel.DOUBLE_COLUMN, 56538.90, 113594.08);
+
+            query(tester, model, BaseDataModel.FLOAT_COLUMN, Operator.EQ, 10.2f);
+            query(tester, model, BaseDataModel.FLOAT_COLUMN, Operator.EQ, 1.9f);
+            query(tester, model, BaseDataModel.FLOAT_COLUMN, Operator.LT, 5.3f);
+            query(tester, model, BaseDataModel.FLOAT_COLUMN, Operator.LTE, 5.3f);
+            query(tester, model, BaseDataModel.FLOAT_COLUMN, Operator.GT, 5.3f);
+            query(tester, model, BaseDataModel.FLOAT_COLUMN, Operator.GTE, 5.3f);
+            query(tester, model, BaseDataModel.FLOAT_COLUMN, Operator.EQ, 5.9f);
+            query(tester, model, BaseDataModel.FLOAT_COLUMN, Operator.LT, 1.8f);
+            query(tester, model, BaseDataModel.FLOAT_COLUMN, Operator.GT, 10.2f);
+
+            rangeQuery(tester, model, BaseDataModel.FLOAT_COLUMN, 4.6f, 6.7f);
+
+            query(tester, model, BaseDataModel.INET_COLUMN, Operator.EQ, InetAddressType.instance.fromString("170.63.206.57"));
+            query(tester, model, BaseDataModel.INET_COLUMN, Operator.EQ, InetAddressType.instance.fromString("170.63.206.56"));
+            query(tester, model, BaseDataModel.INET_COLUMN, Operator.EQ, InetAddressType.instance.fromString("205.204.196.65"));
+            query(tester, model, BaseDataModel.INET_COLUMN, Operator.EQ, InetAddressType.instance.fromString("164.165.67.10"));
+            query(tester, model, BaseDataModel.INET_COLUMN, Operator.EQ, InetAddressType.instance.fromString("204.196.242.71"));
+
+            rangeQuery(tester, model, BaseDataModel.INT_COLUMN, 2977853, 6784240);
+
+            query(tester, model, BaseDataModel.SMALLINT_COLUMN, Operator.EQ, (short) 164);
+            query(tester, model, BaseDataModel.SMALLINT_COLUMN, Operator.LT, (short) 164);
+            query(tester, model, BaseDataModel.SMALLINT_COLUMN, Operator.LTE, (short) 164);
+            query(tester, model, BaseDataModel.SMALLINT_COLUMN, Operator.GT, (short) 164);
+            query(tester, model, BaseDataModel.SMALLINT_COLUMN, Operator.GTE, (short) 164);
+            query(tester, model, BaseDataModel.SMALLINT_COLUMN, Operator.EQ, (short) 2);
+            query(tester, model, BaseDataModel.SMALLINT_COLUMN, Operator.LT, (short) 30);
+            query(tester, model, BaseDataModel.SMALLINT_COLUMN, Operator.GT, (short) 1861);
+
+            rangeQuery(tester, model, BaseDataModel.SMALLINT_COLUMN, (short) 126, (short) 383);
+
+            query(tester, model, BaseDataModel.TINYINT_COLUMN, Operator.EQ, (byte) 16);
+            query(tester, model, BaseDataModel.TINYINT_COLUMN, Operator.LT, (byte) 16);
+            query(tester, model, BaseDataModel.TINYINT_COLUMN, Operator.LTE, (byte) 16);
+            query(tester, model, BaseDataModel.TINYINT_COLUMN, Operator.GT, (byte) 16);
+            query(tester, model, BaseDataModel.TINYINT_COLUMN, Operator.GTE, (byte) 16);
+            query(tester, model, BaseDataModel.TINYINT_COLUMN, Operator.EQ, (byte) 1);
+            query(tester, model, BaseDataModel.TINYINT_COLUMN, Operator.LT, (byte) 2);
+            query(tester, model, BaseDataModel.TINYINT_COLUMN, Operator.GT, (byte) 117);
+
+            rangeQuery(tester, model, BaseDataModel.TINYINT_COLUMN, (byte) 12, (byte) 47);
+
+            query(tester, model, BaseDataModel.TEXT_COLUMN, Operator.EQ, "Alaska");
+            query(tester, model, BaseDataModel.TEXT_COLUMN, Operator.EQ, "Wyoming");
+            query(tester, model, BaseDataModel.TEXT_COLUMN, Operator.EQ, "Franklin");
+            query(tester, model, BaseDataModel.TEXT_COLUMN, Operator.EQ, "State of Michigan");
+            query(tester, model, BaseDataModel.TEXT_COLUMN, Operator.EQ, "Michigan");
+            query(tester, model, BaseDataModel.TEXT_COLUMN, Operator.EQ, "Louisiana");
+            query(tester, model, BaseDataModel.TEXT_COLUMN, Operator.EQ, "Massachusetts");
+
+            query(tester, model, BaseDataModel.TIME_COLUMN, Operator.EQ, TimeType.instance.fromString("00:43:07"));
+            query(tester, model, BaseDataModel.TIME_COLUMN, Operator.LT, TimeType.instance.fromString("00:43:07"));
+            query(tester, model, BaseDataModel.TIME_COLUMN, Operator.LTE, TimeType.instance.fromString("00:43:07"));
+            query(tester, model, BaseDataModel.TIME_COLUMN, Operator.GT, TimeType.instance.fromString("00:43:07"));
+            query(tester, model, BaseDataModel.TIME_COLUMN, Operator.GTE, TimeType.instance.fromString("00:43:07"));
+            query(tester, model, BaseDataModel.TIME_COLUMN, Operator.EQ, TimeType.instance.fromString("00:15:57"));
+            query(tester, model, BaseDataModel.TIME_COLUMN, Operator.LT, TimeType.instance.fromString("00:15:50"));
+            query(tester, model, BaseDataModel.TIME_COLUMN, Operator.GT, TimeType.instance.fromString("01:30:45"));
+
+            rangeQuery(tester, model, BaseDataModel.TIME_COLUMN, TimeType.instance.fromString("00:38:13"), TimeType.instance.fromString("00:56:07"));
+
+            query(tester, model, BaseDataModel.TIMESTAMP_COLUMN, Operator.EQ, TimestampType.instance.fromString("2013-06-17T00:00:00"));
+            query(tester, model, BaseDataModel.TIMESTAMP_COLUMN, Operator.LT, TimestampType.instance.fromString("2013-06-17T00:00:00"));
+            query(tester, model, BaseDataModel.TIMESTAMP_COLUMN, Operator.LTE, TimestampType.instance.fromString("2013-06-17T00:00:00"));
+            query(tester, model, BaseDataModel.TIMESTAMP_COLUMN, Operator.GT, TimestampType.instance.fromString("2013-06-17T00:00:00"));
+            query(tester, model, BaseDataModel.TIMESTAMP_COLUMN, Operator.GTE, TimestampType.instance.fromString("2013-06-17T00:00:00"));
+            query(tester, model, BaseDataModel.TIMESTAMP_COLUMN, Operator.EQ, TimestampType.instance.fromString("2017-01-01T00:00:00"));
+            query(tester, model, BaseDataModel.TIMESTAMP_COLUMN, Operator.LT, TimestampType.instance.fromString("2000-01-01T00:00:00"));
+            query(tester, model, BaseDataModel.TIMESTAMP_COLUMN, Operator.GT, TimestampType.instance.fromString("2020-01-01T00:00:00"));
+
+            rangeQuery(tester, model, BaseDataModel.TIMESTAMP_COLUMN,
+                       TimestampType.instance.fromString("2013-6-17T00:00:00"),
+                       TimestampType.instance.fromString("2018-6-19T00:00:00"));
+
+            query(tester, model, BaseDataModel.UUID_COLUMN, Operator.EQ, UUIDType.instance.fromString("e37394dc-d17b-11e8-a8d5-f2801f1b9fd1"));
+            query(tester, model, BaseDataModel.UUID_COLUMN, Operator.EQ, UUIDType.instance.fromString("752355f8-405b-4d94-88f3-9992cda30f1e"));
+            query(tester, model, BaseDataModel.UUID_COLUMN, Operator.EQ, UUIDType.instance.fromString("ac0aa734-d17f-11e8-a8d5-f2801f1b9fd1"));
+            query(tester, model, BaseDataModel.UUID_COLUMN, Operator.EQ, UUIDType.instance.fromString("c6eec0b0-0eef-40e8-ac38-3a82110443e4"));
+            query(tester, model, BaseDataModel.UUID_COLUMN, Operator.EQ, UUIDType.instance.fromString("e37394dc-d17b-11e8-a8d5-f2801f1b9fd1"));
+
+            query(tester, model, BaseDataModel.TIMEUUID_COLUMN, Operator.EQ, UUIDType.instance.fromString("ee6136d2-d17c-11e8-a8d5-f2801f1b9fd1"));
+            query(tester, model, BaseDataModel.TIMEUUID_COLUMN, Operator.LT, UUIDType.instance.fromString("ee6136d2-d17c-11e8-a8d5-f2801f1b9fd1"));
+            query(tester, model, BaseDataModel.TIMEUUID_COLUMN, Operator.LTE, UUIDType.instance.fromString("ee6136d2-d17c-11e8-a8d5-f2801f1b9fd1"));
+            query(tester, model, BaseDataModel.TIMEUUID_COLUMN, Operator.GT, UUIDType.instance.fromString("ee6136d2-d17c-11e8-a8d5-f2801f1b9fd1"));
+            query(tester, model, BaseDataModel.TIMEUUID_COLUMN, Operator.GTE, UUIDType.instance.fromString("ee6136d2-d17c-11e8-a8d5-f2801f1b9fd1"));
+            query(tester, model, BaseDataModel.TIMEUUID_COLUMN, Operator.EQ, UUIDType.instance.fromString("2a421a68-d182-11e8-a8d5-f2801f1b9fd1"));
+
+            andQuery(tester, model,
+                     BaseDataModel.TIMESTAMP_COLUMN, Operator.GTE, TimestampType.instance.fromString("2013-06-20T00:00:00"),
+                     BaseDataModel.UUID_COLUMN, Operator.EQ, UUIDType.instance.fromString("752355f8-405b-4d94-88f3-9992cda30f1e"),
+                     false);
+
+            andQuery(tester, model,
+                     BaseDataModel.TIMESTAMP_COLUMN, Operator.GTE, TimestampType.instance.fromString("2018-06-20T00:00:00"),
+                     BaseDataModel.TEXT_COLUMN, Operator.EQ, "Texas",
+                     false);
+
+            andQuery(tester, model,
+                     BaseDataModel.SMALLINT_COLUMN, Operator.LTE, (short) 126,
+                     BaseDataModel.TINYINT_COLUMN, Operator.LTE, (byte) 9,
+                     false);
+
+            andQuery(tester, model,
+                     BaseDataModel.SMALLINT_COLUMN, Operator.LTE, (short) 126,
+                     BaseDataModel.NON_INDEXED_COLUMN, Operator.GT, 0,
+                     true);
+
+            andQuery(tester, model,
+                     BaseDataModel.TEXT_COLUMN, Operator.EQ, "Alaska",
+                     BaseDataModel.NON_INDEXED_COLUMN, Operator.EQ, 2,
+                     true);
+
+
+            andQuery(tester, model,
+                     BaseDataModel.UUID_COLUMN, Operator.EQ, UUIDType.instance.fromString("e37394dc-d17b-11e8-a8d5-f2801f1b9fd1"),
+                     BaseDataModel.NON_INDEXED_COLUMN, Operator.LT, 3,
+                     true);
+
+            // with partition column filtering
+            String firstPartitionKey = model.keyColumns().get(0).left;
+
+            andQuery(tester, model,
+                     BaseDataModel.TEXT_COLUMN, Operator.EQ, "Alaska",
+                     firstPartitionKey, Operator.EQ, 0,
+                     true);
+
+            andQuery(tester, model,
+                     BaseDataModel.TEXT_COLUMN, Operator.EQ, "Kentucky",
+                     firstPartitionKey, Operator.GT, 4,
+                     true);
+
+            andQuery(tester, model,
+                     BaseDataModel.TEXT_COLUMN, Operator.EQ, "Wyoming",
+                     firstPartitionKey, Operator.LT, 200,
+                     true);
+
+            if (model.keyColumns().size() > 1)
+            {
+                String secondPrimaryKey = model.keyColumns().get(1).left;
+
+                andQuery(tester, model,
+                         BaseDataModel.BIGINT_COLUMN, Operator.EQ, 4800000000L,
+                         secondPrimaryKey, Operator.EQ, 0,
+                         true);
+
+                andQuery(tester, model,
+                         BaseDataModel.DOUBLE_COLUMN, Operator.EQ, 82169.60,
+                         secondPrimaryKey, Operator.GT, 0,
+                         true);
+
+                andQuery(tester, model,
+                         BaseDataModel.DOUBLE_COLUMN, Operator.LT, 1948.54,
+                         secondPrimaryKey, Operator.LTE, 2,
+                         true);
+
+                andQuery(tester, model,
+                         BaseDataModel.TEXT_COLUMN, Operator.EQ, "Alaska",
+                         firstPartitionKey, Operator.EQ, 0,
+                         secondPrimaryKey, Operator.GTE, -1);
+
+                andQuery(tester, model,
+                         BaseDataModel.TEXT_COLUMN, Operator.EQ, "Kentucky",
+                         firstPartitionKey, Operator.GT, 4,
+                         secondPrimaryKey, Operator.LT, 50);
+
+                andQuery(tester, model,
+                         BaseDataModel.TEXT_COLUMN, Operator.EQ, "Wyoming",
+                         firstPartitionKey, Operator.LT, 200,
+                         secondPrimaryKey, Operator.GT, 0);
+            }
+        }
+
+        void query(BaseDataModel.Executor tester, BaseDataModel model, String column, Operator operator, Object value) throws Throwable
+        {
+            String query = String.format(BaseDataModel.SIMPLE_SELECT_TEMPLATE, BaseDataModel.ASCII_COLUMN, column, operator);
+            String queryValidator = String.format(BaseDataModel.SIMPLE_SELECT_WITH_FILTERING_TEMPLATE, BaseDataModel.ASCII_COLUMN, column, operator);
+            validate(tester, model, query, queryValidator, value, limit);
+        }
+
+        void andQuery(BaseDataModel.Executor tester, BaseDataModel model,
+                      String column1, Operator operator1, Object value1,
+                      String column2, Operator operator2, Object value2,
+                      boolean filtering) throws Throwable
+        {
+            String query = String.format(filtering ? BaseDataModel.TWO_CLAUSE_AND_QUERY_FILTERING_TEMPLATE : BaseDataModel.TWO_CLAUSE_AND_QUERY_TEMPLATE,
+                                         BaseDataModel.ASCII_COLUMN, column1, operator1, column2, operator2);
+
+            String queryValidator = String.format(BaseDataModel.TWO_CLAUSE_AND_QUERY_FILTERING_TEMPLATE,
+                                                  BaseDataModel.ASCII_COLUMN, column1, operator1, column2, operator2);
+
+            validate(tester, model,query, queryValidator, value1, value2, limit);
+        }
+
+        void andQuery(BaseDataModel.Executor tester, BaseDataModel model,
+                      String column1, Operator operator1, Object value1,
+                      String column2, Operator operator2, Object value2,
+                      String column3, Operator operator3, Object value3) throws Throwable
+        {
+            // TODO: If we support indexes in all columns, ALLOW FILTERING might go away here...
+            String query = String.format(BaseDataModel.THREE_CLAUSE_AND_QUERY_FILTERING_TEMPLATE,
+                                         BaseDataModel.ASCII_COLUMN, column1, operator1, column2, operator2, column3, operator3);
+
+            String queryValidator = String.format(BaseDataModel.THREE_CLAUSE_AND_QUERY_FILTERING_TEMPLATE,
+                                                  BaseDataModel.ASCII_COLUMN, column1, operator1, column2, operator2, column3, operator3);
+
+            validate(tester, model, query, queryValidator, value1, value2, value3, limit);
+        }
+
+        void rangeQuery(BaseDataModel.Executor tester, BaseDataModel model, String column, Object value1, Object value2) throws Throwable
+        {
+            String template = "SELECT %s FROM %%s WHERE %s > ? AND %s < ? LIMIT ? ALLOW FILTERING";
+            String templateWithFiltering = "SELECT %s FROM %%s WHERE %s > ? AND %s < ? LIMIT ? ALLOW FILTERING";
+
+            String query = String.format(template, BaseDataModel.ASCII_COLUMN, column, column);
+            String queryValidator = String.format(templateWithFiltering, BaseDataModel.ASCII_COLUMN, column, column);
+            validate(tester, model, query, queryValidator, value1, value2, limit);
+        }
+
+        private void validate(BaseDataModel.Executor tester, BaseDataModel model, String query, String validator, Object... values) throws Throwable
+        {
+            try
+            {
+                tester.counterReset();
+
+                List<Object> actual = model.executeIndexed(tester, query, fetchSize, values);
+
+                // This could be more strict, but it serves as a reasonable paging-aware lower bound:
+                int pageCount = (int) Math.ceil(actual.size() / (double) Math.min(actual.size(), fetchSize));
+                assertThat("Expected more calls to " + StorageAttachedIndexSearcher.class, tester.getCounter(), Matchers.greaterThanOrEqualTo((long) Math.max(1, pageCount)));
+
+                List<Object> expected = model.executeNonIndexed(tester, validator, fetchSize, values);
+
+                assertEquals(expected, actual);
+            }
+            catch (Throwable ex)
+            {
+                ex.printStackTrace();
+                throw ex;
+            }
+        }
+
+        @Override
+        public String toString()
+        {
+            return MoreObjects.toStringHelper(this).add("limit", limit).add("fetchSize", fetchSize).toString();
+        }
+    }
+}

test/unit/org/apache/cassandra/index/sai/cql/InetAddressTypeEquivalencyTest.java

@@ -0,0 +1,200 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.cql;
+
+import java.net.InetAddress;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import org.apache.cassandra.index.sai.SAITester;
+import org.apache.cassandra.index.sai.cql.types.InetTest;
+
+/**
+ * This is testing that we can query ipv4 addresses using ipv6 equivalent addresses.
+ *
+ * The remaining InetAddressType tests are now handled by {@link InetTest}
+ */
+public class InetAddressTypeEquivalencyTest extends SAITester
+{
+    @Before
+    public void createTableAndIndex()
+    {
+        requireNetwork();
+
+        createTable("CREATE TABLE %s (pk int, ck int, ip inet, PRIMARY KEY(pk, ck ))");
+
+        disableCompaction();
+    }
+
+    @Test
+    public void mixedWorkloadQueryTest() throws Throwable
+    {
+        createIndex("CREATE CUSTOM INDEX ON %s(ip) USING 'StorageAttachedIndex'");
+
+        execute("INSERT INTO %s (pk, ck, ip) VALUES (1, 1, '127.0.0.1')");
+        execute("INSERT INTO %s (pk, ck, ip) VALUES (1, 2, '127.0.0.1')");
+        execute("INSERT INTO %s (pk, ck, ip) VALUES (1, 3, '127.0.0.2')");
+        execute("INSERT INTO %s (pk, ck, ip) VALUES (1, 4, '::ffff:7f00:3')");
+        execute("INSERT INTO %s (pk, ck, ip) VALUES (1, 5, '2002:4559:1fe2::4559:1fe2')");
+        execute("INSERT INTO %s (pk, ck, ip) VALUES (1, 6, '2002:4559:1fe2::4559:1fe2')");
+        execute("INSERT INTO %s (pk, ck, ip) VALUES (1, 7, '2002:4559:1fe2::4559:1fe2')");
+        execute("INSERT INTO %s (pk, ck, ip) VALUES (1, 8, '2002:4559:1fe2::4559:1fe3')");
+
+        runQueries();
+    }
+
+    private void runQueries() throws Throwable
+    {
+        // EQ single ipv4 address
+        assertRows(execute("SELECT * FROM %s WHERE ip = '127.0.0.1'"),
+                row(1, 1, InetAddress.getByName("127.0.0.1")),
+                row(1, 2, InetAddress.getByName("127.0.0.1")));
+
+        // EQ mapped-ipv4 address
+        assertRows(execute("SELECT * FROM %s WHERE ip = '::ffff:7f00:1'"),
+                row(1, 1, InetAddress.getByName("127.0.0.1")),
+                row(1, 2, InetAddress.getByName("127.0.0.1")));
+
+        // EQ ipv6 address
+        assertRows(execute("SELECT * FROM %s WHERE ip = '2002:4559:1fe2::4559:1fe2'"),
+                row(1, 5, InetAddress.getByName("2002:4559:1fe2::4559:1fe2")),
+                row(1, 6, InetAddress.getByName("2002:4559:1fe2::4559:1fe2")),
+                row(1, 7, InetAddress.getByName("2002:4559:1fe2::4559:1fe2")));
+
+        // GT ipv4 address
+        assertRows(execute("SELECT * FROM %s WHERE ip > '127.0.0.1'"),
+                row(1, 3, InetAddress.getByName("127.0.0.2")),
+                row(1, 4, InetAddress.getByName("::ffff:7f00:3")),
+                row(1, 5, InetAddress.getByName("2002:4559:1fe2::4559:1fe2")),
+                row(1, 6, InetAddress.getByName("2002:4559:1fe2::4559:1fe2")),
+                row(1, 7, InetAddress.getByName("2002:4559:1fe2::4559:1fe2")),
+                row(1, 8, InetAddress.getByName("2002:4559:1fe2::4559:1fe3")));
+
+        // GT mapped-ipv4 address
+        assertRows(execute("SELECT * FROM %s WHERE ip > '::ffff:7f00:1'"),
+                row(1, 3, InetAddress.getByName("127.0.0.2")),
+                row(1, 4, InetAddress.getByName("::ffff:7f00:3")),
+                row(1, 5, InetAddress.getByName("2002:4559:1fe2::4559:1fe2")),
+                row(1, 6, InetAddress.getByName("2002:4559:1fe2::4559:1fe2")),
+                row(1, 7, InetAddress.getByName("2002:4559:1fe2::4559:1fe2")),
+                row(1, 8, InetAddress.getByName("2002:4559:1fe2::4559:1fe3")));
+
+        // GT ipv6 address
+        assertRows(execute("SELECT * FROM %s WHERE ip > '2002:4559:1fe2::4559:1fe2'"),
+                row(1, 8, InetAddress.getByName("2002:4559:1fe2::4559:1fe3")));
+
+        // LT ipv4 address
+        assertRows(execute("SELECT * FROM %s WHERE ip < '127.0.0.3'"),
+                row(1, 1, InetAddress.getByName("127.0.0.1")),
+                row(1, 2, InetAddress.getByName("127.0.0.1")),
+                row(1, 3, InetAddress.getByName("127.0.0.2")));
+
+        // LT mapped-ipv4 address
+        assertRows(execute("SELECT * FROM %s WHERE ip < '::ffff:7f00:3'"),
+                row(1, 1, InetAddress.getByName("127.0.0.1")),
+                row(1, 2, InetAddress.getByName("127.0.0.1")),
+                row(1, 3, InetAddress.getByName("127.0.0.2")));
+
+        // LT ipv6 address
+        assertRows(execute("SELECT * FROM %s WHERE ip < '2002:4559:1fe2::4559:1fe3'"),
+                row(1, 1, InetAddress.getByName("127.0.0.1")),
+                row(1, 2, InetAddress.getByName("127.0.0.1")),
+                row(1, 3, InetAddress.getByName("127.0.0.2")),
+                row(1, 4, InetAddress.getByName("::ffff:7f00:3")),
+                row(1, 5, InetAddress.getByName("2002:4559:1fe2::4559:1fe2")),
+                row(1, 6, InetAddress.getByName("2002:4559:1fe2::4559:1fe2")),
+                row(1, 7, InetAddress.getByName("2002:4559:1fe2::4559:1fe2")));
+
+        // GE ipv4 address
+        assertRows(execute("SELECT * FROM %s WHERE ip >= '127.0.0.2'"),
+                row(1, 3, InetAddress.getByName("127.0.0.2")),
+                row(1, 4, InetAddress.getByName("::ffff:7f00:3")),
+                row(1, 5, InetAddress.getByName("2002:4559:1fe2::4559:1fe2")),
+                row(1, 6, InetAddress.getByName("2002:4559:1fe2::4559:1fe2")),
+                row(1, 7, InetAddress.getByName("2002:4559:1fe2::4559:1fe2")),
+                row(1, 8, InetAddress.getByName("2002:4559:1fe2::4559:1fe3")));
+
+        // GE mapped-ipv4 address
+        assertRows(execute("SELECT * FROM %s WHERE ip >= '::ffff:7f00:2'"),
+                row(1, 3, InetAddress.getByName("127.0.0.2")),
+                row(1, 4, InetAddress.getByName("::ffff:7f00:3")),
+                row(1, 5, InetAddress.getByName("2002:4559:1fe2::4559:1fe2")),
+                row(1, 6, InetAddress.getByName("2002:4559:1fe2::4559:1fe2")),
+                row(1, 7, InetAddress.getByName("2002:4559:1fe2::4559:1fe2")),
+                row(1, 8, InetAddress.getByName("2002:4559:1fe2::4559:1fe3")));
+
+        // GE ipv6 address
+        assertRows(execute("SELECT * FROM %s WHERE ip >= '2002:4559:1fe2::4559:1fe3'"),
+                row(1, 8, InetAddress.getByName("2002:4559:1fe2::4559:1fe3")));
+
+        // LE ipv4 address
+        assertRows(execute("SELECT * FROM %s WHERE ip <= '127.0.0.2'"),
+                row(1, 1, InetAddress.getByName("127.0.0.1")),
+                row(1, 2, InetAddress.getByName("127.0.0.1")),
+                row(1, 3, InetAddress.getByName("127.0.0.2")));
+
+        // LE mapped-ipv4 address
+        assertRows(execute("SELECT * FROM %s WHERE ip <= '::ffff:7f00:2'"),
+                row(1, 1, InetAddress.getByName("127.0.0.1")),
+                row(1, 2, InetAddress.getByName("127.0.0.1")),
+                row(1, 3, InetAddress.getByName("127.0.0.2")));
+
+        // LE ipv6 address
+        assertRows(execute("SELECT * FROM %s WHERE ip <= '2002:4559:1fe2::4559:1fe2'"),
+                row(1, 1, InetAddress.getByName("127.0.0.1")),
+                row(1, 2, InetAddress.getByName("127.0.0.1")),
+                row(1, 3, InetAddress.getByName("127.0.0.2")),
+                row(1, 4, InetAddress.getByName("::ffff:7f00:3")),
+                row(1, 5, InetAddress.getByName("2002:4559:1fe2::4559:1fe2")),
+                row(1, 6, InetAddress.getByName("2002:4559:1fe2::4559:1fe2")),
+                row(1, 7, InetAddress.getByName("2002:4559:1fe2::4559:1fe2")));
+
+        // ipv4 range
+        assertRows(execute("SELECT * FROM %s WHERE ip >= '127.0.0.1' AND ip <= '127.0.0.3'"),
+                row(1, 1, InetAddress.getByName("127.0.0.1")),
+                row(1, 2, InetAddress.getByName("127.0.0.1")),
+                row(1, 3, InetAddress.getByName("127.0.0.2")),
+                row(1, 4, InetAddress.getByName("127.0.0.3")));
+
+        // ipv4 and mapped ipv4 range
+        assertRows(execute("SELECT * FROM %s WHERE ip >= '127.0.0.1' AND ip <= '::ffff:7f00:3'"),
+                row(1, 1, InetAddress.getByName("127.0.0.1")),
+                row(1, 2, InetAddress.getByName("127.0.0.1")),
+                row(1, 3, InetAddress.getByName("127.0.0.2")),
+                row(1, 4, InetAddress.getByName("127.0.0.3")));
+
+        // ipv6 range
+        assertRows(execute("SELECT * FROM %s WHERE ip >= '2002:4559:1fe2::4559:1fe2' AND ip <= '2002:4559:1fe2::4559:1fe3'"),
+                row(1, 5, InetAddress.getByName("2002:4559:1fe2::4559:1fe2")),
+                row(1, 6, InetAddress.getByName("2002:4559:1fe2::4559:1fe2")),
+                row(1, 7, InetAddress.getByName("2002:4559:1fe2::4559:1fe2")),
+                row(1, 8, InetAddress.getByName("2002:4559:1fe2::4559:1fe3")));
+
+        // Full ipv6 range
+        assertRows(execute("SELECT * FROM %s WHERE ip >= '::' AND ip <= 'ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff'"),
+                row(1, 1, InetAddress.getByName("127.0.0.1")),
+                row(1, 2, InetAddress.getByName("127.0.0.1")),
+                row(1, 3, InetAddress.getByName("127.0.0.2")),
+                row(1, 4, InetAddress.getByName("::ffff:7f00:3")),
+                row(1, 5, InetAddress.getByName("2002:4559:1fe2::4559:1fe2")),
+                row(1, 6, InetAddress.getByName("2002:4559:1fe2::4559:1fe2")),
+                row(1, 7, InetAddress.getByName("2002:4559:1fe2::4559:1fe2")),
+                row(1, 8, InetAddress.getByName("2002:4559:1fe2::4559:1fe3")));
+    }
+}

test/unit/org/apache/cassandra/index/sai/cql/MixedIndexImplementationsTest.java

@@ -0,0 +1,190 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.cql;
+
+import org.junit.Test;
+
+import org.apache.cassandra.cql3.restrictions.StatementRestrictions;
+import org.apache.cassandra.index.sai.SAITester;
+import org.apache.cassandra.index.sai.StorageAttachedIndex;
+
+import static org.junit.Assert.assertNotNull;
+
+/**
+ * Tests behaviour when there are non-storage-attached indexes in the table.
+ */
+public class MixedIndexImplementationsTest extends SAITester
+{
+    /**
+     * Tests that storage-attached indexes can be dropped when there are other indexes in the same table, and vice versa.
+     */
+    @Test
+    public void shouldDropOtherIndex() throws Throwable
+    {
+        createTable("CREATE TABLE %s (k int PRIMARY KEY, v1 int, v2 int)");
+
+        String ossIndex = createIndex("CREATE INDEX ON %s(v1)");
+        String ndiIndex = createIndex(
+                String.format("CREATE CUSTOM INDEX ON %%s(v2) USING '%s'", StorageAttachedIndex.class.getName()));
+
+        // drop non-storage-attached index when a SAI index exists
+        dropIndex("DROP INDEX %s." + ossIndex);
+
+        // drop storage-attached index when a non-SAI exists
+        createIndex("CREATE INDEX ON %s(v1)");
+        dropIndex("DROP INDEX %s." + ndiIndex);
+    }
+
+    /**
+     * Tests that storage-attached index queries can include restrictions over columns indexed by other indexes.
+     */
+    @Test
+    public void shouldAcceptColumnsWithOtherIndex() throws Throwable
+    {
+        createTable("CREATE TABLE %s (k int PRIMARY KEY, v1 int, v2 int)");
+
+        createIndex("CREATE INDEX ON %s(v1)");
+        createIndex(String.format("CREATE CUSTOM INDEX ON %%s(v2) USING '%s'", StorageAttachedIndex.class.getName()));
+
+        String insert = "INSERT INTO %s(k, v1, v2) VALUES (?, ?, ?)";
+        execute(insert, 0, 0, 0);
+        execute(insert, 1, 0, 1);
+        execute(insert, 2, 1, 0);
+        execute(insert, 3, 1, 1);
+
+        waitForIndexQueryable();
+
+        String ossSelect = "SELECT * FROM %s WHERE v1 = ?";
+        assertRowsIgnoringOrder(execute(ossSelect, 0), new Object[][]{{0, 0, 0}, {1, 0, 1}});
+        assertRowsIgnoringOrder(execute(ossSelect, 1), new Object[][]{{2, 1, 0}, {3, 1, 1}});
+
+        String ndiSelect = "SELECT * FROM %s WHERE v1 = ? AND v2 = ? ALLOW FILTERING";
+        assertRowsIgnoringOrder(execute(ndiSelect, 0, 0), new Object[]{0, 0, 0});
+        assertRowsIgnoringOrder(execute(ndiSelect, 0, 1), new Object[]{1, 0, 1});
+        assertRowsIgnoringOrder(execute(ndiSelect, 1, 0), new Object[]{2, 1, 0});
+        assertRowsIgnoringOrder(execute(ndiSelect, 1, 1), new Object[]{3, 1, 1});
+    }
+
+    @Test
+    public void shouldRequireAllowFilteringWithOtherIndex() throws Throwable
+    {
+        createTable("CREATE TABLE %s (" +
+                    "k1 int, k2 int, " +
+                    "s1 int static, " +
+                    "c1 int, c2 int, c3 int, c4 int," +
+                    "r1 int, r2 int, r3 int, " +
+                    "PRIMARY KEY((k1, k2), c1, c2, c3, c4))");
+
+        createIndex("CREATE INDEX ON %s(k1)");
+        createIndex("CREATE INDEX ON %s(c4)");
+        createIndex("CREATE INDEX ON %s(r3)");
+        createIndex("CREATE INDEX ON %s(s1)");
+        createIndex(String.format("CREATE CUSTOM INDEX ON %%s(c2) USING '%s'", StorageAttachedIndex.class.getName()));
+        createIndex(String.format("CREATE CUSTOM INDEX ON %%s(c3) USING '%s'", StorageAttachedIndex.class.getName()));
+        createIndex(String.format("CREATE CUSTOM INDEX ON %%s(r1) USING '%s'", StorageAttachedIndex.class.getName()));
+        createIndex(String.format("CREATE CUSTOM INDEX ON %%s(r2) USING '%s'", StorageAttachedIndex.class.getName()));
+
+        // without using the not-SAI index
+        testAllowFiltering("SELECT * FROM %s", false);
+        testAllowFiltering("SELECT * FROM %s WHERE c2=0", false);
+        testAllowFiltering("SELECT * FROM %s WHERE c4=0", false);
+        testAllowFiltering("SELECT * FROM %s WHERE r1=0", false);
+        testAllowFiltering("SELECT * FROM %s WHERE r2=0", false);
+        testAllowFiltering("SELECT * FROM %s WHERE c2=0 AND c3=0", true);
+        testAllowFiltering("SELECT * FROM %s WHERE c2=0 AND r1=0", true);
+        testAllowFiltering("SELECT * FROM %s WHERE c2=0 AND r2=0", true);
+        testAllowFiltering("SELECT * FROM %s WHERE c3=0 AND r1=0", true);
+        testAllowFiltering("SELECT * FROM %s WHERE c3=0 AND r2=0", true);
+        testAllowFiltering("SELECT * FROM %s WHERE r1=0 AND r2=0", true);
+        testAllowFiltering("SELECT * FROM %s WHERE c2=0 AND c3=0 AND r1=0", true);
+        testAllowFiltering("SELECT * FROM %s WHERE c2=0 AND c3=0 AND r2=0", true);
+        testAllowFiltering("SELECT * FROM %s WHERE c2=0 AND r1=0 AND r2=0", true);
+        testAllowFiltering("SELECT * FROM %s WHERE c3=0 AND r1=0 AND r2=0", true);
+        testAllowFiltering("SELECT * FROM %s WHERE c2=0 AND c3=0 AND r1=0 AND r2=0", true);
+
+        // using the not-SAI index on partition key
+        testAllowFiltering("SELECT * FROM %s WHERE k1=0", false);
+        testAllowFiltering("SELECT * FROM %s WHERE k1=0 AND c2=0", true);
+        testAllowFiltering("SELECT * FROM %s WHERE k1=0 AND c2=0 AND c3=0", true);
+        testAllowFiltering("SELECT * FROM %s WHERE k1=0 AND c2=0 AND r1=0", true);
+        testAllowFiltering("SELECT * FROM %s WHERE k1=0 AND c2=0 AND r2=0", true);
+        testAllowFiltering("SELECT * FROM %s WHERE k1=0 AND c3=0 AND r1=0", true);
+        testAllowFiltering("SELECT * FROM %s WHERE k1=0 AND c3=0 AND r2=0", true);
+        testAllowFiltering("SELECT * FROM %s WHERE k1=0 AND r1=0 AND r2=0", true);
+        testAllowFiltering("SELECT * FROM %s WHERE k1=0 AND c2=0 AND c3=0 AND r1=0", true);
+        testAllowFiltering("SELECT * FROM %s WHERE k1=0 AND c2=0 AND c3=0 AND r2=0", true);
+        testAllowFiltering("SELECT * FROM %s WHERE k1=0 AND c2=0 AND r1=0 AND r2=0", true);
+        testAllowFiltering("SELECT * FROM %s WHERE k1=0 AND c3=0 AND r1=0 AND r2=0", true);
+        testAllowFiltering("SELECT * FROM %s WHERE k1=0 AND c2=0 AND c3=0 AND r1=0 AND r2=0", true);
+
+        // using the not-SAI index on clustering key
+        testAllowFiltering("SELECT * FROM %s WHERE c4=0", false);
+        testAllowFiltering("SELECT * FROM %s WHERE c4=0 AND c2=0", true);
+        testAllowFiltering("SELECT * FROM %s WHERE c4=0 AND c2=0 AND c3=0", true);
+        testAllowFiltering("SELECT * FROM %s WHERE c4=0 AND c2=0 AND r1=0", true);
+        testAllowFiltering("SELECT * FROM %s WHERE c4=0 AND c2=0 AND r2=0", true);
+        testAllowFiltering("SELECT * FROM %s WHERE c4=0 AND c3=0 AND r1=0", true);
+        testAllowFiltering("SELECT * FROM %s WHERE c4=0 AND c3=0 AND r2=0", true);
+        testAllowFiltering("SELECT * FROM %s WHERE c4=0 AND r1=0 AND r2=0", true);
+        testAllowFiltering("SELECT * FROM %s WHERE c4=0 AND c2=0 AND c3=0 AND r1=0", true);
+        testAllowFiltering("SELECT * FROM %s WHERE c4=0 AND c2=0 AND c3=0 AND r2=0", true);
+        testAllowFiltering("SELECT * FROM %s WHERE c4=0 AND c2=0 AND r1=0 AND r2=0", true);
+        testAllowFiltering("SELECT * FROM %s WHERE c4=0 AND c3=0 AND r1=0 AND r2=0", true);
+        testAllowFiltering("SELECT * FROM %s WHERE c4=0 AND c2=0 AND c3=0 AND r1=0 AND r2=0", true);
+
+        // using the not-SAI index on regular column
+        testAllowFiltering("SELECT * FROM %s WHERE r3=0", false);
+        testAllowFiltering("SELECT * FROM %s WHERE r3=0 AND c2=0", true);
+        testAllowFiltering("SELECT * FROM %s WHERE r3=0 AND c2=0 AND c3=0", true);
+        testAllowFiltering("SELECT * FROM %s WHERE r3=0 AND c2=0 AND r1=0", true);
+        testAllowFiltering("SELECT * FROM %s WHERE r3=0 AND c2=0 AND r2=0", true);
+        testAllowFiltering("SELECT * FROM %s WHERE r3=0 AND c3=0 AND r1=0", true);
+        testAllowFiltering("SELECT * FROM %s WHERE r3=0 AND c3=0 AND r2=0", true);
+        testAllowFiltering("SELECT * FROM %s WHERE r3=0 AND r1=0 AND r2=0", true);
+        testAllowFiltering("SELECT * FROM %s WHERE r3=0 AND c2=0 AND c3=0 AND r1=0", true);
+        testAllowFiltering("SELECT * FROM %s WHERE r3=0 AND c2=0 AND c3=0 AND r2=0", true);
+        testAllowFiltering("SELECT * FROM %s WHERE r3=0 AND c2=0 AND r1=0 AND r2=0", true);
+        testAllowFiltering("SELECT * FROM %s WHERE r3=0 AND c3=0 AND r1=0 AND r2=0", true);
+        testAllowFiltering("SELECT * FROM %s WHERE r3=0 AND c2=0 AND c3=0 AND r1=0 AND r2=0", true);
+
+        // using the not-SAI index on static column
+        testAllowFiltering("SELECT * FROM %s WHERE s1=0", false);
+        testAllowFiltering("SELECT * FROM %s WHERE s1=0 AND c2=0", true);
+        testAllowFiltering("SELECT * FROM %s WHERE s1=0 AND c2=0 AND c3=0", true);
+        testAllowFiltering("SELECT * FROM %s WHERE s1=0 AND c2=0 AND r1=0", true);
+        testAllowFiltering("SELECT * FROM %s WHERE s1=0 AND c2=0 AND r2=0", true);
+        testAllowFiltering("SELECT * FROM %s WHERE s1=0 AND c3=0 AND r1=0", true);
+        testAllowFiltering("SELECT * FROM %s WHERE s1=0 AND c3=0 AND r2=0", true);
+        testAllowFiltering("SELECT * FROM %s WHERE s1=0 AND r1=0 AND r2=0", true);
+        testAllowFiltering("SELECT * FROM %s WHERE s1=0 AND c2=0 AND c3=0 AND r1=0", true);
+        testAllowFiltering("SELECT * FROM %s WHERE s1=0 AND c2=0 AND c3=0 AND r2=0", true);
+        testAllowFiltering("SELECT * FROM %s WHERE s1=0 AND c2=0 AND r1=0 AND r2=0", true);
+        testAllowFiltering("SELECT * FROM %s WHERE s1=0 AND c3=0 AND r1=0 AND r2=0", true);
+        testAllowFiltering("SELECT * FROM %s WHERE s1=0 AND c2=0 AND c3=0 AND r1=0 AND r2=0", true);
+    }
+
+    private void testAllowFiltering(String query, boolean requiresAllowFiltering) throws Throwable
+    {
+        if (requiresAllowFiltering)
+            assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE, query);
+        else
+            assertNotNull(execute(query));
+
+        assertNotNull(execute(query + " ALLOW FILTERING"));
+    }
+}

test/unit/org/apache/cassandra/index/sai/cql/MultipleColumnIndexTest.java

@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.cql;
+
+import org.junit.Test;
+
+import org.apache.cassandra.index.sai.SAITester;
+
+import static org.junit.Assert.assertEquals;
+
+public class MultipleColumnIndexTest extends SAITester
+{
+    // Note: Full testing of multiple map index types is done in the
+    // types/collections/maps/MultiMap*Test tests
+    // This is just testing that the indexes can be created
+    @Test
+    public void canCreateMultipleMapIndexesOnSameColumn()
+    {
+        createTable("CREATE TABLE %s (pk int, ck int, value map<int,int>, PRIMARY KEY(pk, ck))");
+        createIndex("CREATE CUSTOM INDEX ON %s(KEYS(value)) USING 'StorageAttachedIndex'");
+        createIndex("CREATE CUSTOM INDEX ON %s(VALUES(value)) USING 'StorageAttachedIndex'");
+        createIndex("CREATE CUSTOM INDEX ON %s(ENTRIES(value)) USING 'StorageAttachedIndex'");
+    }
+
+    @Test
+    public void indexNamedAsColumnWillCoExistWithGeneratedIndexNames() throws Throwable
+    {
+        createTable("CREATE TABLE %s(id int PRIMARY KEY, text_map map<text, text>)");
+
+        createIndex("CREATE CUSTOM INDEX text_map ON %s(keys(text_map)) USING 'StorageAttachedIndex'");
+        createIndex("CREATE CUSTOM INDEX ON %s(values(text_map)) USING 'StorageAttachedIndex'");
+        createIndex("CREATE CUSTOM INDEX ON %s(entries(text_map)) USING 'StorageAttachedIndex'");
+
+        waitForIndexQueryable();
+
+        execute("INSERT INTO %s(id, text_map) values (1, {'k1':'v1', 'k2':'v2'})");
+        execute("INSERT INTO %s(id, text_map) values (2, {'k1':'v1', 'k3':'v3'})");
+        execute("INSERT INTO %s(id, text_map) values (3, {'k4':'v4', 'k5':'v5'})");
+
+        assertEquals(1, execute("SELECT * FROM %s WHERE text_map['k1'] = 'v1' AND text_map['k2'] = 'v2' ALLOW FILTERING").size());
+        assertEquals(2, execute("SELECT * FROM %s WHERE text_map CONTAINS 'v1'").size());
+        assertEquals(2, execute("SELECT * FROM %s WHERE text_map CONTAINS KEY 'k1'").size());
+        assertEquals(1, execute("SELECT * FROM %s WHERE text_map CONTAINS KEY 'k1' AND text_map CONTAINS KEY 'k2' ALLOW FILTERING").size());
+        assertEquals(2, execute("SELECT * FROM %s WHERE text_map['k1'] = 'v1' AND text_map CONTAINS KEY 'k1' AND text_map CONTAINS 'v1' ALLOW FILTERING").size());
+        assertEquals(1, execute("SELECT * FROM %s WHERE text_map['k1'] = 'v1' AND text_map CONTAINS KEY 'k1' AND text_map CONTAINS KEY 'k2' AND text_map CONTAINS 'v1' ALLOW FILTERING").size());
+        assertEquals(0, execute("SELECT * FROM %s WHERE text_map['k1'] = 'v1' AND text_map CONTAINS KEY 'k1' AND text_map CONTAINS KEY 'k4' ALLOW FILTERING").size());
+    }
+}

test/unit/org/apache/cassandra/index/sai/cql/QueryTimeToLiveTest.java

@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.cql;
+
+import org.junit.Test;
+
+public class QueryTimeToLiveTest extends AbstractQueryTester
+{
+    @Test
+    public void testTimeToLive() throws Throwable
+    {
+        IndexQuerySupport.timeToLive(executor, dataModel, sets);
+    }
+}

test/unit/org/apache/cassandra/index/sai/cql/QueryWriteLifecycleTest.java

@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.cql;
+
+import org.junit.Test;
+
+public class QueryWriteLifecycleTest extends AbstractQueryTester
+{
+    @Test
+    public void testWriteLifecycle() throws Throwable
+    {
+        IndexQuerySupport.writeLifecycle(executor, dataModel, sets);
+    }
+}

test/unit/org/apache/cassandra/index/sai/cql/SingleNodeExecutor.java

@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.cql;
+
+import java.util.List;
+import java.util.stream.Collectors;
+
+import com.datastax.driver.core.SimpleStatement;
+import org.apache.cassandra.index.sai.SAITester;
+import org.apache.cassandra.inject.Injections;
+
+public class SingleNodeExecutor implements BaseDataModel.Executor
+{
+    private final SAITester tester;
+    private final Injections.Counter counter;
+
+    public SingleNodeExecutor(SAITester tester, Injections.Counter counter)
+    {
+        this.tester = tester;
+        this.counter = counter;
+    }
+
+    @Override
+    public void createTable(String statement)
+    {
+        tester.createTable(statement);
+    }
+
+    @Override
+    public void waitForIndexQueryable(String keyspace, String table)
+    {
+        tester.waitForIndexQueryable(keyspace, table);
+    }
+
+    @Override
+    public void executeLocal(String query, Object... values) throws Throwable
+    {
+        tester.execute(query, values);
+    }
+
+    @Override
+    public List<Object> executeRemote(String query, int fetchSize, Object... values)
+    {
+        SimpleStatement statement = new SimpleStatement(query, values);
+        statement.setFetchSize(fetchSize);
+        return tester.sessionNet().execute(statement).all().stream().map(r -> r.getObject(0)).collect(Collectors.toList());
+    }
+
+    @Override
+    public void counterReset()
+    {
+        counter.reset();
+    }
+
+    @Override
+    public long getCounter()
+    {
+        return counter.get();
+    }
+}

test/unit/org/apache/cassandra/index/sai/cql/StorageAttachedIndexDDLTest.java

@@ -0,0 +1,320 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+package org.apache.cassandra.index.sai.cql;
+
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.datastax.driver.core.ResultSet;
+import com.datastax.driver.core.exceptions.InvalidConfigurationInQueryException;
+import com.datastax.driver.core.exceptions.InvalidQueryException;
+import org.apache.cassandra.cql3.CQL3Type;
+import org.apache.cassandra.cql3.ColumnIdentifier;
+import org.apache.cassandra.cql3.restrictions.IndexRestrictions;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.ReversedType;
+import org.apache.cassandra.index.SecondaryIndexManager;
+import org.apache.cassandra.index.sai.IndexContext;
+import org.apache.cassandra.index.sai.SAITester;
+import org.apache.cassandra.index.sai.StorageAttachedIndex;
+import org.apache.cassandra.inject.ActionBuilder;
+import org.apache.cassandra.inject.Expression;
+import org.apache.cassandra.inject.Injection;
+import org.apache.cassandra.inject.Injections;
+import org.apache.cassandra.inject.InvokePointBuilder;
+import org.apache.cassandra.schema.SchemaConstants;
+import org.apache.cassandra.schema.TableMetadata;
+
+import static org.apache.cassandra.cql3.statements.schema.CreateIndexStatement.INVALID_CUSTOM_INDEX_TARGET;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+public class StorageAttachedIndexDDLTest extends SAITester
+{
+    private static final Injections.Counter saiCreationCounter = Injections.newCounter("IndexCreationCounter")
+                                                                           .add(InvokePointBuilder.newInvokePoint().onClass(StorageAttachedIndex.class).onMethod("register"))
+                                                                           .build();
+
+    private static final Injection FAIL_INDEX_GC_TRANSACTION = Injections.newCustom("fail_index_gc_transaction")
+                                                                         .add(InvokePointBuilder.newInvokePoint().onClass("org.apache.cassandra.index.SecondaryIndexManager$IndexGCTransaction")
+                                                                                                .onMethod("<init>"))
+                                                                         .add(ActionBuilder.newActionBuilder().actions().doThrow(RuntimeException.class, Expression.quote("Injected failure!")))
+                                                                         .build();
+
+    @Before
+    public void setup() throws Throwable
+    {
+        requireNetwork();
+
+        startJMXServer();
+
+        createMBeanServerConnection();
+
+        Injections.inject(saiCreationCounter, FAIL_INDEX_GC_TRANSACTION);
+
+        saiCreationCounter.reset();
+    }
+
+    @After
+    public void removeInjections()
+    {
+        Injections.deleteAll();
+    }
+
+    @Test
+    public void shouldFailUnsupportedType() throws Throwable
+    {
+        for (CQL3Type.Native cql3Type : CQL3Type.Native.values())
+        {
+            if (cql3Type == CQL3Type.Native.EMPTY)
+                continue;
+
+            String createTableTemplate = "CREATE TABLE %%s (id text PRIMARY KEY, %s %s)";
+            createTable(String.format(createTableTemplate, cql3Type, cql3Type));
+
+            if (StorageAttachedIndex.SUPPORTED_TYPES.contains(cql3Type))
+            {
+                try
+                {
+                    executeNet(String.format("CREATE CUSTOM INDEX ON %%s(%s) USING 'StorageAttachedIndex'", cql3Type));
+                }
+                catch (RuntimeException e)
+                {
+                    fail("Index creation on supported type " + cql3Type + " should have succeeded.");
+                }
+            }
+            else
+                assertThatThrownBy(() -> executeNet(String.format("CREATE CUSTOM INDEX ON %%s(%s) USING 'StorageAttachedIndex'", cql3Type)))
+                        .isInstanceOf(InvalidQueryException.class);
+        }
+    }
+
+    @Test
+    public void shouldFailCreationOnPartitionKey()
+    {
+        createTable("CREATE TABLE %s (id text PRIMARY KEY, val text)");
+        assertThatThrownBy(() -> executeNet("CREATE CUSTOM INDEX ON %s(id) USING 'StorageAttachedIndex'"))
+                .isInstanceOf(InvalidQueryException.class)
+                .hasMessageContaining("Cannot create secondary index on the only partition key column id");
+    }
+
+    @Test
+    public void shouldFailCreationUsingMode()
+    {
+        createTable("CREATE TABLE %s (id text PRIMARY KEY, val text)");
+
+        assertThatThrownBy(() -> executeNet("CREATE CUSTOM INDEX ON %s(val) USING " +
+                                            "'StorageAttachedIndex' WITH OPTIONS = { 'mode' : 'CONTAINS' }")).isInstanceOf(InvalidConfigurationInQueryException.class);
+    }
+
+    @Test
+    public void shouldFailCreateWithUserType()
+    {
+        String typeName = createType("CREATE TYPE %s (a text, b int, c double)");
+        createTable("CREATE TABLE %s (id text PRIMARY KEY, val " + typeName + ")");
+
+        assertThatThrownBy(() -> executeNet("CREATE CUSTOM INDEX ON %s(val) " +
+                                            "USING 'StorageAttachedIndex'")).isInstanceOf(InvalidQueryException.class);
+    }
+
+    @Test
+    public void shouldNotFailCreateWithTupleType() throws Throwable
+    {
+        createTable("CREATE TABLE %s (id text PRIMARY KEY, val tuple<text, int, double>)");
+
+        executeNet("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex'");
+
+        TableMetadata metadata = currentTableMetadata();
+        AbstractType<?> tuple = metadata.getColumn(ColumnIdentifier.getInterned("val", false)).type;
+        assertFalse(tuple.isMultiCell());
+        assertFalse(tuple.isCollection());
+        assertTrue(tuple.isTuple());
+    }
+
+    @Test
+    public void shouldFailCreateWithInvalidCharactersInColumnName()
+    {
+        String invalidColumn = "/invalid";
+        createTable(String.format("CREATE TABLE %%s (id text PRIMARY KEY, \"%s\" text)", invalidColumn));
+
+        assertThatThrownBy(() -> executeNet(String.format("CREATE CUSTOM INDEX ON %%s(\"%s\")" +
+                                                          " USING 'StorageAttachedIndex'", invalidColumn)))
+                .isInstanceOf(InvalidQueryException.class)
+                .hasMessage(String.format(INVALID_CUSTOM_INDEX_TARGET, invalidColumn, SchemaConstants.NAME_LENGTH));
+    }
+
+    @Test
+    public void shouldCreateIndexIfExists()
+    {
+        createTable("CREATE TABLE %s (id text PRIMARY KEY, val text)");
+
+        createIndex("CREATE CUSTOM INDEX IF NOT EXISTS ON %s(val) USING 'StorageAttachedIndex' ");
+
+        createIndex("CREATE CUSTOM INDEX IF NOT EXISTS ON %s(val) USING 'StorageAttachedIndex' ");
+
+        assertEquals(1, saiCreationCounter.get());
+    }
+
+    @Test
+    public void shouldBeCaseSensitiveByDefault() throws Throwable
+    {
+        createTable("CREATE TABLE %s (id text PRIMARY KEY, val text)");
+
+        createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex'");
+        waitForIndexQueryable();
+
+        execute("INSERT INTO %s (id, val) VALUES ('1', 'Camel')");
+
+        assertEquals(1, execute("SELECT id FROM %s WHERE val = 'Camel'").size());
+
+        assertEquals(0, execute("SELECT id FROM %s WHERE val = 'camel'").size());
+    }
+
+    @Test
+    public void shouldBeNonNormalizedByDefault() throws Throwable
+    {
+        createTable("CREATE TABLE %s (id text PRIMARY KEY, val text)");
+
+        createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex'");
+        waitForIndexQueryable();
+
+        execute("INSERT INTO %s (id, val) VALUES ('1', 'Cam\u00E1l')");
+
+        assertEquals(1, execute("SELECT id FROM %s WHERE val = 'Cam\u00E1l'").size());
+
+        // Both \u00E1 and \u0061\u0301 are visible as the character á, but without NFC normalization, they won't match.
+        assertEquals(0, execute("SELECT id FROM %s WHERE val = 'Cam\u0061\u0301l'").size());
+    }
+
+    @Test
+    public void shouldCreateIndexOnReversedType() throws Throwable
+    {
+        createTable("CREATE TABLE %s (id text, ck1 text, ck2 int, val text, PRIMARY KEY (id,ck1,ck2)) WITH CLUSTERING ORDER BY (ck1 desc, ck2 desc)");
+
+        String indexNameCk1 = createIndex("CREATE CUSTOM INDEX ON %s(ck1) USING 'StorageAttachedIndex'");
+        String indexNameCk2 = createIndex("CREATE CUSTOM INDEX ON %s(ck2) USING 'StorageAttachedIndex'");
+
+        execute("insert into %s(id, ck1, ck2, val) values('1', '2', 3, '3')");
+        execute("insert into %s(id, ck1, ck2, val) values('1', '3', 4, '4')");
+        assertEquals(1, executeNet("SELECT * FROM %s WHERE ck1='3'").all().size());
+        assertEquals(2, executeNet("SELECT * FROM %s WHERE ck2>=0").all().size());
+        assertEquals(2, executeNet("SELECT * FROM %s WHERE ck2<=4").all().size());
+
+        SecondaryIndexManager sim = getCurrentColumnFamilyStore().indexManager;
+        StorageAttachedIndex index = (StorageAttachedIndex) sim.getIndexByName(indexNameCk1);
+        IndexContext context = index.getIndexContext();
+        assertTrue(context.isLiteral());
+        assertTrue(context.getValidator() instanceof ReversedType);
+
+        index = (StorageAttachedIndex) sim.getIndexByName(indexNameCk2);
+        context = index.getIndexContext();
+        assertFalse(context.isLiteral());
+        assertTrue(context.getValidator() instanceof ReversedType);
+    }
+
+    @Test
+    public void shouldCreateIndexWithAlias()
+    {
+        createTable("CREATE TABLE %s (id text PRIMARY KEY, val text)");
+
+        createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex'");
+
+        assertEquals(1, saiCreationCounter.get());
+    }
+
+    /**
+     * Verify SASI can be created and queries with SAI dependencies.
+     * Not putting in {@link MixedIndexImplementationsTest} because it uses CQLTester which doesn't load SAI dependency.
+     */
+    @Test
+    public void shouldCreateSASI() throws Throwable
+    {
+        createTable(CREATE_TABLE_TEMPLATE);
+
+        createIndex("CREATE CUSTOM INDEX ON %s(v1) USING 'org.apache.cassandra.index.sasi.SASIIndex'");
+        createIndex("CREATE CUSTOM INDEX ON %s(v2) USING 'org.apache.cassandra.index.sasi.SASIIndex' WITH OPTIONS = {'mode': 'CONTAINS',\n" +
+                    "'analyzer_class': 'org.apache.cassandra.index.sasi.analyzer.StandardAnalyzer',\n" +
+                    "'tokenization_enable_stemming': 'true',\n" +
+                    "'tokenization_locale': 'en',\n" +
+                    "'tokenization_skip_stop_words': 'true',\n" +
+                    "'analyzed': 'true',\n" +
+                    "'tokenization_normalize_lowercase': 'true'};");
+
+        execute("INSERT INTO %s (id1, v1, v2) VALUES ('1', 1, '0');");
+
+        ResultSet rows = executeNet("SELECT id1 FROM %s WHERE v1>=0");
+        assertEquals(1, rows.all().size());
+
+        rows = executeNet("SELECT id1 FROM %s WHERE v2 like '0'");
+        assertEquals(1, rows.all().size());
+    }
+
+    @Test
+    public void shouldFailCreationOnMultipleColumns()
+    {
+        createTable("CREATE TABLE %s (id text PRIMARY KEY, val1 text, val2 text)");
+
+        assertThatThrownBy(() -> executeNet("CREATE CUSTOM INDEX ON %s(val1, val2) USING 'StorageAttachedIndex'"))
+                .isInstanceOf(InvalidQueryException.class)
+                .hasMessageContaining("storage-attached index cannot be created over multiple columns");
+    }
+
+    @Test
+    public void shouldFailCreationMultipleIndexesOnSimpleColumn() throws Throwable
+    {
+        createTable("CREATE TABLE %s (id int PRIMARY KEY, v1 TEXT)");
+
+        executeNet("CREATE CUSTOM INDEX index_1 ON %s(v1) USING 'StorageAttachedIndex'");
+        waitForIndexQueryable();
+
+        // same name
+        assertThatThrownBy(() -> executeNet("CREATE CUSTOM INDEX index_1 ON %s(v1) USING 'StorageAttachedIndex'"))
+                .isInstanceOf(InvalidQueryException.class)
+                .hasMessageContaining("Index 'index_1' already exists");
+
+        // different name, same option
+        assertThatThrownBy(() -> executeNet("CREATE CUSTOM INDEX index_2 ON %s(v1) USING 'StorageAttachedIndex'"))
+                .isInstanceOf(InvalidQueryException.class)
+                .hasMessageContaining("Index index_2 is a duplicate of existing index index_1");
+
+        execute("INSERT INTO %s (id, v1) VALUES(1, '1')");
+
+        ResultSet rows = executeNet("SELECT id FROM %s WHERE v1 = '1'");
+        assertEquals(1, rows.all().size());
+    }
+
+    @Test
+    public void shouldRejectQueriesWithCustomExpressions()
+    {
+        createTable(CREATE_TABLE_TEMPLATE);
+
+        String index = createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1"));
+
+        assertThatThrownBy(() -> executeNet(String.format("SELECT * FROM %%s WHERE expr(%s, 0)", index)))
+                .isInstanceOf(InvalidQueryException.class)
+                .hasMessage(String.format(IndexRestrictions.CUSTOM_EXPRESSION_NOT_SUPPORTED, index));
+    }
+}

test/unit/org/apache/cassandra/index/sai/cql/types/AsciiTest.java

@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.cql.types;
+
+import java.util.Collection;
+
+import org.junit.runners.Parameterized;
+
+public class AsciiTest extends IndexingTypeSupport
+{
+    @Parameterized.Parameters(name = "dataset={0},wide={1},scenario={2}")
+    public static Collection<Object[]> generateParameters()
+    {
+        return generateParameters(new DataSet.AsciiDataSet());
+    }
+}