Add native transport deadline, an ultimate deadline for all tasks related to a specific request

* Add an ability to base _replica_ side queries on the queue time
      * Use queue time as a base for message timeouts
      * Use native transport deadline for internode messages
      * Make sure that local runnables respect transport timeouts and deadlines
      * Make sure that remote mutation handler respects message expiration times

Patch by Alex Petrov; reviewed by Caleb Rackliffe for CASSANDRA-19534

CHANGES.txt

@@ -5404,7 +5404,7 @@ Merged from 1.0:
  * avoid recursion in leveled compaction (CASSANDRA-4587)
  * increase stack size under Java7 to 180K
  * Log(info) schema changes (CASSANDRA-4547)
- * Change nodetool setcachecapcity to manipulate global caches (CASSANDRA-4563)
+ * Change nodetool setcachecapacity to manipulate global caches (CASSANDRA-4563)
  * (cql3) fix setting compaction strategy (CASSANDRA-4597)
  * fix broken system.schema_* timestamps on system startup (CASSANDRA-4561)
  * fix wrong skip of cache saving (CASSANDRA-4533)

src/java/org/apache/cassandra/auth/CassandraAuthorizer.java

@@ -46,12 +46,12 @@ import org.apache.cassandra.schema.SchemaConstants;
 import org.apache.cassandra.cql3.statements.SelectStatement;
 import org.apache.cassandra.service.ClientState;
 import org.apache.cassandra.service.QueryState;
+import org.apache.cassandra.service.reads.range.RangeCommands;
+import org.apache.cassandra.transport.Dispatcher;
 import org.apache.cassandra.transport.messages.ResultMessage;
 import org.apache.cassandra.utils.ByteBufferUtil;
 import org.apache.cassandra.utils.Pair;
 
-import static org.apache.cassandra.utils.Clock.Global.nanoTime;
-
 /**
  * CassandraAuthorizer is an IAuthorizer implementation that keeps
  * user permissions internally in C* using the system_auth.role_permissions
@@ -421,7 +421,7 @@ public class CassandraAuthorizer implements IAuthorizer
 
     ResultMessage.Rows select(SelectStatement statement, QueryOptions options)
     {
-        return statement.execute(QueryState.forInternalCalls(), options, nanoTime());
+        return statement.execute(QueryState.forInternalCalls(), options, Dispatcher.RequestTime.forImmediateExecution());
     }
 
     /**
@@ -439,7 +439,7 @@ public class CassandraAuthorizer implements IAuthorizer
         QueryProcessor.instance.processBatch(statement,
                                              QueryState.forInternalCalls(),
                                              BatchQueryOptions.withoutPerStatementVariables(options),
-                                             nanoTime());
+                                             Dispatcher.RequestTime.forImmediateExecution());
     }
 
     public static ConsistencyLevel authWriteConsistencyLevel()

src/java/org/apache/cassandra/auth/CassandraNetworkAuthorizer.java

@@ -39,11 +39,12 @@ import org.apache.cassandra.db.marshal.UTF8Type;
 import org.apache.cassandra.exceptions.ConfigurationException;
 import org.apache.cassandra.schema.SchemaConstants;
 import org.apache.cassandra.service.ClientState;
+import org.apache.cassandra.service.reads.range.RangeCommands;
+import org.apache.cassandra.transport.Dispatcher;
 import org.apache.cassandra.transport.messages.ResultMessage;
 import org.apache.cassandra.utils.ByteBufferUtil;
 
 import static org.apache.cassandra.service.QueryState.forInternalCalls;
-import static org.apache.cassandra.utils.Clock.Global.nanoTime;
 
 public class CassandraNetworkAuthorizer implements INetworkAuthorizer
 {
@@ -61,7 +62,7 @@ public class CassandraNetworkAuthorizer implements INetworkAuthorizer
     @VisibleForTesting
     ResultMessage.Rows select(SelectStatement statement, QueryOptions options)
     {
-        return statement.execute(forInternalCalls(), options, nanoTime());
+        return statement.execute(forInternalCalls(), options, Dispatcher.RequestTime.forImmediateExecution());
     }
 
     /**

src/java/org/apache/cassandra/auth/CassandraRoleManager.java

@@ -44,12 +44,13 @@ import org.apache.cassandra.db.marshal.UTF8Type;
 import org.apache.cassandra.exceptions.*;
 import org.apache.cassandra.service.ClientState;
 import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.service.reads.range.RangeCommands;
+import org.apache.cassandra.transport.Dispatcher;
 import org.apache.cassandra.transport.messages.ResultMessage;
 import org.apache.cassandra.utils.ByteBufferUtil;
 import org.mindrot.jbcrypt.BCrypt;
 
 import static org.apache.cassandra.service.QueryState.forInternalCalls;
-import static org.apache.cassandra.utils.Clock.Global.nanoTime;
 
 /**
  * Responsible for the creation, maintenance and deletion of roles
@@ -561,7 +562,7 @@ public class CassandraRoleManager implements IRoleManager
     @VisibleForTesting
     ResultMessage.Rows select(SelectStatement statement, QueryOptions options)
     {
-        return statement.execute(forInternalCalls(), options, nanoTime());
+        return statement.execute(forInternalCalls(), options, Dispatcher.RequestTime.forImmediateExecution());
     }
 
     @Override

src/java/org/apache/cassandra/auth/PasswordAuthenticator.java

@@ -43,12 +43,13 @@ import org.apache.cassandra.exceptions.AuthenticationException;
 import org.apache.cassandra.exceptions.ConfigurationException;
 import org.apache.cassandra.service.ClientState;
 import org.apache.cassandra.service.QueryState;
+import org.apache.cassandra.service.reads.range.RangeCommands;
+import org.apache.cassandra.transport.Dispatcher;
 import org.apache.cassandra.transport.messages.ResultMessage;
 import org.apache.cassandra.utils.ByteBufferUtil;
 import org.mindrot.jbcrypt.BCrypt;
 
 import static org.apache.cassandra.auth.CassandraRoleManager.consistencyForRoleRead;
-import static org.apache.cassandra.utils.Clock.Global.nanoTime;
 
 /**
  * PasswordAuthenticator is an IAuthenticator implementation
@@ -198,7 +199,7 @@ public class PasswordAuthenticator implements IAuthenticator, AuthCache.BulkLoad
     @VisibleForTesting
     ResultMessage.Rows select(SelectStatement statement, QueryOptions options)
     {
-        return statement.execute(QueryState.forInternalCalls(), options, nanoTime());
+        return statement.execute(QueryState.forInternalCalls(), options, Dispatcher.RequestTime.forImmediateExecution());
     }
 
     public Set<DataResource> protectedResources()

src/java/org/apache/cassandra/batchlog/BatchlogManager.java

@@ -36,6 +36,7 @@ import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.Iterables;
 import com.google.common.util.concurrent.RateLimiter;
 import org.apache.cassandra.concurrent.ScheduledExecutorPlus;
+import org.apache.cassandra.transport.Dispatcher;
 import org.apache.cassandra.utils.TimeUUID;
 import org.apache.cassandra.utils.concurrent.Future;
 import org.slf4j.Logger;
@@ -81,7 +82,6 @@ import static org.apache.cassandra.cql3.QueryProcessor.executeInternal;
 import static org.apache.cassandra.cql3.QueryProcessor.executeInternalWithPaging;
 import static org.apache.cassandra.net.Verb.MUTATION_REQ;
 import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis;
-import static org.apache.cassandra.utils.Clock.Global.nanoTime;
 
 public class BatchlogManager implements BatchlogManagerMBean
 {
@@ -502,7 +502,7 @@ public class BatchlogManager implements BatchlogManagerMBean
 
             ReplicaPlan.ForWrite replicaPlan = new ReplicaPlan.ForWrite(keyspace, liveAndDown.replicationStrategy(),
                                                                         ConsistencyLevel.ONE, liveRemoteOnly.pending(), liveRemoteOnly.all(), liveRemoteOnly.all(), liveRemoteOnly.all());
-            ReplayWriteResponseHandler<Mutation> handler = new ReplayWriteResponseHandler<>(replicaPlan, mutation, nanoTime());
+            ReplayWriteResponseHandler<Mutation> handler = new ReplayWriteResponseHandler<>(replicaPlan, mutation, Dispatcher.RequestTime.forImmediateExecution());
             Message<Mutation> message = Message.outWithFlag(MUTATION_REQ, mutation, MessageFlag.CALL_BACK_ON_FAILURE);
             for (Replica replica : liveRemoteOnly.all())
                 MessagingService.instance().sendWriteWithCallback(message, replica, handler);
@@ -526,9 +526,9 @@ public class BatchlogManager implements BatchlogManagerMBean
             private final Set<InetAddressAndPort> undelivered = Collections.newSetFromMap(new ConcurrentHashMap<>());
 
             // TODO: should we be hinting here, since presumably batch log will retry? Maintaining historical behaviour for the moment.
-            ReplayWriteResponseHandler(ReplicaPlan.ForWrite replicaPlan, Supplier<Mutation> hintOnFailure, long queryStartNanoTime)
+            ReplayWriteResponseHandler(ReplicaPlan.ForWrite replicaPlan, Supplier<Mutation> hintOnFailure, Dispatcher.RequestTime requestTime)
             {
-                super(replicaPlan, null, WriteType.UNLOGGED_BATCH, hintOnFailure, queryStartNanoTime);
+                super(replicaPlan, null, WriteType.UNLOGGED_BATCH, hintOnFailure, requestTime);
                 Iterables.addAll(undelivered, replicaPlan.contacts().endpoints());
             }
 

src/java/org/apache/cassandra/concurrent/DebuggableTask.java

@@ -17,6 +17,9 @@
  */
 package org.apache.cassandra.concurrent;
 
+import java.util.concurrent.Callable;
+
+import org.apache.cassandra.utils.MonotonicClock;
 import org.apache.cassandra.utils.Shared;
 
 import static org.apache.cassandra.utils.Shared.Recursive.INTERFACES;
@@ -29,6 +32,11 @@ import static org.apache.cassandra.utils.Shared.Scope.SIMULATION;
 @Shared(scope = SIMULATION, inner = INTERFACES)
 public interface DebuggableTask
 {
+    public default long elapsedSinceCreation()
+    {
+        return MonotonicClock.Global.approxTime.now() - creationTimeNanos();
+    }
+
     public long creationTimeNanos();
 
     public long startTimeNanos();
@@ -36,6 +44,7 @@ public interface DebuggableTask
     public String description();
     
     interface RunnableDebuggableTask extends Runnable, DebuggableTask {}
+    interface CallableDebuggableTask<T> extends Callable<T>, DebuggableTask {}
 
     /**
      * Wraps a {@link DebuggableTask} to include the name of the thread running it.

src/java/org/apache/cassandra/concurrent/FutureTask.java

@@ -89,6 +89,34 @@ public class FutureTask<V> extends AsyncFuture<V> implements RunnableFuture<V>
 
     public static <T> Callable<T> callable(Runnable run)
     {
+        if (run instanceof DebuggableTask.RunnableDebuggableTask)
+        {
+            return new DebuggableTask.CallableDebuggableTask<T>()
+            {
+                final RunnableDebuggableTask task = (RunnableDebuggableTask) run;
+                public T call()
+                {
+                    run.run();
+                    return null;
+                }
+
+                public long creationTimeNanos()
+                {
+                    return task.creationTimeNanos();
+                }
+
+                public long startTimeNanos()
+                {
+                    return task.startTimeNanos();
+                }
+
+                public String description()
+                {
+                    return task.description();
+                }
+            };
+        }
+
         return new Callable<T>()
         {
             public T call()

src/java/org/apache/cassandra/concurrent/ResizableThreadPool.java

@@ -79,4 +79,9 @@ public interface ResizableThreadPool
     {
         return -1;
     }
+
+    default long oldestTaskQueueTime()
+    {
+        return 0;
+    }
 }

src/java/org/apache/cassandra/concurrent/SEPExecutor.java

@@ -84,6 +84,21 @@ public class SEPExecutor implements LocalAwareExecutorPlus, SEPExecutorMBean
         completedTasks.incrementAndGet();
     }
 
+    @Override
+    public long oldestTaskQueueTime()
+    {
+        Runnable task = tasks.peek();
+        if (!(task instanceof FutureTask))
+            return 0L;
+
+        FutureTask<?> futureTask = (FutureTask<?>) task;
+        DebuggableTask debuggableTask = futureTask.debuggableTask();
+        if (debuggableTask == null)
+            return 0L;
+
+        return debuggableTask.elapsedSinceCreation();
+    }
+
     @Override
     public int getMaxTasksQueued()
     {

src/java/org/apache/cassandra/config/Config.java

@@ -1173,4 +1173,20 @@ public class Config
 
         logger.info("Node configuration:[{}]", Joiner.on("; ").join(configMap.entrySet()));
     }
+
+    public enum CQLStartTime
+    {
+        REQUEST, // uses a timestamp that represent the start of processing of the request
+        QUEUE    // uses a timestamp that represents when the request was enqueued
+    }
+    public volatile CQLStartTime cql_start_time = CQLStartTime.REQUEST;
+
+    public boolean native_transport_throw_on_overload = false;
+    public double native_transport_queue_max_item_age_threshold = Double.MAX_VALUE;
+    public DurationSpec.LongMillisecondsBound native_transport_min_backoff_on_queue_overload = new DurationSpec.LongMillisecondsBound("10ms");
+    public DurationSpec.LongMillisecondsBound native_transport_max_backoff_on_queue_overload = new DurationSpec.LongMillisecondsBound("200ms");
+
+    // 3.0 Cassandra Driver has its "read" timeout set to 12 seconds. Our recommendation is match this.
+    public DurationSpec.LongMillisecondsBound native_transport_timeout = new DurationSpec.LongMillisecondsBound("12000ms");
+    public boolean enforce_native_deadline_for_hints = false;
 }

src/java/org/apache/cassandra/config/DatabaseDescriptor.java

@@ -906,6 +906,15 @@ public class DatabaseDescriptor
             conf.paxos_state_purging = PaxosStatePurging.legacy;
 
         logInitializationOutcome(logger);
+
+        if (conf.native_transport_min_backoff_on_queue_overload.toMilliseconds() <= 0)
+            throw new IllegalArgumentException("native_transport_min_backoff_on_queue_overload should be positive");
+
+        if (conf.native_transport_min_backoff_on_queue_overload.toMilliseconds() >= conf.native_transport_max_backoff_on_queue_overload.toMilliseconds())
+            throw new IllegalArgumentException(String.format("native_transport_min_backoff_on_queue_overload should be strictly less than native_transport_max_backoff_on_queue_overload, but %s >= %s",
+                                                             conf.native_transport_min_backoff_on_queue_overload,
+                                                             conf.native_transport_max_backoff_on_queue_overload));
+
     }
 
     @VisibleForTesting
@@ -1889,6 +1898,81 @@ public class DatabaseDescriptor
                          getTruncateRpcTimeout(unit));
     }
 
+    /**
+     * How much time the item is allowed to spend in (currently only Native) queue, compared to {@link #nativeTransportIdleTimeout()},
+     * before backpressure starts being applied.
+     *
+     * For example, setting this value to 0.5 means and having the largest of read/range/write/counter timeouts to 10 seconds
+     * means that if any item spends more than 5 seconds in the queue, backpressure will be applied to the socket associated
+     * with this queue.
+     *
+     * Set to 0 or any negative value to fully disable.
+     */
+    public static double getNativeTransportQueueMaxItemAgeThreshold()
+    {
+        return conf.native_transport_queue_max_item_age_threshold;
+    }
+
+    public static void setNativeTransportMaxQueueItemAgeThreshold(double threshold)
+    {
+        conf.native_transport_queue_max_item_age_threshold = threshold;
+    }
+
+    public static long getNativeTransportMinBackoffOnQueueOverload(TimeUnit timeUnit)
+    {
+        return conf.native_transport_min_backoff_on_queue_overload.to(timeUnit);
+    }
+
+    public static long getNativeTransportMaxBackoffOnQueueOverload(TimeUnit timeUnit)
+    {
+        return conf.native_transport_max_backoff_on_queue_overload.to(timeUnit);
+    }
+
+    public static void setNativeTransportBackoffOnQueueOverload(long minBackoffMillis,
+                                                                long maxBackoffMillis,
+                                                                TimeUnit timeUnit)
+    {
+        if (minBackoffMillis <= 0)
+            throw new IllegalArgumentException("native_transport_min_backoff_on_queue_overload should be positive");
+
+        if (minBackoffMillis >= maxBackoffMillis)
+            throw new IllegalArgumentException(String.format("native_transport_max_backoff_on_queue_overload should be greater than native_transport_min_backoff_on_queue_overload, but %s >= %s", minBackoffMillis, maxBackoffMillis));
+
+
+        conf.native_transport_min_backoff_on_queue_overload = new DurationSpec.LongMillisecondsBound(minBackoffMillis, timeUnit);
+        conf.native_transport_max_backoff_on_queue_overload = new DurationSpec.LongMillisecondsBound(maxBackoffMillis, timeUnit);
+    }
+
+    public static long getNativeTransportTimeout(TimeUnit timeUnit)
+    {
+        return conf.native_transport_timeout.to(timeUnit);
+    }
+
+    public static void setNativeTransportTimeout(long dealine, TimeUnit timeUnit)
+    {
+        conf.native_transport_timeout = new DurationSpec.LongMillisecondsBound(dealine, timeUnit);
+    }
+
+    public static boolean getEnforceNativeDeadlineForHints()
+    {
+        return conf.enforce_native_deadline_for_hints;
+    }
+
+    public static void setEnforceNativeDeadlineForHints(boolean value)
+    {
+        conf.enforce_native_deadline_for_hints = value;
+    }
+
+    public static boolean getNativeTransportThrowOnOverload()
+    {
+        return conf.native_transport_throw_on_overload;
+    }
+
+    public static void setNativeTransportThrowOnOverload(boolean throwOnOverload)
+    {
+        conf.native_transport_throw_on_overload = throwOnOverload;
+    }
+
     public static long getPingTimeout(TimeUnit unit)
     {
         return unit.convert(getBlockForPeersTimeoutInSeconds(), TimeUnit.SECONDS);
@@ -4422,4 +4506,14 @@ public class DatabaseDescriptor
     {
         conf.min_tracked_partition_tombstone_count = value;
     }
-}
+
+    public static Config.CQLStartTime getCQLStartTime()
+    {
+        return conf.cql_start_time;
+    }
+
+    public static void setCQLStartTime(Config.CQLStartTime value)
+    {
+        conf.cql_start_time = value;
+    }
+}

src/java/org/apache/cassandra/cql3/CQLStatement.java

@@ -24,6 +24,7 @@ import org.apache.cassandra.audit.AuditLogContext;
 import org.apache.cassandra.cql3.functions.Function;
 import org.apache.cassandra.service.ClientState;
 import org.apache.cassandra.service.QueryState;
+import org.apache.cassandra.transport.Dispatcher;
 import org.apache.cassandra.transport.messages.ResultMessage;
 
 public interface CQLStatement
@@ -75,9 +76,9 @@ public interface CQLStatement
      *
      * @param state the current query state
      * @param options options for this query (consistency, variables, pageSize, ...)
-     * @param queryStartNanoTime the timestamp returned by System.nanoTime() when this statement was received
+     * @param requestTime request enqueue / and start times
      */
-    public ResultMessage execute(QueryState state, QueryOptions options, long queryStartNanoTime);
+    public ResultMessage execute(QueryState state, QueryOptions options, Dispatcher.RequestTime requestTime);
 
     /**
      * Variant of execute used for internal query against the system tables, and thus only query the local node.

src/java/org/apache/cassandra/cql3/CustomPayloadMirroringQueryHandler.java

@@ -23,6 +23,7 @@ import java.util.Map;
 import org.apache.cassandra.cql3.statements.BatchStatement;
 import org.apache.cassandra.service.ClientState;
 import org.apache.cassandra.service.QueryState;
+import org.apache.cassandra.transport.Dispatcher;
 import org.apache.cassandra.transport.messages.ResultMessage;
 import org.apache.cassandra.utils.MD5Digest;
 
@@ -40,13 +41,14 @@ public class CustomPayloadMirroringQueryHandler implements QueryHandler
         return queryProcessor.parse(query, state, options);
     }
 
+    @Override
     public ResultMessage process(CQLStatement statement,
                                  QueryState state,
                                  QueryOptions options,
                                  Map<String, ByteBuffer> customPayload,
-                                 long queryStartNanoTime)
+                                 Dispatcher.RequestTime requestTime)
     {
-        ResultMessage result = queryProcessor.process(statement, state, options, customPayload, queryStartNanoTime);
+        ResultMessage result = queryProcessor.process(statement, state, options, customPayload, requestTime);
         result.setCustomPayload(customPayload);
         return result;
     }
@@ -63,24 +65,26 @@ public class CustomPayloadMirroringQueryHandler implements QueryHandler
         return queryProcessor.getPrepared(id);
     }
 
+    @Override
     public ResultMessage processPrepared(CQLStatement statement,
                                          QueryState state,
                                          QueryOptions options,
                                          Map<String, ByteBuffer> customPayload,
-                                         long queryStartNanoTime)
+                                         Dispatcher.RequestTime requestTime)
     {
-        ResultMessage result = queryProcessor.processPrepared(statement, state, options, customPayload, queryStartNanoTime);
+        ResultMessage result = queryProcessor.processPrepared(statement, state, options, customPayload, requestTime);
         result.setCustomPayload(customPayload);
         return result;
     }
 
+    @Override
     public ResultMessage processBatch(BatchStatement statement,
                                       QueryState state,
                                       BatchQueryOptions options,
                                       Map<String, ByteBuffer> customPayload,
-                                      long queryStartNanoTime)
+                                      Dispatcher.RequestTime requestTime)
     {
-        ResultMessage result = queryProcessor.processBatch(statement, state, options, customPayload, queryStartNanoTime);
+        ResultMessage result = queryProcessor.processBatch(statement, state, options, customPayload, requestTime);
         result.setCustomPayload(customPayload);
         return result;
     }

src/java/org/apache/cassandra/cql3/QueryHandler.java

@@ -25,6 +25,7 @@ import org.apache.cassandra.exceptions.RequestExecutionException;
 import org.apache.cassandra.exceptions.RequestValidationException;
 import org.apache.cassandra.service.ClientState;
 import org.apache.cassandra.service.QueryState;
+import org.apache.cassandra.transport.Dispatcher;
 import org.apache.cassandra.transport.messages.ResultMessage;
 import org.apache.cassandra.utils.MD5Digest;
 
@@ -36,7 +37,7 @@ public interface QueryHandler
                           QueryState state,
                           QueryOptions options,
                           Map<String, ByteBuffer> customPayload,
-                          long queryStartNanoTime) throws RequestExecutionException, RequestValidationException;
+                          Dispatcher.RequestTime requestTime) throws RequestExecutionException, RequestValidationException;
 
     ResultMessage.Prepared prepare(String query,
                                    ClientState clientState,
@@ -48,13 +49,13 @@ public interface QueryHandler
                                   QueryState state,
                                   QueryOptions options,
                                   Map<String, ByteBuffer> customPayload,
-                                  long queryStartNanoTime) throws RequestExecutionException, RequestValidationException;
+                                  Dispatcher.RequestTime requestTime) throws RequestExecutionException, RequestValidationException;
 
     ResultMessage processBatch(BatchStatement statement,
                                QueryState state,
                                BatchQueryOptions options,
                                Map<String, ByteBuffer> customPayload,
-                               long queryStartNanoTime) throws RequestExecutionException, RequestValidationException;
+                               Dispatcher.RequestTime requestTime) throws RequestExecutionException, RequestValidationException;
 
     public static class Prepared
     {

src/java/org/apache/cassandra/cql3/QueryProcessor.java

@@ -67,6 +67,7 @@ import org.apache.cassandra.metrics.CQLMetrics;
 import org.apache.cassandra.service.*;
 import org.apache.cassandra.service.pager.QueryPager;
 import org.apache.cassandra.tracing.Tracing;
+import org.apache.cassandra.transport.Dispatcher;
 import org.apache.cassandra.transport.ProtocolVersion;
 import org.apache.cassandra.transport.messages.ResultMessage;
 import org.apache.cassandra.utils.*;
@@ -245,7 +246,7 @@ public class QueryProcessor implements QueryHandler
         }
     }
 
-    public ResultMessage processStatement(CQLStatement statement, QueryState queryState, QueryOptions options, long queryStartNanoTime)
+    public ResultMessage processStatement(CQLStatement statement, QueryState queryState, QueryOptions options, Dispatcher.RequestTime requestTime)
     throws RequestExecutionException, RequestValidationException
     {
         logger.trace("Process {} @CL.{}", statement, options.getConsistency());
@@ -255,7 +256,7 @@ public class QueryProcessor implements QueryHandler
 
         ResultMessage result = options.getConsistency() == ConsistencyLevel.NODE_LOCAL
                              ? processNodeLocalStatement(statement, queryState, options)
-                             : statement.execute(queryState, options, queryStartNanoTime);
+                             : statement.execute(queryState, options, requestTime);
 
         return result == null ? new ResultMessage.Void() : result;
     }
@@ -316,12 +317,12 @@ public class QueryProcessor implements QueryHandler
         }
     }
 
-    public static ResultMessage process(String queryString, ConsistencyLevel cl, QueryState queryState, long queryStartNanoTime)
+    public static ResultMessage process(String queryString, ConsistencyLevel cl, QueryState queryState, Dispatcher.RequestTime requestTime)
     throws RequestExecutionException, RequestValidationException
     {
         QueryOptions options = QueryOptions.forInternalCalls(cl, Collections.<ByteBuffer>emptyList());
         CQLStatement statement = instance.parse(queryString, queryState, options);
-        return instance.process(statement, queryState, options, queryStartNanoTime);
+        return instance.process(statement, queryState, options, requestTime);
     }
 
     public CQLStatement parse(String queryString, QueryState queryState, QueryOptions options)
@@ -333,12 +334,12 @@ public class QueryProcessor implements QueryHandler
                                  QueryState state,
                                  QueryOptions options,
                                  Map<String, ByteBuffer> customPayload,
-                                 long queryStartNanoTime) throws RequestExecutionException, RequestValidationException
+                                 Dispatcher.RequestTime requestTime) throws RequestExecutionException, RequestValidationException
     {
-        return process(statement, state, options, queryStartNanoTime);
+        return process(statement, state, options, requestTime);
     }
 
-    public ResultMessage process(CQLStatement prepared, QueryState queryState, QueryOptions options, long queryStartNanoTime)
+    public ResultMessage process(CQLStatement prepared, QueryState queryState, QueryOptions options, Dispatcher.RequestTime requestTime)
     throws RequestExecutionException, RequestValidationException
     {
         options.prepare(prepared.getBindVariables());
@@ -348,7 +349,7 @@ public class QueryProcessor implements QueryHandler
         if (!queryState.getClientState().isInternal)
             metrics.regularStatementsExecuted.inc();
 
-        return processStatement(prepared, queryState, options, queryStartNanoTime);
+        return processStatement(prepared, queryState, options, requestTime);
     }
 
     public static CQLStatement parseStatement(String queryStr, ClientState clientState) throws RequestValidationException
@@ -366,7 +367,7 @@ public class QueryProcessor implements QueryHandler
         QueryState queryState = QueryState.forInternalCalls();
         QueryOptions options = QueryOptions.forInternalCalls(cl, values);
         CQLStatement statement = instance.parse(query, queryState, options);
-        ResultMessage result = instance.process(statement, queryState, options, nanoTime());
+        ResultMessage result = instance.process(statement, queryState, options, Dispatcher.RequestTime.forImmediateExecution());
         if (result instanceof ResultMessage.Rows)
             return UntypedResultSet.create(((ResultMessage.Rows)result).result);
         else
@@ -528,7 +529,7 @@ public class QueryProcessor implements QueryHandler
         try
         {
             Prepared prepared = prepareInternal(query);
-            ResultMessage result = prepared.statement.execute(state, makeInternalOptionsWithNowInSec(prepared.statement, state.getNowInSeconds(), values, cl), nanoTime());
+            ResultMessage result = prepared.statement.execute(state, makeInternalOptionsWithNowInSec(prepared.statement, state.getNowInSeconds(), values, cl), Dispatcher.RequestTime.forImmediateExecution());
             if (result instanceof ResultMessage.Rows)
                 return UntypedResultSet.create(((ResultMessage.Rows)result).result);
             else
@@ -588,12 +589,12 @@ public class QueryProcessor implements QueryHandler
      * Note that this only make sense for Selects so this only accept SELECT statements and is only useful in rare
      * cases.
      */
-    public static UntypedResultSet executeInternalWithNow(int nowInSec, long queryStartNanoTime, String query, Object... values)
+    public static UntypedResultSet executeInternalWithNow(int nowInSec, Dispatcher.RequestTime requestTime, String query, Object... values)
     {
         Prepared prepared = prepareInternal(query);
         assert prepared.statement instanceof SelectStatement;
         SelectStatement select = (SelectStatement)prepared.statement;
-        ResultMessage result = select.executeInternal(internalQueryState(), makeInternalOptionsWithNowInSec(prepared.statement, nowInSec, values), nowInSec, queryStartNanoTime);
+        ResultMessage result = select.executeInternal(internalQueryState(), makeInternalOptionsWithNowInSec(prepared.statement, nowInSec, values), nowInSec, requestTime);
         assert result instanceof ResultMessage.Rows;
         return UntypedResultSet.create(((ResultMessage.Rows)result).result);
     }
@@ -794,17 +795,18 @@ public class QueryProcessor implements QueryHandler
         return new ResultMessage.Prepared(statementId, resultMetadata.getResultMetadataId(), preparedMetadata, resultMetadata);
     }
 
+    @Override
     public ResultMessage processPrepared(CQLStatement statement,
                                          QueryState state,
                                          QueryOptions options,
                                          Map<String, ByteBuffer> customPayload,
-                                         long queryStartNanoTime)
+                                         Dispatcher.RequestTime requestTime)
                                                  throws RequestExecutionException, RequestValidationException
     {
-        return processPrepared(statement, state, options, queryStartNanoTime);
+        return processPrepared(statement, state, options, requestTime);
     }
 
-    public ResultMessage processPrepared(CQLStatement statement, QueryState queryState, QueryOptions options, long queryStartNanoTime)
+    public ResultMessage processPrepared(CQLStatement statement, QueryState queryState, QueryOptions options, Dispatcher.RequestTime requestTime)
     throws RequestExecutionException, RequestValidationException
     {
         List<ByteBuffer> variables = options.getValues();
@@ -823,27 +825,27 @@ public class QueryProcessor implements QueryHandler
         }
 
         metrics.preparedStatementsExecuted.inc();
-        return processStatement(statement, queryState, options, queryStartNanoTime);
+        return processStatement(statement, queryState, options, requestTime);
     }
 
     public ResultMessage processBatch(BatchStatement statement,
                                       QueryState state,
                                       BatchQueryOptions options,
                                       Map<String, ByteBuffer> customPayload,
-                                      long queryStartNanoTime)
+                                      Dispatcher.RequestTime requestTime)
                                               throws RequestExecutionException, RequestValidationException
     {
-        return processBatch(statement, state, options, queryStartNanoTime);
+        return processBatch(statement, state, options, requestTime);
     }
 
-    public ResultMessage processBatch(BatchStatement batch, QueryState queryState, BatchQueryOptions options, long queryStartNanoTime)
+    public ResultMessage processBatch(BatchStatement batch, QueryState queryState, BatchQueryOptions options, Dispatcher.RequestTime requestTime)
     throws RequestExecutionException, RequestValidationException
     {
         ClientState clientState = queryState.getClientState().cloneWithKeyspaceIfSet(options.getKeyspace());
         batch.authorize(clientState);
         batch.validate();
         batch.validate(clientState);
-        return batch.execute(queryState, options, queryStartNanoTime);
+        return batch.execute(queryState, options, requestTime);
     }
 
     public static CQLStatement getStatement(String queryStr, ClientState clientState)

src/java/org/apache/cassandra/cql3/UntypedResultSet.java

@@ -35,13 +35,12 @@ import org.apache.cassandra.db.rows.*;
 import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.service.ClientState;
 import org.apache.cassandra.service.pager.QueryPager;
+import org.apache.cassandra.transport.Dispatcher;
 import org.apache.cassandra.transport.ProtocolVersion;
 import org.apache.cassandra.utils.AbstractIterator;
 import org.apache.cassandra.utils.FBUtilities;
 import org.apache.cassandra.utils.TimeUUID;
 
-import static org.apache.cassandra.utils.Clock.Global.nanoTime;
-
 /** a utility for doing internal cql-based queries */
 public abstract class UntypedResultSet implements Iterable<UntypedResultSet.Row>
 {
@@ -274,7 +273,7 @@ public abstract class UntypedResultSet implements Iterable<UntypedResultSet.Row>
                         if (pager.isExhausted())
                             return endOfData();
 
-                        try (PartitionIterator iter = pager.fetchPage(pageSize, cl, clientState, nanoTime()))
+                        try (PartitionIterator iter = pager.fetchPage(pageSize, cl, clientState, Dispatcher.RequestTime.forImmediateExecution()))
                         {
                             currentPage = select.process(iter, nowInSec).rows.iterator();
                         }

src/java/org/apache/cassandra/cql3/statements/AuthenticationStatement.java

@@ -26,6 +26,7 @@ import org.apache.cassandra.exceptions.RequestValidationException;
 import org.apache.cassandra.exceptions.UnauthorizedException;
 import org.apache.cassandra.service.ClientState;
 import org.apache.cassandra.service.QueryState;
+import org.apache.cassandra.transport.Dispatcher;
 import org.apache.cassandra.transport.messages.ResultMessage;
 
 public abstract class AuthenticationStatement extends CQLStatement.Raw implements CQLStatement
@@ -35,7 +36,8 @@ public abstract class AuthenticationStatement extends CQLStatement.Raw implement
         return this;
     }
 
-    public ResultMessage execute(QueryState state, QueryOptions options, long queryStartNanoTime)
+    @Override
+    public ResultMessage execute(QueryState state, QueryOptions options, Dispatcher.RequestTime requestTime)
     throws RequestExecutionException, RequestValidationException
     {
         return execute(state.getClientState());
@@ -43,6 +45,7 @@ public abstract class AuthenticationStatement extends CQLStatement.Raw implement
 
     public abstract ResultMessage execute(ClientState state) throws RequestExecutionException, RequestValidationException;
 
+    @Override
     public ResultMessage executeLocally(QueryState state, QueryOptions options)
     {
         // executeLocally is for local query only, thus altering users doesn't make sense and is not supported

src/java/org/apache/cassandra/cql3/statements/AuthorizationStatement.java

@@ -26,6 +26,7 @@ import org.apache.cassandra.exceptions.RequestExecutionException;
 import org.apache.cassandra.exceptions.RequestValidationException;
 import org.apache.cassandra.service.ClientState;
 import org.apache.cassandra.service.QueryState;
+import org.apache.cassandra.transport.Dispatcher;
 import org.apache.cassandra.transport.messages.ResultMessage;
 import org.apache.commons.lang3.builder.ToStringBuilder;
 import org.apache.commons.lang3.builder.ToStringStyle;
@@ -37,7 +38,8 @@ public abstract class AuthorizationStatement extends CQLStatement.Raw implements
         return this;
     }
 
-    public ResultMessage execute(QueryState state, QueryOptions options, long queryStartNanoTime)
+    @Override
+    public ResultMessage execute(QueryState state, QueryOptions options, Dispatcher.RequestTime requestTime)
     throws RequestValidationException, RequestExecutionException
     {
         return execute(state.getClientState());

src/java/org/apache/cassandra/cql3/statements/BatchStatement.java

@@ -44,6 +44,7 @@ import org.apache.cassandra.exceptions.*;
 import org.apache.cassandra.metrics.BatchMetrics;
 import org.apache.cassandra.service.*;
 import org.apache.cassandra.tracing.Tracing;
+import org.apache.cassandra.transport.Dispatcher;
 import org.apache.cassandra.transport.messages.ResultMessage;
 import org.apache.cassandra.utils.FBUtilities;
 import org.apache.cassandra.utils.NoSpamLogger;
@@ -52,7 +53,6 @@ import org.apache.cassandra.utils.Pair;
 import static java.util.function.Predicate.isEqual;
 
 import static org.apache.cassandra.cql3.statements.RequestValidations.checkFalse;
-import static org.apache.cassandra.utils.Clock.Global.nanoTime;
 
 /**
  * A <code>BATCH</code> statement parsed from a CQL query.
@@ -271,7 +271,7 @@ public class BatchStatement implements CQLStatement
                                                   boolean local,
                                                   long batchTimestamp,
                                                   int nowInSeconds,
-                                                  long queryStartNanoTime)
+                                                  Dispatcher.RequestTime requestTime)
     {
         if (statements.isEmpty())
             return Collections.emptyList();
@@ -308,7 +308,7 @@ public class BatchStatement implements CQLStatement
             }
             QueryOptions statementOptions = options.forStatement(i);
             long timestamp = attrs.getTimestamp(batchTimestamp, statementOptions);
-            statement.addUpdates(collector, partitionKeys.get(i), state, statementOptions, local, timestamp, nowInSeconds, queryStartNanoTime);
+            statement.addUpdates(collector, partitionKeys.get(i), state, statementOptions, local, timestamp, nowInSeconds, requestTime);
         }
 
         if (tablesWithZeroGcGs != null)
@@ -396,12 +396,12 @@ public class BatchStatement implements CQLStatement
     }
 
 
-    public ResultMessage execute(QueryState queryState, QueryOptions options, long queryStartNanoTime)
+    public ResultMessage execute(QueryState queryState, QueryOptions options, Dispatcher.RequestTime requestTime)
     {
-        return execute(queryState, BatchQueryOptions.withoutPerStatementVariables(options), queryStartNanoTime);
+        return execute(queryState, BatchQueryOptions.withoutPerStatementVariables(options), requestTime);
     }
 
-    public ResultMessage execute(QueryState queryState, BatchQueryOptions options, long queryStartNanoTime)
+    public ResultMessage execute(QueryState queryState, BatchQueryOptions options, Dispatcher.RequestTime requestTime)
     {
         long timestamp = options.getTimestamp(queryState);
         int nowInSeconds = options.getNowInSeconds(queryState);
@@ -419,18 +419,18 @@ public class BatchStatement implements CQLStatement
             statements.get(i).validateDiskUsage(options.forStatement(i), clientState);
 
         if (hasConditions)
-            return executeWithConditions(options, queryState, queryStartNanoTime);
+            return executeWithConditions(options, queryState, requestTime);
 
         if (updatesVirtualTables)
-            executeInternalWithoutCondition(queryState, options, queryStartNanoTime);
+            executeInternalWithoutCondition(queryState, options, requestTime);
         else    
-            executeWithoutConditions(getMutations(clientState, options, false, timestamp, nowInSeconds, queryStartNanoTime),
-                                     options.getConsistency(), queryStartNanoTime);
+            executeWithoutConditions(getMutations(clientState, options, false, timestamp, nowInSeconds, requestTime),
+                                     options.getConsistency(), requestTime);
 
         return new ResultMessage.Void();
     }
 
-    private void executeWithoutConditions(List<? extends IMutation> mutations, ConsistencyLevel cl, long queryStartNanoTime) throws RequestExecutionException, RequestValidationException
+    private void executeWithoutConditions(List<? extends IMutation> mutations, ConsistencyLevel cl, Dispatcher.RequestTime requestTime) throws RequestExecutionException, RequestValidationException
     {
         if (mutations.isEmpty())
             return;
@@ -441,7 +441,7 @@ public class BatchStatement implements CQLStatement
         updatePartitionsPerBatchMetrics(mutations.size());
 
         boolean mutateAtomic = (isLogged() && mutations.size() > 1);
-        StorageProxy.mutateWithTriggers(mutations, cl, mutateAtomic, queryStartNanoTime);
+        StorageProxy.mutateWithTriggers(mutations, cl, mutateAtomic, requestTime);
     }
 
     private void updatePartitionsPerBatchMetrics(int updatedPartitions)
@@ -455,7 +455,7 @@ public class BatchStatement implements CQLStatement
         }
     }
 
-    private ResultMessage executeWithConditions(BatchQueryOptions options, QueryState state, long queryStartNanoTime)
+    private ResultMessage executeWithConditions(BatchQueryOptions options, QueryState state, Dispatcher.RequestTime requestTime)
     {
         Pair<CQL3CasRequest, Set<ColumnMetadata>> p = makeCasRequest(options, state);
         CQL3CasRequest casRequest = p.left;
@@ -472,7 +472,7 @@ public class BatchStatement implements CQLStatement
                                                    options.getConsistency(),
                                                    state.getClientState(),
                                                    options.getNowInSeconds(state),
-                                                   queryStartNanoTime))
+                                                   requestTime))
         {
             return new ResultMessage.Rows(ModificationStatement.buildCasResultSet(ksName,
                                                                                   tableName,
@@ -561,16 +561,16 @@ public class BatchStatement implements CQLStatement
         if (hasConditions)
             return executeInternalWithConditions(batchOptions, queryState);
 
-        executeInternalWithoutCondition(queryState, batchOptions, nanoTime());
+        executeInternalWithoutCondition(queryState, batchOptions, Dispatcher.RequestTime.forImmediateExecution());
         return new ResultMessage.Void();
     }
 
-    private ResultMessage executeInternalWithoutCondition(QueryState queryState, BatchQueryOptions batchOptions, long queryStartNanoTime)
+    private ResultMessage executeInternalWithoutCondition(QueryState queryState, BatchQueryOptions batchOptions, Dispatcher.RequestTime requestTime)
     {
         long timestamp = batchOptions.getTimestamp(queryState);
         int nowInSeconds = batchOptions.getNowInSeconds(queryState);
 
-        for (IMutation mutation : getMutations(queryState.getClientState(), batchOptions, true, timestamp, nowInSeconds, queryStartNanoTime))
+        for (IMutation mutation : getMutations(queryState.getClientState(), batchOptions, true, timestamp, nowInSeconds, requestTime))
             mutation.apply();
         return null;
     }

src/java/org/apache/cassandra/cql3/statements/DescribeStatement.java

@@ -46,6 +46,7 @@ import org.apache.cassandra.service.ClientState;
 import org.apache.cassandra.service.QueryState;
 import org.apache.cassandra.service.StorageService;
 import org.apache.cassandra.service.pager.PagingState;
+import org.apache.cassandra.transport.Dispatcher;
 import org.apache.cassandra.transport.messages.ResultMessage;
 import org.apache.cassandra.utils.ByteBufferUtil;
 import org.apache.cassandra.utils.FBUtilities;
@@ -124,7 +125,7 @@ public abstract class DescribeStatement<T> extends CQLStatement.Raw implements C
     }
 
     @Override
-    public final ResultMessage execute(QueryState state, QueryOptions options, long queryStartNanoTime) throws RequestValidationException, RequestExecutionException
+    public final ResultMessage execute(QueryState state, QueryOptions options, Dispatcher.RequestTime requestTime) throws RequestValidationException, RequestExecutionException
     {
         return executeLocally(state, options);
     }

src/java/org/apache/cassandra/cql3/statements/ModificationStatement.java

@@ -58,6 +58,7 @@ import org.apache.cassandra.service.disk.usage.DiskUsageBroadcaster;
 import org.apache.cassandra.service.paxos.Ballot;
 import org.apache.cassandra.service.paxos.BallotGenerator;
 import org.apache.cassandra.service.paxos.Commit.Proposal;
+import org.apache.cassandra.transport.Dispatcher;
 import org.apache.cassandra.transport.messages.ResultMessage;
 import org.apache.cassandra.triggers.TriggerExecutor;
 import org.apache.cassandra.utils.FBUtilities;
@@ -67,7 +68,6 @@ import org.apache.cassandra.utils.Pair;
 import static org.apache.cassandra.cql3.statements.RequestValidations.checkFalse;
 import static org.apache.cassandra.cql3.statements.RequestValidations.checkNull;
 import static org.apache.cassandra.service.paxos.Ballot.Flag.NONE;
-import static org.apache.cassandra.utils.Clock.Global.nanoTime;
 
 /*
  * Abstract parent class of individual modifications, i.e. INSERT, UPDATE and DELETE.
@@ -411,7 +411,7 @@ public abstract class ModificationStatement implements CQLStatement.SingleKeyspa
                                                            boolean local,
                                                            ConsistencyLevel cl,
                                                            int nowInSeconds,
-                                                           long queryStartNanoTime)
+                                                           Dispatcher.RequestTime requestTime)
     {
         if (!requiresRead())
             return null;
@@ -446,7 +446,7 @@ public abstract class ModificationStatement implements CQLStatement.SingleKeyspa
             }
         }
 
-        try (PartitionIterator iter = group.execute(cl, null, queryStartNanoTime))
+        try (PartitionIterator iter = group.execute(cl, null, requestTime))
         {
             return asMaterializedMap(iter);
         }
@@ -477,7 +477,7 @@ public abstract class ModificationStatement implements CQLStatement.SingleKeyspa
                && getRestrictions().isColumnRange();
     }
 
-    public ResultMessage execute(QueryState queryState, QueryOptions options, long queryStartNanoTime)
+    public ResultMessage execute(QueryState queryState, QueryOptions options, Dispatcher.RequestTime requestTime)
     throws RequestExecutionException, RequestValidationException
     {
         if (options.getConsistency() == null)
@@ -487,15 +487,15 @@ public abstract class ModificationStatement implements CQLStatement.SingleKeyspa
                                                 queryState.getClientState());
 
         return hasConditions()
-             ? executeWithCondition(queryState, options, queryStartNanoTime)
-             : executeWithoutCondition(queryState, options, queryStartNanoTime);
+             ? executeWithCondition(queryState, options, requestTime)
+             : executeWithoutCondition(queryState, options, requestTime);
     }
 
-    private ResultMessage executeWithoutCondition(QueryState queryState, QueryOptions options, long queryStartNanoTime)
+    private ResultMessage executeWithoutCondition(QueryState queryState, QueryOptions options, Dispatcher.RequestTime requestTime)
     throws RequestExecutionException, RequestValidationException
     {
         if (isVirtual())
-            return executeInternalWithoutCondition(queryState, options, queryStartNanoTime);
+            return executeInternalWithoutCondition(queryState, options, requestTime);
 
         ConsistencyLevel cl = options.getConsistency();
         if (isCounter())
@@ -511,14 +511,14 @@ public abstract class ModificationStatement implements CQLStatement.SingleKeyspa
                          false,
                          options.getTimestamp(queryState),
                          options.getNowInSeconds(queryState),
-                         queryStartNanoTime);
+                         requestTime);
         if (!mutations.isEmpty())
-            StorageProxy.mutateWithTriggers(mutations, cl, false, queryStartNanoTime);
+            StorageProxy.mutateWithTriggers(mutations, cl, false, requestTime);
 
         return null;
     }
 
-    private ResultMessage executeWithCondition(QueryState queryState, QueryOptions options, long queryStartNanoTime)
+    private ResultMessage executeWithCondition(QueryState queryState, QueryOptions options, Dispatcher.RequestTime requestTime)
     {
         CQL3CasRequest request = makeCasRequest(queryState, options);
 
@@ -530,7 +530,7 @@ public abstract class ModificationStatement implements CQLStatement.SingleKeyspa
                                                    options.getConsistency(),
                                                    queryState.getClientState(),
                                                    options.getNowInSeconds(queryState),
-                                                   queryStartNanoTime))
+                                                   requestTime))
         {
             return new ResultMessage.Rows(buildCasResultSet(result, queryState, options));
         }
@@ -663,15 +663,15 @@ public abstract class ModificationStatement implements CQLStatement.SingleKeyspa
     {
         return hasConditions()
                ? executeInternalWithCondition(queryState, options)
-               : executeInternalWithoutCondition(queryState, options, nanoTime());
+               : executeInternalWithoutCondition(queryState, options, Dispatcher.RequestTime.forImmediateExecution());
     }
 
-    public ResultMessage executeInternalWithoutCondition(QueryState queryState, QueryOptions options, long queryStartNanoTime)
+    public ResultMessage executeInternalWithoutCondition(QueryState queryState, QueryOptions options, Dispatcher.RequestTime requestTime)
     throws RequestValidationException, RequestExecutionException
     {
         long timestamp = options.getTimestamp(queryState);
         int nowInSeconds = options.getNowInSeconds(queryState);
-        for (IMutation mutation : getMutations(queryState.getClientState(), options, true, timestamp, nowInSeconds, queryStartNanoTime))
+        for (IMutation mutation : getMutations(queryState.getClientState(), options, true, timestamp, nowInSeconds, requestTime))
             mutation.apply();
         return null;
     }
@@ -724,12 +724,12 @@ public abstract class ModificationStatement implements CQLStatement.SingleKeyspa
                                                    boolean local,
                                                    long timestamp,
                                                    int nowInSeconds,
-                                                   long queryStartNanoTime)
+                                                   Dispatcher.RequestTime requestTime)
     {
         List<ByteBuffer> keys = buildPartitionKeyNames(options, state);
         HashMultiset<ByteBuffer> perPartitionKeyCounts = HashMultiset.create(keys);
         SingleTableUpdatesCollector collector = new SingleTableUpdatesCollector(metadata, updatedColumns, perPartitionKeyCounts);
-        addUpdates(collector, keys, state, options, local, timestamp, nowInSeconds, queryStartNanoTime);
+        addUpdates(collector, keys, state, options, local, timestamp, nowInSeconds, requestTime);
         return collector.toMutations();
     }
 
@@ -740,7 +740,7 @@ public abstract class ModificationStatement implements CQLStatement.SingleKeyspa
                           boolean local,
                           long timestamp,
                           int nowInSeconds,
-                          long queryStartNanoTime)
+                          Dispatcher.RequestTime requestTime)
     {
         if (hasSlices())
         {
@@ -758,7 +758,7 @@ public abstract class ModificationStatement implements CQLStatement.SingleKeyspa
                                                            local,
                                                            timestamp,
                                                            nowInSeconds,
-                                                           queryStartNanoTime);
+                                                           requestTime);
             for (ByteBuffer key : keys)
             {
                 Validation.validateKey(metadata(), key);
@@ -778,7 +778,7 @@ public abstract class ModificationStatement implements CQLStatement.SingleKeyspa
             if (restrictions.hasClusteringColumnsRestrictions() && clusterings.isEmpty())
                 return;
 
-            UpdateParameters params = makeUpdateParameters(keys, clusterings, state, options, local, timestamp, nowInSeconds, queryStartNanoTime);
+            UpdateParameters params = makeUpdateParameters(keys, clusterings, state, options, local, timestamp, nowInSeconds, requestTime);
 
             for (ByteBuffer key : keys)
             {
@@ -830,7 +830,7 @@ public abstract class ModificationStatement implements CQLStatement.SingleKeyspa
                                                   boolean local,
                                                   long timestamp,
                                                   int nowInSeconds,
-                                                  long queryStartNanoTime)
+                                                  Dispatcher.RequestTime requestTime)
     {
         if (clusterings.contains(Clustering.STATIC_CLUSTERING))
             return makeUpdateParameters(keys,
@@ -841,7 +841,7 @@ public abstract class ModificationStatement implements CQLStatement.SingleKeyspa
                                         local,
                                         timestamp,
                                         nowInSeconds,
-                                        queryStartNanoTime);
+                                        requestTime);
 
         return makeUpdateParameters(keys,
                                     new ClusteringIndexNamesFilter(clusterings, false),
@@ -851,7 +851,7 @@ public abstract class ModificationStatement implements CQLStatement.SingleKeyspa
                                     local,
                                     timestamp,
                                     nowInSeconds,
-                                    queryStartNanoTime);
+                                    requestTime);
     }
 
     private UpdateParameters makeUpdateParameters(Collection<ByteBuffer> keys,
@@ -862,7 +862,7 @@ public abstract class ModificationStatement implements CQLStatement.SingleKeyspa
                                                   boolean local,
                                                   long timestamp,
                                                   int nowInSeconds,
-                                                  long queryStartNanoTime)
+                                                  Dispatcher.RequestTime requestTime)
     {
         // Some lists operation requires reading
         Map<DecoratedKey, Partition> lists =
@@ -872,7 +872,7 @@ public abstract class ModificationStatement implements CQLStatement.SingleKeyspa
                               local,
                               options.getConsistency(),
                               nowInSeconds,
-                              queryStartNanoTime);
+                              requestTime);
 
         return new UpdateParameters(metadata(),
                                     updatedColumns(),

src/java/org/apache/cassandra/cql3/statements/SelectStatement.java

@@ -73,6 +73,7 @@ import org.apache.cassandra.service.StorageProxy;
 import org.apache.cassandra.service.pager.AggregationQueryPager;
 import org.apache.cassandra.service.pager.PagingState;
 import org.apache.cassandra.service.pager.QueryPager;
+import org.apache.cassandra.transport.Dispatcher;
 import org.apache.cassandra.transport.ProtocolVersion;
 import org.apache.cassandra.transport.messages.ResultMessage;
 import org.apache.cassandra.utils.ByteBufferUtil;
@@ -88,7 +89,6 @@ import static org.apache.cassandra.cql3.statements.RequestValidations.checkNull;
 import static org.apache.cassandra.cql3.statements.RequestValidations.checkTrue;
 import static org.apache.cassandra.cql3.statements.RequestValidations.invalidRequest;
 import static org.apache.cassandra.utils.ByteBufferUtil.UNSET_BYTE_BUFFER;
-import static org.apache.cassandra.utils.Clock.Global.nanoTime;
 
 /**
  * Encapsulates a completely parsed SELECT query, including the target
@@ -246,7 +246,7 @@ public class SelectStatement implements CQLStatement.SingleKeyspaceCqlStatement
             Guardrails.allowFilteringEnabled.ensureEnabled(state);
     }
 
-    public ResultMessage.Rows execute(QueryState state, QueryOptions options, long queryStartNanoTime)
+    public ResultMessage.Rows execute(QueryState state, QueryOptions options, Dispatcher.RequestTime requestTime)
     {
         ConsistencyLevel cl = options.getConsistency();
         checkNotNull(cl, "Invalid empty consistency level");
@@ -268,7 +268,7 @@ public class SelectStatement implements CQLStatement.SingleKeyspaceCqlStatement
             query.trackWarnings();
 
         if (aggregationSpec == null && (pageSize <= 0 || (query.limits().count() <= pageSize)))
-            return execute(query, options, state.getClientState(), selectors, nowInSec, userLimit, null, queryStartNanoTime);
+            return execute(query, options, state.getClientState(), selectors, nowInSec, userLimit, null, requestTime);
 
         QueryPager pager = getPager(query, options);
 
@@ -280,7 +280,7 @@ public class SelectStatement implements CQLStatement.SingleKeyspaceCqlStatement
                        nowInSec,
                        userLimit,
                        aggregationSpec,
-                       queryStartNanoTime);
+                       requestTime);
     }
 
 
@@ -328,9 +328,9 @@ public class SelectStatement implements CQLStatement.SingleKeyspaceCqlStatement
                                        int nowInSec,
                                        int userLimit,
                                        AggregationSpecification aggregationSpec,
-                                       long queryStartNanoTime)
+                                       Dispatcher.RequestTime requestTime)
     {
-        try (PartitionIterator data = query.execute(options.getConsistency(), state, queryStartNanoTime))
+        try (PartitionIterator data = query.execute(options.getConsistency(), state, requestTime))
         {
             return processResults(data, options, selectors, nowInSec, userLimit, aggregationSpec);
         }
@@ -372,7 +372,7 @@ public class SelectStatement implements CQLStatement.SingleKeyspaceCqlStatement
             return pager.state();
         }
 
-        public abstract PartitionIterator fetchPage(int pageSize, long queryStartNanoTime);
+        public abstract PartitionIterator fetchPage(int pageSize, Dispatcher.RequestTime requestTime);
 
         public static class NormalPager extends Pager
         {
@@ -386,9 +386,9 @@ public class SelectStatement implements CQLStatement.SingleKeyspaceCqlStatement
                 this.clientState = clientState;
             }
 
-            public PartitionIterator fetchPage(int pageSize, long queryStartNanoTime)
+            public PartitionIterator fetchPage(int pageSize, Dispatcher.RequestTime requestTime)
             {
-                return pager.fetchPage(pageSize, consistency, clientState, queryStartNanoTime);
+                return pager.fetchPage(pageSize, consistency, clientState, requestTime);
             }
         }
 
@@ -402,7 +402,7 @@ public class SelectStatement implements CQLStatement.SingleKeyspaceCqlStatement
                 this.executionController = executionController;
             }
 
-            public PartitionIterator fetchPage(int pageSize, long queryStartNanoTime)
+            public PartitionIterator fetchPage(int pageSize, Dispatcher.RequestTime requestTime)
             {
                 return pager.fetchPageInternal(pageSize, executionController);
             }
@@ -417,7 +417,7 @@ public class SelectStatement implements CQLStatement.SingleKeyspaceCqlStatement
                                        int nowInSec,
                                        int userLimit,
                                        AggregationSpecification aggregationSpec,
-                                       long queryStartNanoTime)
+                                       Dispatcher.RequestTime requestTime)
     {
         Guardrails.pageSize.guard(pageSize, table(), false, state.getClientState());
 
@@ -444,7 +444,7 @@ public class SelectStatement implements CQLStatement.SingleKeyspaceCqlStatement
                   + " you must either remove the ORDER BY or the IN and sort client side, or disable paging for this query");
 
         ResultMessage.Rows msg;
-        try (PartitionIterator page = pager.fetchPage(pageSize, queryStartNanoTime))
+        try (PartitionIterator page = pager.fetchPage(pageSize, requestTime))
         {
             msg = processResults(page, options, selectors, nowInSec, userLimit, aggregationSpec);
         }
@@ -476,13 +476,13 @@ public class SelectStatement implements CQLStatement.SingleKeyspaceCqlStatement
 
     public ResultMessage.Rows executeLocally(QueryState state, QueryOptions options) throws RequestExecutionException, RequestValidationException
     {
-        return executeInternal(state, options, options.getNowInSeconds(state), nanoTime());
+        return executeInternal(state, options, options.getNowInSeconds(state), Dispatcher.RequestTime.forImmediateExecution());
     }
 
     public ResultMessage.Rows executeInternal(QueryState state,
                                               QueryOptions options,
                                               int nowInSec,
-                                              long queryStartNanoTime)
+                                              Dispatcher.RequestTime requestTime)
     {
         int userLimit = getLimit(options);
         int userPerPartitionLimit = getPerPartitionLimit(options);
@@ -519,7 +519,7 @@ public class SelectStatement implements CQLStatement.SingleKeyspaceCqlStatement
                            nowInSec,
                            userLimit,
                            aggregationSpec,
-                           queryStartNanoTime);
+                           requestTime);
         }
     }
 

src/java/org/apache/cassandra/cql3/statements/TruncateStatement.java

@@ -34,6 +34,7 @@ import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.service.ClientState;
 import org.apache.cassandra.service.QueryState;
 import org.apache.cassandra.service.StorageProxy;
+import org.apache.cassandra.transport.Dispatcher;
 import org.apache.cassandra.transport.messages.ResultMessage;
 import org.apache.commons.lang3.builder.ToStringBuilder;
 import org.apache.commons.lang3.builder.ToStringStyle;
@@ -61,7 +62,8 @@ public class TruncateStatement extends QualifiedStatement implements CQLStatemen
         Guardrails.dropTruncateTableEnabled.ensureEnabled(state);
     }
 
-    public ResultMessage execute(QueryState state, QueryOptions options, long queryStartNanoTime) throws InvalidRequestException, TruncateException
+    @Override
+    public ResultMessage execute(QueryState state, QueryOptions options, Dispatcher.RequestTime requestTime) throws InvalidRequestException, TruncateException
     {
         try
         {

src/java/org/apache/cassandra/cql3/statements/UseStatement.java

@@ -25,6 +25,7 @@ import org.apache.cassandra.cql3.QueryOptions;
 import org.apache.cassandra.cql3.QueryProcessor;
 import org.apache.cassandra.exceptions.InvalidRequestException;
 import org.apache.cassandra.exceptions.UnauthorizedException;
+import org.apache.cassandra.transport.Dispatcher;
 import org.apache.cassandra.transport.messages.ResultMessage;
 import org.apache.cassandra.service.ClientState;
 import org.apache.cassandra.service.QueryState;
@@ -32,7 +33,6 @@ import org.apache.commons.lang3.builder.ToStringBuilder;
 import org.apache.commons.lang3.builder.ToStringStyle;
 
 import static org.apache.cassandra.cql3.statements.RequestValidations.checkTrue;
-import static org.apache.cassandra.utils.Clock.Global.nanoTime;
 
 public class UseStatement extends CQLStatement.Raw implements CQLStatement
 {
@@ -59,7 +59,8 @@ public class UseStatement extends CQLStatement.Raw implements CQLStatement
         checkTrue(DatabaseDescriptor.getUseStatementsEnabled(), "USE statements prohibited. (see use_statements_enabled in cassandra.yaml)");
     }
 
-    public ResultMessage execute(QueryState state, QueryOptions options, long queryStartNanoTime) throws InvalidRequestException
+    @Override
+    public ResultMessage execute(QueryState state, QueryOptions options, Dispatcher.RequestTime requestTime) throws InvalidRequestException
     {
         QueryProcessor.metrics.useStatementsExecuted.inc();
         state.getClientState().setKeyspace(keyspace);
@@ -70,7 +71,7 @@ public class UseStatement extends CQLStatement.Raw implements CQLStatement
     {
         // In production, internal queries are exclusively on the system keyspace and 'use' is thus useless
         // but for some unit tests we need to set the keyspace (e.g. for tests with DROP INDEX)
-        return execute(state, options, nanoTime());
+        return execute(state, options, Dispatcher.RequestTime.forImmediateExecution());
     }
     
     @Override

src/java/org/apache/cassandra/cql3/statements/schema/AlterSchemaStatement.java

@@ -32,6 +32,7 @@ import org.apache.cassandra.schema.Keyspaces.KeyspacesDiff;
 import org.apache.cassandra.service.ClientState;
 import org.apache.cassandra.service.ClientWarn;
 import org.apache.cassandra.service.QueryState;
+import org.apache.cassandra.transport.Dispatcher;
 import org.apache.cassandra.transport.Event.SchemaChange;
 import org.apache.cassandra.transport.messages.ResultMessage;
 
@@ -53,7 +54,8 @@ abstract public class AlterSchemaStatement implements CQLStatement.SingleKeyspac
         this.state = state;
     }
 
-    public ResultMessage execute(QueryState state, QueryOptions options, long queryStartNanoTime)
+    @Override
+    public ResultMessage execute(QueryState state, QueryOptions options, Dispatcher.RequestTime requestTime)
     {
         return execute(state, false);
     }

src/java/org/apache/cassandra/db/CounterMutationVerbHandler.java

@@ -25,8 +25,7 @@ import org.apache.cassandra.net.IVerbHandler;
 import org.apache.cassandra.net.Message;
 import org.apache.cassandra.net.MessagingService;
 import org.apache.cassandra.service.StorageProxy;
-
-import static org.apache.cassandra.utils.Clock.Global.nanoTime;
+import org.apache.cassandra.transport.Dispatcher;
 
 public class CounterMutationVerbHandler implements IVerbHandler<CounterMutation>
 {
@@ -36,7 +35,6 @@ public class CounterMutationVerbHandler implements IVerbHandler<CounterMutation>
 
     public void doVerb(final Message<CounterMutation> message)
     {
-        long queryStartNanoTime = nanoTime();
         final CounterMutation cm = message.payload;
         logger.trace("Applying forwarded {}", cm);
 
@@ -51,6 +49,6 @@ public class CounterMutationVerbHandler implements IVerbHandler<CounterMutation>
         StorageProxy.applyCounterMutationOnLeader(cm,
                                                   localDataCenter,
                                                   () -> MessagingService.instance().send(message.emptyResponse(), message.from()),
-                                                  queryStartNanoTime);
+                                                  Dispatcher.RequestTime.forImmediateExecution());
     }
 }

src/java/org/apache/cassandra/db/MutationVerbHandler.java

@@ -22,7 +22,9 @@ import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.net.*;
 import org.apache.cassandra.tracing.Tracing;
 
+import static java.util.concurrent.TimeUnit.NANOSECONDS;
 import static org.apache.cassandra.db.commitlog.CommitLogSegment.ENTRY_OVERHEAD_SIZE;
+import static org.apache.cassandra.utils.MonotonicClock.Global.approxTime;
 
 public class MutationVerbHandler implements IVerbHandler<Mutation>
 {
@@ -41,6 +43,13 @@ public class MutationVerbHandler implements IVerbHandler<Mutation>
 
     public void doVerb(Message<Mutation> message)
     {
+        if (approxTime.now() > message.expiresAtNanos())
+        {
+            Tracing.trace("Discarding mutation from {} (timed out)", message.from());
+            MessagingService.instance().metrics.recordDroppedMessage(message, message.elapsedSinceCreated(NANOSECONDS), NANOSECONDS);
+            return;
+        }
+
         message.payload.validateSize(MessagingService.current_version, ENTRY_OVERHEAD_SIZE);
 
         // Check if there were any forwarding headers in this message

src/java/org/apache/cassandra/db/PartitionRangeReadCommand.java

@@ -53,6 +53,7 @@ import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.service.ClientState;
 import org.apache.cassandra.service.StorageProxy;
 import org.apache.cassandra.tracing.Tracing;
+import org.apache.cassandra.transport.Dispatcher;
 
 /**
  * A read command that selects a (part of a) range of partitions.
@@ -300,9 +301,9 @@ public class PartitionRangeReadCommand extends ReadCommand implements PartitionR
         return dataRange.isReversed();
     }
 
-    public PartitionIterator execute(ConsistencyLevel consistency, ClientState state, long queryStartNanoTime) throws RequestExecutionException
+    public PartitionIterator execute(ConsistencyLevel consistency, ClientState state, Dispatcher.RequestTime requestTime) throws RequestExecutionException
     {
-        return StorageProxy.getRangeSlice(this, consistency, queryStartNanoTime);
+        return StorageProxy.getRangeSlice(this, consistency, requestTime);
     }
 
     protected void recordLatency(TableMetrics metric, long latencyNanos)
@@ -519,7 +520,7 @@ public class PartitionRangeReadCommand extends ReadCommand implements PartitionR
         }
 
         @Override
-        public PartitionIterator execute(ConsistencyLevel consistency, ClientState state, long queryStartNanoTime) throws RequestExecutionException
+        public PartitionIterator execute(ConsistencyLevel consistency, ClientState state, Dispatcher.RequestTime requestTime) throws RequestExecutionException
         {
             return executeInternal(executionController());
         }

src/java/org/apache/cassandra/db/ReadCommand.java

@@ -66,6 +66,7 @@ import org.apache.cassandra.schema.SchemaProvider;
 import org.apache.cassandra.service.ActiveRepairService;
 import org.apache.cassandra.service.ClientWarn;
 import org.apache.cassandra.tracing.Tracing;
+import org.apache.cassandra.transport.Dispatcher;
 import org.apache.cassandra.utils.FBUtilities;
 import org.apache.cassandra.utils.ObjectSizes;
 import org.apache.cassandra.utils.TimeUUID;
@@ -73,8 +74,8 @@ import org.apache.cassandra.utils.TimeUUID;
 import static com.google.common.collect.Iterables.any;
 import static com.google.common.collect.Iterables.filter;
 import static org.apache.cassandra.utils.Clock.Global.nanoTime;
-import static org.apache.cassandra.utils.MonotonicClock.Global.approxTime;
 import static org.apache.cassandra.db.partitions.UnfilteredPartitionIterators.MergeListener.NOOP;
+import static org.apache.cassandra.utils.MonotonicClock.Global.approxTime;
 
 /**
  * General interface for storage-engine read commands (common to both range and
@@ -745,14 +746,19 @@ public abstract class ReadCommand extends AbstractReadQuery
     /**
      * Creates a message for this command.
      */
-    public Message<ReadCommand> createMessage(boolean trackRepairedData)
+    public Message<ReadCommand> createMessage(boolean trackRepairedData, Dispatcher.RequestTime requestTime)
     {
-        Message<ReadCommand> msg = trackRepairedData
-                                   ? Message.outWithFlags(verb(), this, MessageFlag.CALL_BACK_ON_FAILURE, MessageFlag.TRACK_REPAIRED_DATA)
-                                   : Message.outWithFlag(verb(), this, MessageFlag.CALL_BACK_ON_FAILURE);
+        List<MessageFlag> flags = new ArrayList<>(3);
+        flags.add(MessageFlag.CALL_BACK_ON_FAILURE);
         if (trackWarnings)
-            msg = msg.withFlag(MessageFlag.TRACK_WARNINGS);
-        return msg;
+            flags.add(MessageFlag.TRACK_WARNINGS);
+        if (trackRepairedData)
+            flags.add(MessageFlag.TRACK_REPAIRED_DATA);
+
+        return Message.outWithFlags(verb(),
+                                    this,
+                                    requestTime,
+                                    flags);
     }
 
     public abstract Verb verb();

src/java/org/apache/cassandra/db/ReadQuery.java

@@ -26,6 +26,7 @@ import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.service.ClientState;
 import org.apache.cassandra.service.pager.QueryPager;
 import org.apache.cassandra.service.pager.PagingState;
+import org.apache.cassandra.transport.Dispatcher;
 import org.apache.cassandra.transport.ProtocolVersion;
 import org.apache.cassandra.utils.FBUtilities;
 
@@ -48,7 +49,7 @@ public interface ReadQuery
                 return ReadExecutionController.empty();
             }
 
-            public PartitionIterator execute(ConsistencyLevel consistency, ClientState state, long queryStartNanoTime) throws RequestExecutionException
+            public PartitionIterator execute(ConsistencyLevel consistency, ClientState state, Dispatcher.RequestTime requestTime) throws RequestExecutionException
             {
                 return EmptyIterators.partition();
             }
@@ -143,7 +144,7 @@ public interface ReadQuery
      * @param state client state
      * @return the result of the query.
      */
-    public PartitionIterator execute(ConsistencyLevel consistency, ClientState state, long queryStartNanoTime) throws RequestExecutionException;
+    public PartitionIterator execute(ConsistencyLevel consistency, ClientState state, Dispatcher.RequestTime requestTime) throws RequestExecutionException;
 
     /**
      * Execute the query for internal queries (that is, it basically executes the query locally).

src/java/org/apache/cassandra/db/SinglePartitionReadCommand.java

@@ -51,6 +51,7 @@ import org.apache.cassandra.schema.IndexMetadata;
 import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.service.*;
 import org.apache.cassandra.tracing.Tracing;
+import org.apache.cassandra.transport.Dispatcher;
 import org.apache.cassandra.utils.FBUtilities;
 import org.apache.cassandra.utils.btree.BTreeSet;
 
@@ -436,12 +437,12 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar
     }
 
     @Override
-    public PartitionIterator execute(ConsistencyLevel consistency, ClientState state, long queryStartNanoTime) throws RequestExecutionException
+    public PartitionIterator execute(ConsistencyLevel consistency, ClientState state, Dispatcher.RequestTime requestTime) throws RequestExecutionException
     {
         if (clusteringIndexFilter.isEmpty(metadata().comparator))
             return EmptyIterators.partition();
 
-        return StorageProxy.read(Group.one(this), consistency, queryStartNanoTime);
+        return StorageProxy.read(Group.one(this), consistency, requestTime);
     }
 
     protected void recordLatency(TableMetrics metric, long latencyNanos)
@@ -1218,9 +1219,9 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar
                    new Group(commands, limits);
         }
 
-        public PartitionIterator execute(ConsistencyLevel consistency, ClientState state, long queryStartNanoTime) throws RequestExecutionException
+        public PartitionIterator execute(ConsistencyLevel consistency, ClientState state, Dispatcher.RequestTime requestTime) throws RequestExecutionException
         {
-            return StorageProxy.read(this, consistency, queryStartNanoTime);
+            return StorageProxy.read(this, consistency, requestTime);
         }
     }
 
@@ -1232,13 +1233,13 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar
         }
 
         @Override
-        public PartitionIterator execute(ConsistencyLevel consistency, ClientState state, long queryStartNanoTime) throws RequestExecutionException
+        public PartitionIterator execute(ConsistencyLevel consistency, ClientState state, Dispatcher.RequestTime requestTime) throws RequestExecutionException
         {
             if (queries.size() == 1)
-                return queries.get(0).execute(consistency, state, queryStartNanoTime);
+                return queries.get(0).execute(consistency, state, requestTime);
 
             return PartitionIterators.concat(queries.stream()
-                                                    .map(q -> q.execute(consistency, state, queryStartNanoTime))
+                                                    .map(q -> q.execute(consistency, state, requestTime))
                                                     .collect(Collectors.toList()));
         }
     }
@@ -1311,7 +1312,7 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar
         }
 
         @Override
-        public PartitionIterator execute(ConsistencyLevel consistency, ClientState state, long queryStartNanoTime) throws RequestExecutionException
+        public PartitionIterator execute(ConsistencyLevel consistency, ClientState state, Dispatcher.RequestTime requestTime) throws RequestExecutionException
         {
             return executeInternal(executionController());
         }

src/java/org/apache/cassandra/db/view/TableViews.java

@@ -38,6 +38,7 @@ import org.apache.cassandra.schema.TableId;
 import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.schema.TableMetadataRef;
 import org.apache.cassandra.service.StorageProxy;
+import org.apache.cassandra.transport.Dispatcher;
 import org.apache.cassandra.utils.FBUtilities;
 import org.apache.cassandra.utils.btree.BTree;
 import org.apache.cassandra.utils.btree.BTreeSet;
@@ -149,7 +150,7 @@ public class TableViews extends AbstractCollection<View>
 
         // Read modified rows
         int nowInSec = FBUtilities.nowInSeconds();
-        long queryStartNanoTime = nanoTime();
+        Dispatcher.RequestTime requestTime = Dispatcher.RequestTime.forImmediateExecution();
         SinglePartitionReadCommand command = readExistingRowsCommand(update, views, nowInSec);
         if (command == null)
             return;
@@ -166,7 +167,7 @@ public class TableViews extends AbstractCollection<View>
         Keyspace.openAndGetStore(update.metadata()).metric.viewReadTime.update(nanoTime() - start, TimeUnit.NANOSECONDS);
 
         if (!mutations.isEmpty())
-            StorageProxy.mutateMV(update.partitionKey().getKey(), mutations, writeCommitLog, baseComplete, queryStartNanoTime);
+            StorageProxy.mutateMV(update.partitionKey().getKey(), mutations, writeCommitLog, baseComplete, requestTime);
     }
 
 

src/java/org/apache/cassandra/db/view/ViewBuilderTask.java

@@ -56,11 +56,11 @@ import org.apache.cassandra.gms.Gossiper;
 import org.apache.cassandra.io.sstable.ReducingKeyIterator;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
 import org.apache.cassandra.service.StorageProxy;
+import org.apache.cassandra.transport.Dispatcher;
 import org.apache.cassandra.utils.FBUtilities;
 import org.apache.cassandra.utils.TimeUUID;
 import org.apache.cassandra.utils.concurrent.Refs;
 
-import static org.apache.cassandra.utils.Clock.Global.nanoTime;
 import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID;
 
 public class ViewBuilderTask extends CompactionInfo.Holder implements Callable<Long>
@@ -115,7 +115,7 @@ public class ViewBuilderTask extends CompactionInfo.Holder implements Callable<L
                                                        .generateViewUpdates(Collections.singleton(view), data, empty, nowInSec, true);
 
             AtomicLong noBase = new AtomicLong(Long.MAX_VALUE);
-            mutations.forEachRemaining(m -> StorageProxy.mutateMV(key.getKey(), m, true, noBase, nanoTime()));
+            mutations.forEachRemaining(m -> StorageProxy.mutateMV(key.getKey(), m, true, noBase, Dispatcher.RequestTime.forImmediateExecution()));
         }
     }
 

src/java/org/apache/cassandra/locator/AbstractReplicationStrategy.java

@@ -45,6 +45,7 @@ import org.apache.cassandra.service.ClientState;
 import org.apache.cassandra.service.DatacenterSyncWriteResponseHandler;
 import org.apache.cassandra.service.DatacenterWriteResponseHandler;
 import org.apache.cassandra.service.WriteResponseHandler;
+import org.apache.cassandra.transport.Dispatcher;
 import org.apache.cassandra.utils.FBUtilities;
 import org.cliffc.high_scale_lib.NonBlockingHashMap;
 
@@ -135,32 +136,32 @@ public abstract class AbstractReplicationStrategy
                                                                        Runnable callback,
                                                                        WriteType writeType,
                                                                        Supplier<Mutation> hintOnFailure,
-                                                                       long queryStartNanoTime)
+                                                                       Dispatcher.RequestTime requestTime)
     {
         return getWriteResponseHandler(replicaPlan, callback, writeType, hintOnFailure,
-                                       queryStartNanoTime, DatabaseDescriptor.getIdealConsistencyLevel());
+                                       requestTime, DatabaseDescriptor.getIdealConsistencyLevel());
     }
 
     public <T> AbstractWriteResponseHandler<T> getWriteResponseHandler(ReplicaPlan.ForWrite replicaPlan,
                                                                        Runnable callback,
                                                                        WriteType writeType,
                                                                        Supplier<Mutation> hintOnFailure,
-                                                                       long queryStartNanoTime,
+                                                                       Dispatcher.RequestTime requestTime,
                                                                        ConsistencyLevel idealConsistencyLevel)
     {
         AbstractWriteResponseHandler<T> resultResponseHandler;
         if (replicaPlan.consistencyLevel().isDatacenterLocal())
         {
             // block for in this context will be localnodes block.
-            resultResponseHandler = new DatacenterWriteResponseHandler<T>(replicaPlan, callback, writeType, hintOnFailure, queryStartNanoTime);
+            resultResponseHandler = new DatacenterWriteResponseHandler<T>(replicaPlan, callback, writeType, hintOnFailure, requestTime);
         }
         else if (replicaPlan.consistencyLevel() == ConsistencyLevel.EACH_QUORUM && (this instanceof NetworkTopologyStrategy))
         {
-            resultResponseHandler = new DatacenterSyncWriteResponseHandler<T>(replicaPlan, callback, writeType, hintOnFailure, queryStartNanoTime);
+            resultResponseHandler = new DatacenterSyncWriteResponseHandler<T>(replicaPlan, callback, writeType, hintOnFailure, requestTime);
         }
         else
         {
-            resultResponseHandler = new WriteResponseHandler<T>(replicaPlan, callback, writeType, hintOnFailure, queryStartNanoTime);
+            resultResponseHandler = new WriteResponseHandler<T>(replicaPlan, callback, writeType, hintOnFailure, requestTime);
         }
 
         //Check if tracking the ideal consistency level is configured
@@ -180,7 +181,7 @@ public abstract class AbstractReplicationStrategy
                                                                                        callback,
                                                                                        writeType,
                                                                                        hintOnFailure,
-                                                                                       queryStartNanoTime,
+                                                                                       requestTime,
                                                                                        idealConsistencyLevel);
                 resultResponseHandler.setIdealCLResponseHandler(idealHandler);
             }

src/java/org/apache/cassandra/metrics/ClientMetrics.java

@@ -18,13 +18,23 @@
  */
 package org.apache.cassandra.metrics;
 
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 
+import com.google.common.annotations.VisibleForTesting;
+
 import com.codahale.metrics.Gauge;
 import com.codahale.metrics.Histogram;
 import com.codahale.metrics.Meter;
 import com.codahale.metrics.Reservoir;
+import com.codahale.metrics.Timer;
 import org.apache.cassandra.transport.*;
 
 import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics;
@@ -41,15 +51,17 @@ public final class ClientMetrics
     private Meter authSuccess;
     private Meter authFailure;
     private AtomicInteger pausedConnections;
-    
+
     @SuppressWarnings({ "unused", "FieldCanBeLocal" })
     private Gauge<Integer> pausedConnectionsGauge;
-    
+    private Meter connectionPaused;
     private Meter requestDiscarded;
     private Meter requestDispatched;
 
+    private Meter timedOutBeforeProcessing;
     private Meter protocolException;
     private Meter unknownException;
+    private Timer queueTime;
 
     private ClientMetrics()
     {
@@ -65,7 +77,17 @@ public final class ClientMetrics
         authFailure.mark();
     }
 
-    public void pauseConnection() { pausedConnections.incrementAndGet(); }
+    @VisibleForTesting
+    public int getNumberOfPausedConnections()
+    {
+        return (int) connectionPaused.getCount();
+    }
+
+    public void pauseConnection()
+    {
+        connectionPaused.mark();
+        pausedConnections.incrementAndGet();
+    }
     public void unpauseConnection() { pausedConnections.decrementAndGet(); }
 
     public void markRequestDiscarded() { requestDiscarded.mark(); }
@@ -81,6 +103,8 @@ public final class ClientMetrics
         return clients;
     }
 
+    public void markTimedOutBeforeProcessing() { timedOutBeforeProcessing.mark(); }
+
     public void markProtocolException()
     {
         protocolException.mark();
@@ -120,11 +144,14 @@ public final class ClientMetrics
 
         pausedConnections = new AtomicInteger();
         pausedConnectionsGauge = registerGauge("PausedConnections", pausedConnections::get);
+        connectionPaused = registerMeter("ConnectionPaused");
         requestDiscarded = registerMeter("RequestDiscarded");
         requestDispatched = registerMeter("RequestDispatched");
 
+        timedOutBeforeProcessing = registerMeter("TimedOutBeforeProcessing");
         protocolException = registerMeter("ProtocolException");
         unknownException = registerMeter("UnknownException");
+        queueTime = registerTimer("Queued");
 
         initialized = true;
     }
@@ -191,4 +218,14 @@ public final class ClientMetrics
     {
         return Metrics.meter(factory.createMetricName(name));
     }
+
+    public com.codahale.metrics.Timer registerTimer(String name)
+    {
+        return Metrics.timer(factory.createMetricName(name));
+    }
+
+    public void queueTime(long value, TimeUnit unit)
+    {
+        queueTime.update(value, unit);
+    }
 }

src/java/org/apache/cassandra/metrics/ThreadPoolMetrics.java

@@ -40,6 +40,7 @@ public class ThreadPoolMetrics
     public static final String TOTAL_BLOCKED_TASKS = "TotalBlockedTasks";
     public static final String MAX_POOL_SIZE = "MaxPoolSize";
     public static final String MAX_TASKS_QUEUED = "MaxTasksQueued";
+    public static final String OLDEST_TASK_QUEUE_TIME = "OldestTaskQueueTime";
 
     /** Number of active tasks. */
     public final Gauge<Integer> activeTasks;
@@ -62,6 +63,9 @@ public class ThreadPoolMetrics
     /** Maximum number of threads before it will start queuing tasks */
     public final Gauge<Integer> maxPoolSize;
 
+    /** For how long the oldest task in the queue was queued */
+    public final Gauge<Long> oldestTaskQueueTime;
+
     /** Maximum number of tasks queued before a task get blocked */
     public final Gauge<Integer> maxTasksQueued;
 
@@ -87,6 +91,7 @@ public class ThreadPoolMetrics
         completedTasks = executor::getCompletedTaskCount;
         maxPoolSize = executor::getMaximumPoolSize;
         maxTasksQueued = executor::getMaxTasksQueued;
+        oldestTaskQueueTime = executor::oldestTaskQueueTime;
     }
 
     public ThreadPoolMetrics register()
@@ -98,6 +103,7 @@ public class ThreadPoolMetrics
         Metrics.register(makeMetricName(path, poolName, TOTAL_BLOCKED_TASKS), totalBlocked);
         Metrics.register(makeMetricName(path, poolName, MAX_POOL_SIZE), maxPoolSize);
         Metrics.register(makeMetricName(path, poolName, MAX_TASKS_QUEUED), maxTasksQueued);
+        Metrics.register(makeMetricName(path, poolName, OLDEST_TASK_QUEUE_TIME), oldestTaskQueueTime);
         return Metrics.register(this);
     }
 
@@ -110,6 +116,7 @@ public class ThreadPoolMetrics
         Metrics.remove(makeMetricName(path, poolName, TOTAL_BLOCKED_TASKS));
         Metrics.remove(makeMetricName(path, poolName, MAX_POOL_SIZE));
         Metrics.remove(makeMetricName(path, poolName, MAX_TASKS_QUEUED));
+        Metrics.remove(makeMetricName(path, poolName, OLDEST_TASK_QUEUE_TIME));
         Metrics.remove(this);
     }
 

src/java/org/apache/cassandra/net/InboundMessageHandler.java

@@ -403,24 +403,24 @@ public class InboundMessageHandler extends AbstractMessageHandler
     {
         /**
          * Actually handle the message. Runs on the appropriate {@link Stage} for the {@link Verb}.
-         *
+         * <p>
          * Small messages will come pre-deserialized. Large messages will be deserialized on the stage,
          * just in time, and only then processed.
          */
         public void run()
         {
             Header header = header();
-            long currentTimeNanos = approxTime.now();
-            boolean expired = approxTime.isAfter(currentTimeNanos, header.expiresAtNanos);
+            long approxStartTimeNanos = approxTime.now();
+            boolean expired = approxTime.isAfter(approxStartTimeNanos, header.expiresAtNanos);
 
             boolean processed = false;
             try
             {
-                callbacks.onExecuting(size(), header, currentTimeNanos - header.createdAtNanos, NANOSECONDS);
+                callbacks.onExecuting(size(), header, approxStartTimeNanos - header.createdAtNanos, NANOSECONDS);
 
                 if (expired)
                 {
-                    callbacks.onExpired(size(), header, currentTimeNanos - header.createdAtNanos, NANOSECONDS);
+                    callbacks.onExpired(size(), header, approxStartTimeNanos - header.createdAtNanos, NANOSECONDS);
                     return;
                 }
 
@@ -441,7 +441,7 @@ public class InboundMessageHandler extends AbstractMessageHandler
 
                 releaseResources();
 
-                callbacks.onExecuted(size(), header, approxTime.now() - currentTimeNanos, NANOSECONDS);
+                callbacks.onExecuted(size(), header, approxTime.now() - approxStartTimeNanos, NANOSECONDS);
             }
         }
 

src/java/org/apache/cassandra/net/Message.java

@@ -22,6 +22,7 @@ import java.nio.ByteBuffer;
 import java.util.Collections;
 import java.util.EnumMap;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -44,6 +45,7 @@ import org.apache.cassandra.io.util.DataOutputPlus;
 import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.tracing.Tracing;
 import org.apache.cassandra.tracing.Tracing.TraceType;
+import org.apache.cassandra.transport.Dispatcher;
 import org.apache.cassandra.utils.MonotonicClockTranslation;
 import org.apache.cassandra.utils.NoSpamLogger;
 import org.apache.cassandra.utils.TimeUUID;
@@ -225,6 +227,23 @@ public class Message<T>
         return outWithParam(nextId(), verb, 0, payload, flag2.addTo(flag1.addTo(0)), null, null);
     }
 
+    public static <T> Message<T> outWithFlags(Verb verb, T payload, Dispatcher.RequestTime requestTime, List<MessageFlag> flags)
+    {
+        assert !verb.isResponse();
+        int encodedFlags = 0;
+        for (MessageFlag flag : flags)
+            encodedFlags = flag.addTo(encodedFlags);
+
+        return new Message<T>(new Header(nextId(),
+                                         verb,
+                                         getBroadcastAddressAndPort(),
+                                         requestTime.startedAtNanos(),
+                                         requestTime.computeDeadline(verb.expiresAfterNanos()),
+                                         encodedFlags,
+                                         buildParams(null, null)),
+                              payload);
+    }
+
     @VisibleForTesting
     static <T> Message<T> outWithParam(long id, Verb verb, T payload, ParamType paramType, Object paramValue)
     {
@@ -767,10 +786,8 @@ public class Message<T>
                  : extractHeaderPre40(buf, currentTimeNanos, version);
         }
 
-        private static long getExpiresAtNanos(long createdAtNanos, long currentTimeNanos, long expirationPeriodNanos)
+        private static long getExpiresAtNanos(long createdAtNanos, long expirationPeriodNanos)
         {
-            if (!DatabaseDescriptor.hasCrossNodeTimeout() || createdAtNanos > currentTimeNanos)
-                createdAtNanos = currentTimeNanos;
             return createdAtNanos + expirationPeriodNanos;
         }
 
@@ -796,7 +813,7 @@ public class Message<T>
             long currentTimeNanos = approxTime.now();
             MonotonicClockTranslation timeSnapshot = approxTime.translate();
             long creationTimeNanos = calculateCreationTimeNanos(in.readInt(), timeSnapshot, currentTimeNanos);
-            long expiresAtNanos = getExpiresAtNanos(creationTimeNanos, currentTimeNanos, TimeUnit.MILLISECONDS.toNanos(in.readUnsignedVInt()));
+            long expiresAtNanos = getExpiresAtNanos(creationTimeNanos, TimeUnit.MILLISECONDS.toNanos(in.readUnsignedVInt()));
             Verb verb = Verb.fromId(Ints.checkedCast(in.readUnsignedVInt()));
             int flags = Ints.checkedCast(in.readUnsignedVInt());
             Map<ParamType, Object> params = deserializeParams(in, version);
@@ -849,7 +866,7 @@ public class Message<T>
             Map<ParamType, Object> params = extractParams(buf, index, version);
 
             long createdAtNanos = calculateCreationTimeNanos(createdAtMillis, timeSnapshot, currentTimeNanos);
-            long expiresAtNanos = getExpiresAtNanos(createdAtNanos, currentTimeNanos, TimeUnit.MILLISECONDS.toNanos(expiresInMillis));
+            long expiresAtNanos = getExpiresAtNanos(createdAtNanos, TimeUnit.MILLISECONDS.toNanos(expiresInMillis));
 
             return new Header(id, verb, from, createdAtNanos, expiresAtNanos, flags, params);
         }
@@ -1150,6 +1167,10 @@ public class Message<T>
         @VisibleForTesting
         static long calculateCreationTimeNanos(int messageTimestampMillis, MonotonicClockTranslation timeSnapshot, long currentTimeNanos)
         {
+            // We do not trust external time source, so we override their value with current time
+            if (!DatabaseDescriptor.hasCrossNodeTimeout())
+                return currentTimeNanos;
+
             long currentTimeMillis = timeSnapshot.toMillisSinceEpoch(currentTimeNanos);
             // Reconstruct the message construction time sent by the remote host (we sent only the lower 4 bytes, assuming the
             // higher 4 bytes wouldn't change between the sender and receiver)

src/java/org/apache/cassandra/repair/RepairRunnable.java

@@ -38,6 +38,7 @@ import com.google.common.collect.Iterables;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Sets;
 
+import org.apache.cassandra.transport.Dispatcher;
 import org.apache.cassandra.utils.TimeUUID;
 import org.apache.cassandra.repair.state.ParticipateState;
 import org.apache.cassandra.utils.concurrent.Future;
@@ -88,7 +89,6 @@ import static org.apache.cassandra.repair.state.AbstractState.COMPLETE;
 import static org.apache.cassandra.repair.state.AbstractState.INIT;
 import static org.apache.cassandra.service.QueryState.forInternalCalls;
 import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis;
-import static org.apache.cassandra.utils.Clock.Global.nanoTime;
 
 public class RepairRunnable implements Runnable, ProgressEventNotifier, RepairNotifier
 {
@@ -532,7 +532,7 @@ public class RepairRunnable implements Runnable, ProgressEventNotifier, RepairNo
                     QueryOptions options = QueryOptions.forInternalCalls(ConsistencyLevel.ONE, Lists.newArrayList(sessionIdBytes,
                                                                                                                   tminBytes,
                                                                                                                   tmaxBytes));
-                    ResultMessage.Rows rows = statement.execute(forInternalCalls(), options, nanoTime());
+                    ResultMessage.Rows rows = statement.execute(forInternalCalls(), options, Dispatcher.RequestTime.forImmediateExecution());
                     UntypedResultSet result = UntypedResultSet.create(rows.result);
 
                     for (UntypedResultSet.Row r : result)

src/java/org/apache/cassandra/service/AbstractWriteResponseHandler.java

@@ -32,6 +32,7 @@ import org.apache.cassandra.db.Mutation;
 import org.apache.cassandra.locator.EndpointsForToken;
 import org.apache.cassandra.locator.ReplicaPlan;
 import org.apache.cassandra.locator.ReplicaPlan.ForWrite;
+import org.apache.cassandra.transport.Dispatcher;
 import org.apache.cassandra.utils.concurrent.Condition;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -62,7 +63,6 @@ import static org.apache.cassandra.service.StorageProxy.WritePerformer;
 import static org.apache.cassandra.utils.concurrent.Condition.newOneTimeCondition;
 import static org.apache.cassandra.locator.Replicas.countInOurDc;
 
-
 public abstract class AbstractWriteResponseHandler<T> implements RequestCallback<T>
 {
     protected static final Logger logger = LoggerFactory.getLogger(AbstractWriteResponseHandler.class);
@@ -78,7 +78,7 @@ public abstract class AbstractWriteResponseHandler<T> implements RequestCallback
         AtomicIntegerFieldUpdater.newUpdater(AbstractWriteResponseHandler.class, "failures");
     private volatile int failures = 0;
     private final Map<InetAddressAndPort, RequestFailureReason> failureReasonByEndpoint;
-    private final long queryStartNanoTime;
+    private final Dispatcher.RequestTime requestTime;
     private @Nullable final Supplier<Mutation> hintOnFailure;
 
     /**
@@ -97,17 +97,17 @@ public abstract class AbstractWriteResponseHandler<T> implements RequestCallback
     /**
      * @param callback           A callback to be called when the write is successful.
      * @param hintOnFailure
-     * @param queryStartNanoTime
+     * @param requestTime
      */
     protected AbstractWriteResponseHandler(ForWrite replicaPlan, Runnable callback, WriteType writeType,
-                                           Supplier<Mutation> hintOnFailure, long queryStartNanoTime)
+                                           Supplier<Mutation> hintOnFailure, Dispatcher.RequestTime requestTime)
     {
         this.replicaPlan = replicaPlan;
         this.callback = callback;
         this.writeType = writeType;
         this.hintOnFailure = hintOnFailure;
         this.failureReasonByEndpoint = new ConcurrentHashMap<>();
-        this.queryStartNanoTime = queryStartNanoTime;
+        this.requestTime = requestTime;
     }
 
     public void get() throws WriteTimeoutException, WriteFailureException
@@ -144,10 +144,11 @@ public abstract class AbstractWriteResponseHandler<T> implements RequestCallback
 
     public final long currentTimeoutNanos()
     {
+        long now = nanoTime();
         long requestTimeout = writeType == COUNTER
                               ? getCounterWriteRpcTimeout(NANOSECONDS)
                               : getWriteRpcTimeout(NANOSECONDS);
-        return requestTimeout - (nanoTime() - queryStartNanoTime);
+        return requestTime.computeTimeout(now, requestTimeout);
     }
 
     /**
@@ -282,7 +283,7 @@ public abstract class AbstractWriteResponseHandler<T> implements RequestCallback
         if (blockFor() + n > candidateReplicaCount())
             signal();
 
-        if (hintOnFailure != null && StorageProxy.shouldHint(replicaPlan.lookup(from)))
+        if (hintOnFailure != null && StorageProxy.shouldHint(replicaPlan.lookup(from)) && requestTime.shouldSendHints())
             StorageProxy.submitHint(hintOnFailure.get(), replicaPlan.lookup(from), null);
     }
 
@@ -310,7 +311,7 @@ public abstract class AbstractWriteResponseHandler<T> implements RequestCallback
             }
             else
             {
-                replicaPlan.keyspace().metric.idealCLWriteLatency.addNano(nanoTime() - queryStartNanoTime);
+                replicaPlan.keyspace().metric.idealCLWriteLatency.addNano(nanoTime() - requestTime.startedAtNanos());
             }
         }
     }
@@ -344,7 +345,8 @@ public abstract class AbstractWriteResponseHandler<T> implements RequestCallback
 
                 writePerformer.apply(mutation, replicaPlan.withContacts(uncontacted),
                                      (AbstractWriteResponseHandler<IMutation>) this,
-                                     localDC);
+                                     localDC,
+                                     requestTime);
             }
         }
         catch (InterruptedException e)

src/java/org/apache/cassandra/service/BatchlogResponseHandler.java

@@ -25,6 +25,7 @@ import org.apache.cassandra.exceptions.WriteFailureException;
 import org.apache.cassandra.exceptions.WriteTimeoutException;
 import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.net.Message;
+import org.apache.cassandra.transport.Dispatcher;
 
 public class BatchlogResponseHandler<T> extends AbstractWriteResponseHandler<T>
 {
@@ -34,9 +35,9 @@ public class BatchlogResponseHandler<T> extends AbstractWriteResponseHandler<T>
     private static final AtomicIntegerFieldUpdater<BatchlogResponseHandler> requiredBeforeFinishUpdater
             = AtomicIntegerFieldUpdater.newUpdater(BatchlogResponseHandler.class, "requiredBeforeFinish");
 
-    public BatchlogResponseHandler(AbstractWriteResponseHandler<T> wrapped, int requiredBeforeFinish, BatchlogCleanup cleanup, long queryStartNanoTime)
+    public BatchlogResponseHandler(AbstractWriteResponseHandler<T> wrapped, int requiredBeforeFinish, BatchlogCleanup cleanup, Dispatcher.RequestTime requestTime)
     {
-        super(wrapped.replicaPlan, wrapped.callback, wrapped.writeType, null, queryStartNanoTime);
+        super(wrapped.replicaPlan, wrapped.callback, wrapped.writeType, null, requestTime);
         this.wrapped = wrapped;
         this.requiredBeforeFinish = requiredBeforeFinish;
         this.cleanup = cleanup;

src/java/org/apache/cassandra/service/DatacenterSyncWriteResponseHandler.java

@@ -31,6 +31,7 @@ import org.apache.cassandra.locator.ReplicaPlan;
 import org.apache.cassandra.net.Message;
 import org.apache.cassandra.db.ConsistencyLevel;
 import org.apache.cassandra.db.WriteType;
+import org.apache.cassandra.transport.Dispatcher;
 
 /**
  * This class blocks for a quorum of responses _in all datacenters_ (CL.EACH_QUORUM).
@@ -46,10 +47,10 @@ public class DatacenterSyncWriteResponseHandler<T> extends AbstractWriteResponse
                                               Runnable callback,
                                               WriteType writeType,
                                               Supplier<Mutation> hintOnFailure,
-                                              long queryStartNanoTime)
+                                              Dispatcher.RequestTime requestTime)
     {
         // Response is been managed by the map so make it 1 for the superclass.
-        super(replicaPlan, callback, writeType, hintOnFailure, queryStartNanoTime);
+        super(replicaPlan, callback, writeType, hintOnFailure, requestTime);
         assert replicaPlan.consistencyLevel() == ConsistencyLevel.EACH_QUORUM;
 
         if (replicaPlan.replicationStrategy() instanceof NetworkTopologyStrategy)

src/java/org/apache/cassandra/service/DatacenterWriteResponseHandler.java

@@ -23,6 +23,7 @@ import org.apache.cassandra.locator.InOurDc;
 import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.locator.ReplicaPlan;
 import org.apache.cassandra.net.Message;
+import org.apache.cassandra.transport.Dispatcher;
 
 import java.util.function.Predicate;
 import java.util.function.Supplier;
@@ -38,9 +39,9 @@ public class DatacenterWriteResponseHandler<T> extends WriteResponseHandler<T>
                                           Runnable callback,
                                           WriteType writeType,
                                           Supplier<Mutation> hintOnFailure,
-                                          long queryStartNanoTime)
+                                          Dispatcher.RequestTime requestTime)
     {
-        super(replicaPlan, callback, writeType, hintOnFailure, queryStartNanoTime);
+        super(replicaPlan, callback, writeType, hintOnFailure, requestTime);
         assert replicaPlan.consistencyLevel().isDatacenterLocal();
     }
 

src/java/org/apache/cassandra/service/StorageProxy.java

@@ -21,6 +21,7 @@ import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -133,6 +134,7 @@ import org.apache.cassandra.service.reads.ReadCallback;
 import org.apache.cassandra.service.reads.range.RangeCommands;
 import org.apache.cassandra.service.reads.repair.ReadRepair;
 import org.apache.cassandra.tracing.Tracing;
+import org.apache.cassandra.transport.Dispatcher;
 import org.apache.cassandra.triggers.TriggerExecutor;
 import org.apache.cassandra.utils.Clock;
 import org.apache.cassandra.utils.FBUtilities;
@@ -217,10 +219,10 @@ public class StorageProxy implements StorageProxyMBean
         MBeanWrapper.instance.registerMBean(instance, MBEAN_NAME);
         HintsService.instance.registerMBean();
 
-        standardWritePerformer = (mutation, targets, responseHandler, localDataCenter) ->
+        standardWritePerformer = (mutation, targets, responseHandler, localDataCenter, requestTime) ->
         {
             assert mutation instanceof Mutation;
-            sendToHintedReplicas((Mutation) mutation, targets, responseHandler, localDataCenter, Stage.MUTATION);
+            sendToHintedReplicas((Mutation) mutation, targets, responseHandler, localDataCenter, Stage.MUTATION, requestTime);
         };
 
         /*
@@ -229,19 +231,19 @@ public class StorageProxy implements StorageProxyMBean
          * but on the latter case, the verb handler already run on the COUNTER_MUTATION stage, so we must not execute the
          * underlying on the stage otherwise we risk a deadlock. Hence two different performer.
          */
-        counterWritePerformer = (mutation, targets, responseHandler, localDataCenter) ->
+        counterWritePerformer = (mutation, targets, responseHandler, localDataCenter, requestTime) ->
         {
             EndpointsForToken selected = targets.contacts().withoutSelf();
             Replicas.temporaryAssertFull(selected); // TODO CASSANDRA-14548
-            counterWriteTask(mutation, targets.withContacts(selected), responseHandler, localDataCenter).run();
+            counterWriteTask(mutation, targets.withContacts(selected), responseHandler, localDataCenter, requestTime).run();
         };
 
-        counterWriteOnCoordinatorPerformer = (mutation, targets, responseHandler, localDataCenter) ->
+        counterWriteOnCoordinatorPerformer = (mutation, targets, responseHandler, localDataCenter, requestTime) ->
         {
             EndpointsForToken selected = targets.contacts().withoutSelf();
             Replicas.temporaryAssertFull(selected); // TODO CASSANDRA-14548
             Stage.COUNTER_MUTATION.executor()
-                                  .execute(counterWriteTask(mutation, targets.withContacts(selected), responseHandler, localDataCenter));
+                                  .execute(counterWriteTask(mutation, targets.withContacts(selected), responseHandler, localDataCenter, requestTime));
         };
 
 
@@ -307,7 +309,7 @@ public class StorageProxy implements StorageProxyMBean
                                   ConsistencyLevel consistencyForCommit,
                                   ClientState clientState,
                                   int nowInSeconds,
-                                  long queryStartNanoTime)
+                                  Dispatcher.RequestTime requestTime)
     throws UnavailableException, IsBootstrappingException, RequestFailureException, RequestTimeoutException, InvalidRequestException, CasWriteUnknownResultException
     {
         if (DatabaseDescriptor.getPartitionDenylistEnabled() && DatabaseDescriptor.getDenylistWritesEnabled() && !partitionDenylist.isKeyPermitted(keyspaceName, cfName, key.getKey()))
@@ -319,7 +321,7 @@ public class StorageProxy implements StorageProxyMBean
 
         return Paxos.useV2()
                 ? Paxos.cas(key, request, consistencyForPaxos, consistencyForCommit, clientState)
-                : legacyCas(keyspaceName, cfName, key, request, consistencyForPaxos, consistencyForCommit, clientState, nowInSeconds, queryStartNanoTime);
+                : legacyCas(keyspaceName, cfName, key, request, consistencyForPaxos, consistencyForCommit, clientState, nowInSeconds, requestTime);
     }
 
     public static RowIterator legacyCas(String keyspaceName,
@@ -330,10 +332,9 @@ public class StorageProxy implements StorageProxyMBean
                                         ConsistencyLevel consistencyForCommit,
                                         ClientState clientState,
                                         int nowInSeconds,
-                                        long queryStartNanoTime)
+                                        Dispatcher.RequestTime requestTime)
     throws UnavailableException, IsBootstrappingException, RequestFailureException, RequestTimeoutException, InvalidRequestException
     {
-        final long startTimeForMetrics = nanoTime();
         try
         {
             TableMetadata metadata = Schema.instance.validateTable(keyspaceName, cfName);
@@ -342,11 +343,11 @@ public class StorageProxy implements StorageProxyMBean
             {
                 // read the current values and check they validate the conditions
                 Tracing.trace("Reading existing values for CAS precondition");
-                SinglePartitionReadCommand readCommand = (SinglePartitionReadCommand) request.readCommand(nowInSeconds);
+                SinglePartitionReadCommand readCommand = request.readCommand(nowInSeconds);
                 ConsistencyLevel readConsistency = consistencyForPaxos == ConsistencyLevel.LOCAL_SERIAL ? ConsistencyLevel.LOCAL_QUORUM : ConsistencyLevel.QUORUM;
 
                 FilteredPartition current;
-                try (RowIterator rowIter = readOne(readCommand, readConsistency, queryStartNanoTime))
+                try (RowIterator rowIter = readOne(readCommand, readConsistency, requestTime))
                 {
                     current = FilteredPartition.create(rowIter);
                 }
@@ -382,7 +383,7 @@ public class StorageProxy implements StorageProxyMBean
                            consistencyForPaxos,
                            consistencyForCommit,
                            consistencyForCommit,
-                           queryStartNanoTime,
+                           requestTime,
                            casWriteMetrics,
                            updateProposer);
 
@@ -424,7 +425,9 @@ public class StorageProxy implements StorageProxyMBean
         }
         finally
         {
-            final long latency = nanoTime() - startTimeForMetrics;
+            // We track latency based on request processing time, since the amount of time that request spends in the queue
+            // is not a representative metric of replica performance.
+            long latency = nanoTime() - requestTime.startedAtNanos();
             casWriteMetrics.addNano(latency);
             writeMetricsForLevel(consistencyForPaxos).addNano(latency);
         }
@@ -460,7 +463,7 @@ public class StorageProxy implements StorageProxyMBean
      *     {@link ConsistencyLevel#LOCAL_SERIAL}).
      * @param consistencyForReplayCommits the consistency for the commit phase of "replayed" in-progress operations.
      * @param consistencyForCommit the consistency for the commit phase of _this_ operation update.
-     * @param queryStartNanoTime the nano time for the start of the query this is part of. This is the base time for
+     * @param requestTime the nano time for the start of the query this is part of. This is the base time for
      *     timeouts.
      * @param casMetrics the metrics to update for this operation.
      * @param createUpdateProposal method called after a successful 'prepare' phase to obtain 1) the actual update of
@@ -475,7 +478,7 @@ public class StorageProxy implements StorageProxyMBean
                                        ConsistencyLevel consistencyForPaxos,
                                        ConsistencyLevel consistencyForReplayCommits,
                                        ConsistencyLevel consistencyForCommit,
-                                       long queryStartNanoTime,
+                                       Dispatcher.RequestTime requestTime,
                                        CASClientRequestMetrics casMetrics,
                                        Function<Ballot, Pair<PartitionUpdate, RowIterator>> createUpdateProposal)
     throws UnavailableException, IsBootstrappingException, RequestFailureException, RequestTimeoutException, InvalidRequestException
@@ -490,12 +493,13 @@ public class StorageProxy implements StorageProxyMBean
             consistencyForCommit.validateForCasCommit(latestRs);
 
             long timeoutNanos = DatabaseDescriptor.getCasContentionTimeout(NANOSECONDS);
-            while (nanoTime() - queryStartNanoTime < timeoutNanos)
+            long deadline = requestTime.computeDeadline(timeoutNanos);
+            while (nanoTime() < deadline)
             {
                 // for simplicity, we'll do a single liveness check at the start of each attempt
                 ReplicaPlan.ForPaxosWrite replicaPlan = ReplicaPlans.forPaxos(keyspace, key, consistencyForPaxos);
                 latestRs = replicaPlan.replicationStrategy();
-                PaxosBallotAndContention pair = beginAndRepairPaxos(queryStartNanoTime,
+                PaxosBallotAndContention pair = beginAndRepairPaxos(requestTime,
                                                                     key,
                                                                     metadata,
                                                                     replicaPlan,
@@ -513,14 +517,14 @@ public class StorageProxy implements StorageProxyMBean
 
                 Commit proposal = Commit.newProposal(ballot, proposalPair.left);
                 Tracing.trace("CAS precondition is met; proposing client-requested updates for {}", ballot);
-                if (proposePaxos(proposal, replicaPlan, true, queryStartNanoTime))
+                if (proposePaxos(proposal, replicaPlan, true, requestTime))
                 {
                     // We skip committing accepted updates when they are empty. This is an optimization which works
                     // because we also skip replaying those same empty update in beginAndRepairPaxos (see the longer
                     // comment there). As empty update are somewhat common (serial reads and non-applying CAS propose
                     // them), this is worth bothering.
                     if (!proposal.update.isEmpty())
-                        commitPaxos(proposal, consistencyForCommit, true, queryStartNanoTime);
+                        commitPaxos(proposal, consistencyForCommit, true, requestTime);
                     RowIterator result = proposalPair.right;
                     if (result != null)
                         Tracing.trace("CAS did not apply");
@@ -563,7 +567,7 @@ public class StorageProxy implements StorageProxyMBean
      * @return the Paxos ballot promised by the replicas if no in-progress requests were seen and a quorum of
      * nodes have seen the mostRecentCommit.  Otherwise, return null.
      */
-    private static PaxosBallotAndContention beginAndRepairPaxos(long queryStartNanoTime,
+    private static PaxosBallotAndContention beginAndRepairPaxos(Dispatcher.RequestTime requestTime,
                                                                 DecoratedKey key,
                                                                 TableMetadata metadata,
                                                                 ReplicaPlan.ForPaxosWrite paxosPlan,
@@ -576,7 +580,8 @@ public class StorageProxy implements StorageProxyMBean
 
         PrepareCallback summary = null;
         int contentions = 0;
-        while (nanoTime() - queryStartNanoTime < timeoutNanos)
+        long deadline = requestTime.computeDeadline(timeoutNanos);
+        while (nanoTime() < deadline)
         {
             // We want a timestamp that is guaranteed to be unique for that node (so that the ballot is globally unique), but if we've got a prepare rejected
             // already we also want to make sure we pick a timestamp that has a chance to be promised, i.e. one that is greater that the most recently known
@@ -592,7 +597,7 @@ public class StorageProxy implements StorageProxyMBean
             {
                 Tracing.trace("Preparing {}", ballot);
                 Commit toPrepare = Commit.newPrepare(key, metadata, ballot);
-                summary = preparePaxos(toPrepare, paxosPlan, queryStartNanoTime);
+                summary = preparePaxos(toPrepare, paxosPlan, requestTime);
                 if (!summary.promised)
                 {
                     Tracing.trace("Some replicas have already promised a higher ballot than ours; aborting");
@@ -628,9 +633,9 @@ public class StorageProxy implements StorageProxyMBean
                     Tracing.trace("Finishing incomplete paxos round {}", inProgress);
                     casMetrics.unfinishedCommit.inc();
                     Commit refreshedInProgress = Commit.newProposal(ballot, inProgress.update);
-                    if (proposePaxos(refreshedInProgress, paxosPlan, false, queryStartNanoTime))
+                    if (proposePaxos(refreshedInProgress, paxosPlan, false, requestTime))
                     {
-                        commitPaxos(refreshedInProgress, consistencyForCommit, false, queryStartNanoTime);
+                        commitPaxos(refreshedInProgress, consistencyForCommit, false, requestTime);
                     }
                     else
                     {
@@ -680,10 +685,10 @@ public class StorageProxy implements StorageProxyMBean
             MessagingService.instance().send(message, target);
     }
 
-    private static PrepareCallback preparePaxos(Commit toPrepare, ReplicaPlan.ForPaxosWrite replicaPlan, long queryStartNanoTime)
+    private static PrepareCallback preparePaxos(Commit toPrepare, ReplicaPlan.ForPaxosWrite replicaPlan, Dispatcher.RequestTime requestTime)
     throws WriteTimeoutException
     {
-        PrepareCallback callback = new PrepareCallback(toPrepare.update.partitionKey(), toPrepare.update.metadata(), replicaPlan.requiredParticipants(), replicaPlan.consistencyLevel(), queryStartNanoTime);
+        PrepareCallback callback = new PrepareCallback(toPrepare.update.partitionKey(), toPrepare.update.metadata(), replicaPlan.requiredParticipants(), replicaPlan.consistencyLevel(), requestTime);
         Message<Commit> message = Message.out(PAXOS_PREPARE_REQ, toPrepare);
 
         boolean hasLocalRequest = false;
@@ -724,10 +729,10 @@ public class StorageProxy implements StorageProxyMBean
      * When {@param backoffIfPartial} is true, the proposer backs off when seeing the proposal being accepted by some but not a quorum.
      * The result of the cooresponding CAS in uncertain as the accepted proposal may or may not be spread to other nodes in later rounds.
      */
-    private static boolean proposePaxos(Commit proposal, ReplicaPlan.ForPaxosWrite replicaPlan, boolean backoffIfPartial, long queryStartNanoTime)
+    private static boolean proposePaxos(Commit proposal, ReplicaPlan.ForPaxosWrite replicaPlan, boolean backoffIfPartial, Dispatcher.RequestTime requestTime)
     throws WriteTimeoutException, CasWriteUnknownResultException
     {
-        ProposeCallback callback = new ProposeCallback(replicaPlan.contacts().size(), replicaPlan.requiredParticipants(), !backoffIfPartial, replicaPlan.consistencyLevel(), queryStartNanoTime);
+        ProposeCallback callback = new ProposeCallback(replicaPlan.contacts().size(), replicaPlan.requiredParticipants(), !backoffIfPartial, replicaPlan.consistencyLevel(), requestTime);
         Message<Commit> message = Message.out(PAXOS_PROPOSE_REQ, proposal);
         for (Replica replica : replicaPlan.contacts())
         {
@@ -761,7 +766,7 @@ public class StorageProxy implements StorageProxyMBean
         return false;
     }
 
-    private static void commitPaxos(Commit proposal, ConsistencyLevel consistencyLevel, boolean allowHints, long queryStartNanoTime) throws WriteTimeoutException
+    private static void commitPaxos(Commit proposal, ConsistencyLevel consistencyLevel, boolean allowHints, Dispatcher.RequestTime requestTime) throws WriteTimeoutException
     {
         boolean shouldBlock = consistencyLevel != ConsistencyLevel.ANY;
         Keyspace keyspace = Keyspace.open(proposal.update.metadata().keyspace);
@@ -774,7 +779,7 @@ public class StorageProxy implements StorageProxyMBean
         if (shouldBlock)
         {
             AbstractReplicationStrategy rs = replicaPlan.replicationStrategy();
-            responseHandler = rs.getWriteResponseHandler(replicaPlan, null, WriteType.SIMPLE, proposal::makeMutation, queryStartNanoTime);
+            responseHandler = rs.getWriteResponseHandler(replicaPlan, null, WriteType.SIMPLE, proposal::makeMutation, requestTime);
         }
 
         Message<Commit> message = Message.outWithFlag(PAXOS_COMMIT_REQ, proposal, MessageFlag.CALL_BACK_ON_FAILURE);
@@ -788,7 +793,7 @@ public class StorageProxy implements StorageProxyMBean
                 if (shouldBlock)
                 {
                     if (replica.isSelf())
-                        commitPaxosLocal(replica, message, responseHandler);
+                        commitPaxosLocal(replica, message, responseHandler, requestTime);
                     else
                         MessagingService.instance().sendWriteWithCallback(message, replica, responseHandler);
                 }
@@ -819,9 +824,9 @@ public class StorageProxy implements StorageProxyMBean
      * submit a fake one that executes immediately on the mutation stage, but generates the necessary backpressure
      * signal for hints
      */
-    private static void commitPaxosLocal(Replica localReplica, final Message<Commit> message, final AbstractWriteResponseHandler<?> responseHandler)
+    private static void commitPaxosLocal(Replica localReplica, final Message<Commit> message, final AbstractWriteResponseHandler<?> responseHandler, Dispatcher.RequestTime requestTime)
     {
-        PAXOS_COMMIT_REQ.stage.maybeExecuteImmediately(new LocalMutationRunnable(localReplica)
+        PAXOS_COMMIT_REQ.stage.maybeExecuteImmediately(new LocalMutationRunnable(localReplica, requestTime)
         {
             public void runMayThrow()
             {
@@ -861,16 +866,14 @@ public class StorageProxy implements StorageProxyMBean
      *
      * @param mutations the mutations to be applied across the replicas
      * @param consistencyLevel the consistency level for the operation
-     * @param queryStartNanoTime the value of nanoTime() when the query started to be processed
+     * @param requestTime object holding times when request got enqueued and started execution
      */
-    public static void mutate(List<? extends IMutation> mutations, ConsistencyLevel consistencyLevel, long queryStartNanoTime)
+    public static void mutate(List<? extends IMutation> mutations, ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime)
     throws UnavailableException, OverloadedException, WriteTimeoutException, WriteFailureException
     {
         Tracing.trace("Determining replicas for mutation");
         final String localDataCenter = DatabaseDescriptor.getEndpointSnitch().getLocalDatacenter();
 
-        long startTime = nanoTime();
-
         List<AbstractWriteResponseHandler<IMutation>> responseHandlers = new ArrayList<>(mutations.size());
         WriteType plainWriteType = mutations.size() <= 1 ? WriteType.SIMPLE : WriteType.UNLOGGED_BATCH;
 
@@ -879,9 +882,9 @@ public class StorageProxy implements StorageProxyMBean
             for (IMutation mutation : mutations)
             {
                 if (mutation instanceof CounterMutation)
-                    responseHandlers.add(mutateCounter((CounterMutation)mutation, localDataCenter, queryStartNanoTime));
+                    responseHandlers.add(mutateCounter((CounterMutation)mutation, localDataCenter, requestTime));
                 else
-                    responseHandlers.add(performWrite(mutation, consistencyLevel, localDataCenter, standardWritePerformer, null, plainWriteType, queryStartNanoTime));
+                    responseHandlers.add(performWrite(mutation, consistencyLevel, localDataCenter, standardWritePerformer, null, plainWriteType, requestTime));
             }
 
             // upgrade to full quorum any failed cheap quorums
@@ -937,7 +940,9 @@ public class StorageProxy implements StorageProxyMBean
         }
         finally
         {
-            long latency = nanoTime() - startTime;
+            // We track latency based on request processing time, since the amount of time that request spends in the queue
+            // is not a representative metric of replica performance.
+            long latency = nanoTime() - requestTime.startedAtNanos();
             writeMetrics.addNano(latency);
             writeMetricsForLevel(consistencyLevel).addNano(latency);
             updateCoordinatorWriteLatencyTableMetric(mutations, latency);
@@ -993,9 +998,9 @@ public class StorageProxy implements StorageProxyMBean
      * @param mutations the mutations to be applied across the replicas
      * @param writeCommitLog if commitlog should be written
      * @param baseComplete time from epoch in ms that the local base mutation was(or will be) completed
-     * @param queryStartNanoTime the value of nanoTime() when the query started to be processed
+     * @param requestTime object holding times when request got enqueued and started execution
      */
-    public static void mutateMV(ByteBuffer dataKey, Collection<Mutation> mutations, boolean writeCommitLog, AtomicLong baseComplete, long queryStartNanoTime)
+    public static void mutateMV(ByteBuffer dataKey, Collection<Mutation> mutations, boolean writeCommitLog, AtomicLong baseComplete, Dispatcher.RequestTime requestTime)
     throws UnavailableException, OverloadedException, WriteTimeoutException
     {
         Tracing.trace("Determining replicas for mutation");
@@ -1026,7 +1031,7 @@ public class StorageProxy implements StorageProxyMBean
                 //Since the base -> view replication is 1:1 we only need to store the BL locally
                 ReplicaPlan.ForWrite replicaPlan = ReplicaPlans.forLocalBatchlogWrite();
                 BatchlogCleanup cleanup = new BatchlogCleanup(mutations.size(),
-                                                              () -> asyncRemoveFromBatchlog(replicaPlan, batchUUID));
+                                                              () -> asyncRemoveFromBatchlog(replicaPlan, batchUUID, requestTime));
 
                 // add a handler for each mutation - includes checking availability, but doesn't initiate any writes, yet
                 for (Mutation mutation : mutations)
@@ -1081,7 +1086,7 @@ public class StorageProxy implements StorageProxyMBean
                                                                   baseComplete,
                                                                   WriteType.BATCH,
                                                                   cleanup,
-                                                                  queryStartNanoTime));
+                                                                  requestTime));
                     }
                 }
 
@@ -1091,7 +1096,7 @@ public class StorageProxy implements StorageProxyMBean
 
                 // Perform remote writes
                 if (!wrappers.isEmpty())
-                    asyncWriteBatchedMutations(wrappers, localDataCenter, Stage.VIEW_MUTATION);
+                    asyncWriteBatchedMutations(wrappers, localDataCenter, Stage.VIEW_MUTATION, requestTime);
             }
         }
         finally
@@ -1104,7 +1109,7 @@ public class StorageProxy implements StorageProxyMBean
     public static void mutateWithTriggers(List<? extends IMutation> mutations,
                                           ConsistencyLevel consistencyLevel,
                                           boolean mutateAtomically,
-                                          long queryStartNanoTime)
+                                          Dispatcher.RequestTime requestTime)
     throws WriteTimeoutException, WriteFailureException, UnavailableException, OverloadedException, InvalidRequestException
     {
         if (DatabaseDescriptor.getPartitionDenylistEnabled() && DatabaseDescriptor.getDenylistWritesEnabled())
@@ -1137,13 +1142,13 @@ public class StorageProxy implements StorageProxyMBean
         writeMetricsForLevel(consistencyLevel).mutationSize.update(size);
 
         if (augmented != null)
-            mutateAtomically(augmented, consistencyLevel, updatesView, queryStartNanoTime);
+            mutateAtomically(augmented, consistencyLevel, updatesView, requestTime);
         else
         {
             if (mutateAtomically || updatesView)
-                mutateAtomically((Collection<Mutation>) mutations, consistencyLevel, updatesView, queryStartNanoTime);
+                mutateAtomically((Collection<Mutation>) mutations, consistencyLevel, updatesView, requestTime);
             else
-                mutate(mutations, consistencyLevel, queryStartNanoTime);
+                mutate(mutations, consistencyLevel, requestTime);
         }
     }
 
@@ -1156,12 +1161,12 @@ public class StorageProxy implements StorageProxyMBean
      * @param mutations the Mutations to be applied across the replicas
      * @param consistency_level the consistency level for the operation
      * @param requireQuorumForRemove at least a quorum of nodes will see update before deleting batchlog
-     * @param queryStartNanoTime the value of nanoTime() when the query started to be processed
+     * @param requestTime object holding times when request got enqueued and started execution
      */
     public static void mutateAtomically(Collection<Mutation> mutations,
                                         ConsistencyLevel consistency_level,
                                         boolean requireQuorumForRemove,
-                                        long queryStartNanoTime)
+                                        Dispatcher.RequestTime requestTime)
     throws UnavailableException, OverloadedException, WriteTimeoutException
     {
         Tracing.trace("Determining replicas for atomic batch");
@@ -1192,7 +1197,7 @@ public class StorageProxy implements StorageProxyMBean
 
             final TimeUUID batchUUID = nextTimeUUID();
             BatchlogCleanup cleanup = new BatchlogCleanup(mutations.size(),
-                                                          () -> asyncRemoveFromBatchlog(replicaPlan, batchUUID));
+                                                          () -> asyncRemoveFromBatchlog(replicaPlan, batchUUID, requestTime));
 
             // add a handler for each mutation - includes checking availability, but doesn't initiate any writes, yet
             for (Mutation mutation : mutations)
@@ -1202,16 +1207,16 @@ public class StorageProxy implements StorageProxyMBean
                                                                                batchConsistencyLevel,
                                                                                WriteType.BATCH,
                                                                                cleanup,
-                                                                               queryStartNanoTime);
+                                                                               requestTime);
                 // exit early if we can't fulfill the CL at this time.
                 wrappers.add(wrapper);
             }
 
             // write to the batchlog
-            syncWriteToBatchlog(mutations, replicaPlan, batchUUID, queryStartNanoTime);
+            syncWriteToBatchlog(mutations, replicaPlan, batchUUID, requestTime);
 
             // now actually perform the writes and wait for them to complete
-            syncWriteBatchedMutations(wrappers, Stage.MUTATION);
+            syncWriteBatchedMutations(wrappers, Stage.MUTATION, requestTime);
         }
         catch (UnavailableException e)
         {
@@ -1265,13 +1270,13 @@ public class StorageProxy implements StorageProxyMBean
         }
     }
 
-    private static void syncWriteToBatchlog(Collection<Mutation> mutations, ReplicaPlan.ForWrite replicaPlan, TimeUUID uuid, long queryStartNanoTime)
+    private static void syncWriteToBatchlog(Collection<Mutation> mutations, ReplicaPlan.ForWrite replicaPlan, TimeUUID uuid, Dispatcher.RequestTime requestTime)
     throws WriteTimeoutException, WriteFailureException
     {
-        WriteResponseHandler<?> handler = new WriteResponseHandler(replicaPlan,
-                                                                   WriteType.BATCH_LOG,
-                                                                   null,
-                                                                   queryStartNanoTime);
+        WriteResponseHandler<?> handler = new WriteResponseHandler<>(replicaPlan,
+                                                                     WriteType.BATCH_LOG,
+                                                                     null,
+                                                                     requestTime);
 
         Batch batch = Batch.createLocal(uuid, FBUtilities.timestampMicros(), mutations);
         Message<Batch> message = Message.out(BATCH_STORE_REQ, batch);
@@ -1280,14 +1285,14 @@ public class StorageProxy implements StorageProxyMBean
             logger.trace("Sending batchlog store request {} to {} for {} mutations", batch.id, replica, batch.size());
 
             if (replica.isSelf())
-                performLocally(Stage.MUTATION, replica, () -> BatchlogManager.store(batch), handler, "Batchlog store");
+                performLocally(Stage.MUTATION, replica, () -> BatchlogManager.store(batch), handler, "Batchlog store", requestTime);
             else
                 MessagingService.instance().sendWithCallback(message, replica.endpoint(), handler);
         }
         handler.get();
     }
 
-    private static void asyncRemoveFromBatchlog(ReplicaPlan.ForWrite replicaPlan, TimeUUID uuid)
+    private static void asyncRemoveFromBatchlog(ReplicaPlan.ForWrite replicaPlan, TimeUUID uuid, Dispatcher.RequestTime requestTime)
     {
         Message<TimeUUID> message = Message.out(Verb.BATCH_REMOVE_REQ, uuid);
         for (Replica target : replicaPlan.contacts())
@@ -1296,13 +1301,13 @@ public class StorageProxy implements StorageProxyMBean
                 logger.trace("Sending batchlog remove request {} to {}", uuid, target);
 
             if (target.isSelf())
-                performLocally(Stage.MUTATION, target, () -> BatchlogManager.remove(uuid), "Batchlog remove");
+                performLocally(Stage.MUTATION, target, () -> BatchlogManager.remove(uuid), "Batchlog remove", requestTime);
             else
                 MessagingService.instance().send(message, target.endpoint());
         }
     }
 
-    private static void asyncWriteBatchedMutations(List<WriteResponseHandlerWrapper> wrappers, String localDataCenter, Stage stage)
+    private static void asyncWriteBatchedMutations(List<WriteResponseHandlerWrapper> wrappers, String localDataCenter, Stage stage, Dispatcher.RequestTime requestTime)
     {
         for (WriteResponseHandlerWrapper wrapper : wrappers)
         {
@@ -1311,7 +1316,7 @@ public class StorageProxy implements StorageProxyMBean
 
             try
             {
-                sendToHintedReplicas(wrapper.mutation, replicas, wrapper.handler, localDataCenter, stage);
+                sendToHintedReplicas(wrapper.mutation, replicas, wrapper.handler, localDataCenter, stage, requestTime);
             }
             catch (OverloadedException | WriteTimeoutException e)
             {
@@ -1320,7 +1325,7 @@ public class StorageProxy implements StorageProxyMBean
         }
     }
 
-    private static void syncWriteBatchedMutations(List<WriteResponseHandlerWrapper> wrappers, Stage stage)
+    private static void syncWriteBatchedMutations(List<WriteResponseHandlerWrapper> wrappers, Stage stage, Dispatcher.RequestTime requestTime)
     throws WriteTimeoutException, OverloadedException
     {
         String localDataCenter = DatabaseDescriptor.getEndpointSnitch().getLocalDatacenter();
@@ -1329,7 +1334,7 @@ public class StorageProxy implements StorageProxyMBean
         {
             EndpointsForToken sendTo = wrapper.handler.replicaPlan.liveAndDown();
             Replicas.temporaryAssertFull(sendTo); // TODO: CASSANDRA-14549
-            sendToHintedReplicas(wrapper.mutation, wrapper.handler.replicaPlan.withContacts(sendTo), wrapper.handler, localDataCenter, stage);
+            sendToHintedReplicas(wrapper.mutation, wrapper.handler.replicaPlan.withContacts(sendTo), wrapper.handler, localDataCenter, stage, requestTime);
         }
 
         for (WriteResponseHandlerWrapper wrapper : wrappers)
@@ -1348,7 +1353,7 @@ public class StorageProxy implements StorageProxyMBean
      * given the list of write endpoints (either standardWritePerformer for
      * standard writes or counterWritePerformer for counter writes).
      * @param callback an optional callback to be run if and when the write is
-     * @param queryStartNanoTime the value of nanoTime() when the query started to be processed
+     * @param requestTime object holding times when request got enqueued and started execution
      */
     public static AbstractWriteResponseHandler<IMutation> performWrite(IMutation mutation,
                                                                        ConsistencyLevel consistencyLevel,
@@ -1356,7 +1361,7 @@ public class StorageProxy implements StorageProxyMBean
                                                                        WritePerformer performer,
                                                                        Runnable callback,
                                                                        WriteType writeType,
-                                                                       long queryStartNanoTime)
+                                                                       Dispatcher.RequestTime requestTime)
     {
         String keyspaceName = mutation.getKeyspaceName();
         Keyspace keyspace = Keyspace.open(keyspaceName);
@@ -1370,9 +1375,9 @@ public class StorageProxy implements StorageProxyMBean
             writeMetrics.remoteRequests.mark();
 
         AbstractReplicationStrategy rs = replicaPlan.replicationStrategy();
-        AbstractWriteResponseHandler<IMutation> responseHandler = rs.getWriteResponseHandler(replicaPlan, callback, writeType, mutation.hintOnFailure(), queryStartNanoTime);
+        AbstractWriteResponseHandler<IMutation> responseHandler = rs.getWriteResponseHandler(replicaPlan, callback, writeType, mutation.hintOnFailure(), requestTime);
 
-        performer.apply(mutation, replicaPlan, responseHandler, localDataCenter);
+        performer.apply(mutation, replicaPlan, responseHandler, localDataCenter, requestTime);
         return responseHandler;
     }
 
@@ -1382,7 +1387,7 @@ public class StorageProxy implements StorageProxyMBean
                                                                         ConsistencyLevel batchConsistencyLevel,
                                                                         WriteType writeType,
                                                                         BatchlogResponseHandler.BatchlogCleanup cleanup,
-                                                                        long queryStartNanoTime)
+                                                                        Dispatcher.RequestTime requestTime)
     {
         Keyspace keyspace = Keyspace.open(mutation.getKeyspaceName());
         Token tk = mutation.key().getToken();
@@ -1395,8 +1400,8 @@ public class StorageProxy implements StorageProxyMBean
             writeMetrics.remoteRequests.mark();
 
         AbstractReplicationStrategy rs = replicaPlan.replicationStrategy();
-        AbstractWriteResponseHandler<IMutation> writeHandler = rs.getWriteResponseHandler(replicaPlan, null, writeType, mutation, queryStartNanoTime);
-        BatchlogResponseHandler<IMutation> batchHandler = new BatchlogResponseHandler<>(writeHandler, batchConsistencyLevel.blockFor(rs), cleanup, queryStartNanoTime);
+        AbstractWriteResponseHandler<IMutation> writeHandler = rs.getWriteResponseHandler(replicaPlan, null, writeType, mutation, requestTime);
+        BatchlogResponseHandler<IMutation> batchHandler = new BatchlogResponseHandler<>(writeHandler, batchConsistencyLevel.blockFor(rs), cleanup, requestTime);
         return new WriteResponseHandlerWrapper(batchHandler, mutation);
     }
 
@@ -1411,7 +1416,7 @@ public class StorageProxy implements StorageProxyMBean
                                                                             AtomicLong baseComplete,
                                                                             WriteType writeType,
                                                                             BatchlogResponseHandler.BatchlogCleanup cleanup,
-                                                                            long queryStartNanoTime)
+                                                                            Dispatcher.RequestTime requestTime)
     {
         Keyspace keyspace = Keyspace.open(mutation.getKeyspaceName());
         ReplicaPlan.ForWrite replicaPlan = ReplicaPlans.forWrite(keyspace, consistencyLevel, liveAndDown, ReplicaPlans.writeAll);
@@ -1419,8 +1424,8 @@ public class StorageProxy implements StorageProxyMBean
         AbstractWriteResponseHandler<IMutation> writeHandler = replicationStrategy.getWriteResponseHandler(replicaPlan, () -> {
             long delay = Math.max(0, currentTimeMillis() - baseComplete.get());
             viewWriteMetrics.viewWriteLatency.update(delay, MILLISECONDS);
-        }, writeType, mutation, queryStartNanoTime);
-        BatchlogResponseHandler<IMutation> batchHandler = new ViewWriteMetricsWrapped(writeHandler, batchConsistencyLevel.blockFor(replicationStrategy), cleanup, queryStartNanoTime);
+        }, writeType, mutation, requestTime);
+        BatchlogResponseHandler<IMutation> batchHandler = new ViewWriteMetricsWrapped(writeHandler, batchConsistencyLevel.blockFor(replicationStrategy), cleanup, requestTime);
         return new WriteResponseHandlerWrapper(batchHandler, mutation);
     }
 
@@ -1458,7 +1463,8 @@ public class StorageProxy implements StorageProxyMBean
                                             ReplicaPlan.ForWrite plan,
                                             AbstractWriteResponseHandler<IMutation> responseHandler,
                                             String localDataCenter,
-                                            Stage stage)
+                                            Stage stage,
+                                            Dispatcher.RequestTime requestTime)
     throws OverloadedException
     {
         // this dc replicas:
@@ -1487,7 +1493,12 @@ public class StorageProxy implements StorageProxyMBean
                 {
                     // belongs on a different server
                     if (message == null)
-                        message = Message.outWithFlag(MUTATION_REQ, mutation, MessageFlag.CALL_BACK_ON_FAILURE);
+                    {
+                        message = Message.outWithFlags(MUTATION_REQ,
+                                                       mutation,
+                                                       requestTime,
+                                                       Collections.singletonList(MessageFlag.CALL_BACK_ON_FAILURE));
+                    }
 
                     String dc = DatabaseDescriptor.getEndpointSnitch().getDatacenter(destination);
 
@@ -1527,13 +1538,13 @@ public class StorageProxy implements StorageProxyMBean
             }
         }
 
-        if (endpointsToHint != null)
+        if (endpointsToHint != null && requestTime.shouldSendHints())
             submitHint(mutation, EndpointsForToken.copyOf(mutation.key().getToken(), endpointsToHint), responseHandler);
 
         if (insertLocal)
         {
             Preconditions.checkNotNull(localReplica);
-            performLocally(stage, localReplica, mutation::apply, responseHandler, mutation);
+            performLocally(stage, localReplica, mutation::apply, responseHandler, mutation, requestTime);
         }
 
         if (localDc != null)
@@ -1600,9 +1611,9 @@ public class StorageProxy implements StorageProxyMBean
         logger.trace("Sending message to {}@{}", message.id(), target);
     }
 
-    private static void performLocally(Stage stage, Replica localReplica, final Runnable runnable, String description)
+    private static void performLocally(Stage stage, Replica localReplica, final Runnable runnable, String description, Dispatcher.RequestTime requestTime)
     {
-        stage.maybeExecuteImmediately(new LocalMutationRunnable(localReplica)
+        stage.maybeExecuteImmediately(new LocalMutationRunnable(localReplica, requestTime)
         {
             public void runMayThrow()
             {
@@ -1630,9 +1641,9 @@ public class StorageProxy implements StorageProxyMBean
         });
     }
 
-    private static void performLocally(Stage stage, Replica localReplica, final Runnable runnable, final RequestCallback<?> handler, Object description)
+    private static void performLocally(Stage stage, Replica localReplica, final Runnable runnable, final RequestCallback<?> handler, Object description, Dispatcher.RequestTime requestTime)
     {
-        stage.maybeExecuteImmediately(new LocalMutationRunnable(localReplica)
+        stage.maybeExecuteImmediately(new LocalMutationRunnable(localReplica, requestTime)
         {
             public void runMayThrow()
             {
@@ -1679,13 +1690,13 @@ public class StorageProxy implements StorageProxyMBean
      * quicker response and because the WriteResponseHandlers don't make it easy to send back an error. We also always gather
      * the write latencies at the coordinator node to make gathering point similar to the case of standard writes.
      */
-    public static AbstractWriteResponseHandler<IMutation> mutateCounter(CounterMutation cm, String localDataCenter, long queryStartNanoTime) throws UnavailableException, OverloadedException
+    public static AbstractWriteResponseHandler<IMutation> mutateCounter(CounterMutation cm, String localDataCenter, Dispatcher.RequestTime requestTime) throws UnavailableException, OverloadedException
     {
         Replica replica = findSuitableReplica(cm.getKeyspaceName(), cm.key(), localDataCenter, cm.consistency());
 
         if (replica.isSelf())
         {
-            return applyCounterMutationOnCoordinator(cm, localDataCenter, queryStartNanoTime);
+            return applyCounterMutationOnCoordinator(cm, localDataCenter, requestTime);
         }
         else
         {
@@ -1704,7 +1715,7 @@ public class StorageProxy implements StorageProxyMBean
 
             // Forward the actual update to the chosen leader replica
             AbstractWriteResponseHandler<IMutation> responseHandler = new WriteResponseHandler<>(ReplicaPlans.forForwardingCounterWrite(keyspace, tk, replica),
-                                                                                                 WriteType.COUNTER, null, queryStartNanoTime);
+                                                                                                 WriteType.COUNTER, null, requestTime);
 
             Tracing.trace("Enqueuing counter update to {}", replica);
             Message message = Message.outWithFlag(Verb.COUNTER_MUTATION_REQ, cm, MessageFlag.CALL_BACK_ON_FAILURE);
@@ -1762,26 +1773,27 @@ public class StorageProxy implements StorageProxyMBean
 
     // Must be called on a replica of the mutation. This replica becomes the
     // leader of this mutation.
-    public static AbstractWriteResponseHandler<IMutation> applyCounterMutationOnLeader(CounterMutation cm, String localDataCenter, Runnable callback, long queryStartNanoTime)
+    public static AbstractWriteResponseHandler<IMutation> applyCounterMutationOnLeader(CounterMutation cm, String localDataCenter, Runnable callback, Dispatcher.RequestTime requestTime)
     throws UnavailableException, OverloadedException
     {
-        return performWrite(cm, cm.consistency(), localDataCenter, counterWritePerformer, callback, WriteType.COUNTER, queryStartNanoTime);
+        return performWrite(cm, cm.consistency(), localDataCenter, counterWritePerformer, callback, WriteType.COUNTER, requestTime);
     }
 
     // Same as applyCounterMutationOnLeader but must with the difference that it use the MUTATION stage to execute the write (while
     // applyCounterMutationOnLeader assumes it is on the MUTATION stage already)
-    public static AbstractWriteResponseHandler<IMutation> applyCounterMutationOnCoordinator(CounterMutation cm, String localDataCenter, long queryStartNanoTime)
+    public static AbstractWriteResponseHandler<IMutation> applyCounterMutationOnCoordinator(CounterMutation cm, String localDataCenter, Dispatcher.RequestTime requestTime)
     throws UnavailableException, OverloadedException
     {
-        return performWrite(cm, cm.consistency(), localDataCenter, counterWriteOnCoordinatorPerformer, null, WriteType.COUNTER, queryStartNanoTime);
+        return performWrite(cm, cm.consistency(), localDataCenter, counterWriteOnCoordinatorPerformer, null, WriteType.COUNTER, requestTime);
     }
 
     private static Runnable counterWriteTask(final IMutation mutation,
                                              final ReplicaPlan.ForWrite replicaPlan,
                                              final AbstractWriteResponseHandler<IMutation> responseHandler,
-                                             final String localDataCenter)
+                                             final String localDataCenter,
+                                             final Dispatcher.RequestTime requestTime)
     {
-        return new DroppableRunnable(Verb.COUNTER_MUTATION_REQ)
+        return new DroppableRunnable(Verb.COUNTER_MUTATION_REQ, requestTime)
         {
             @Override
             public void runMayThrow() throws OverloadedException, WriteTimeoutException
@@ -1790,7 +1802,7 @@ public class StorageProxy implements StorageProxyMBean
 
                 Mutation result = ((CounterMutation) mutation).applyCounterMutation();
                 responseHandler.onResponse(null);
-                sendToHintedReplicas(result, replicaPlan, responseHandler, localDataCenter, Stage.COUNTER_MUTATION);
+                sendToHintedReplicas(result, replicaPlan, responseHandler, localDataCenter, Stage.COUNTER_MUTATION, requestTime);
             }
         };
     }
@@ -1803,17 +1815,17 @@ public class StorageProxy implements StorageProxyMBean
         return true;
     }
 
-    public static RowIterator readOne(SinglePartitionReadCommand command, ConsistencyLevel consistencyLevel, long queryStartNanoTime)
+    public static RowIterator readOne(SinglePartitionReadCommand command, ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime)
     throws UnavailableException, IsBootstrappingException, ReadFailureException, ReadTimeoutException, InvalidRequestException
     {
-        return PartitionIterators.getOnlyElement(read(SinglePartitionReadCommand.Group.one(command), consistencyLevel, queryStartNanoTime), command);
+        return PartitionIterators.getOnlyElement(read(SinglePartitionReadCommand.Group.one(command), consistencyLevel, requestTime), command);
     }
 
     /**
      * Performs the actual reading of a row out of the StorageService, fetching
      * a specific set of column names from a given column family.
      */
-    public static PartitionIterator read(SinglePartitionReadCommand.Group group, ConsistencyLevel consistencyLevel, long queryStartNanoTime)
+    public static PartitionIterator read(SinglePartitionReadCommand.Group group, ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime)
     throws UnavailableException, IsBootstrappingException, ReadFailureException, ReadTimeoutException, InvalidRequestException
     {
         if (StorageService.instance.isBootstrapMode() && !systemKeyspaceQuery(group.queries))
@@ -1839,25 +1851,24 @@ public class StorageProxy implements StorageProxyMBean
         }
 
         return consistencyLevel.isSerialConsistency()
-             ? readWithPaxos(group, consistencyLevel, queryStartNanoTime)
-             : readRegular(group, consistencyLevel, queryStartNanoTime);
+             ? readWithPaxos(group, consistencyLevel, requestTime)
+             : readRegular(group, consistencyLevel, requestTime);
     }
 
-    private static PartitionIterator readWithPaxos(SinglePartitionReadCommand.Group group, ConsistencyLevel consistencyLevel, long queryStartNanoTime)
+    private static PartitionIterator readWithPaxos(SinglePartitionReadCommand.Group group, ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime)
     throws InvalidRequestException, UnavailableException, ReadFailureException, ReadTimeoutException
     {
         return Paxos.useV2()
-                ? Paxos.read(group, consistencyLevel)
-                : legacyReadWithPaxos(group, consistencyLevel, queryStartNanoTime);
+                ? Paxos.read(group, consistencyLevel, requestTime)
+                : legacyReadWithPaxos(group, consistencyLevel, requestTime);
     }
 
-    private static PartitionIterator legacyReadWithPaxos(SinglePartitionReadCommand.Group group, ConsistencyLevel consistencyLevel, long queryStartNanoTime)
+    private static PartitionIterator legacyReadWithPaxos(SinglePartitionReadCommand.Group group, ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime)
     throws InvalidRequestException, UnavailableException, ReadFailureException, ReadTimeoutException
     {
         if (group.queries.size() > 1)
             throw new InvalidRequestException("SERIAL/LOCAL_SERIAL consistency may only be requested for one partition at a time");
 
-        long start = nanoTime();
         SinglePartitionReadCommand command = group.queries.get(0);
         TableMetadata metadata = command.metadata();
         DecoratedKey key = command.partitionKey();
@@ -1888,7 +1899,7 @@ public class StorageProxy implements StorageProxyMBean
                         consistencyLevel,
                         consistencyForReplayCommitsOrFetch,
                         ConsistencyLevel.ANY,
-                        start,
+                        requestTime,
                         casReadMetrics,
                         updateProposer);
             }
@@ -1901,7 +1912,7 @@ public class StorageProxy implements StorageProxyMBean
                 throw new ReadFailureException(consistencyLevel, e.received, e.blockFor, false, e.failureReasonByEndpoint);
             }
 
-            result = fetchRows(group.queries, consistencyForReplayCommitsOrFetch, queryStartNanoTime);
+            result = fetchRows(group.queries, consistencyForReplayCommitsOrFetch, requestTime);
         }
         catch (UnavailableException e)
         {
@@ -1935,7 +1946,9 @@ public class StorageProxy implements StorageProxyMBean
         }
         finally
         {
-            long latency = nanoTime() - start;
+            // We track latency based on request processing time, since the amount of time that request spends in the queue
+            // is not a representative metric of replica performance.
+            long latency = nanoTime() - requestTime.startedAtNanos();
             readMetrics.addNano(latency);
             casReadMetrics.addNano(latency);
             readMetricsForLevel(consistencyLevel).addNano(latency);
@@ -1946,13 +1959,12 @@ public class StorageProxy implements StorageProxyMBean
     }
 
     @SuppressWarnings("resource")
-    private static PartitionIterator readRegular(SinglePartitionReadCommand.Group group, ConsistencyLevel consistencyLevel, long queryStartNanoTime)
+    private static PartitionIterator readRegular(SinglePartitionReadCommand.Group group, ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime)
     throws UnavailableException, ReadFailureException, ReadTimeoutException
     {
-        long start = nanoTime();
         try
         {
-            PartitionIterator result = fetchRows(group.queries, consistencyLevel, queryStartNanoTime);
+            PartitionIterator result = fetchRows(group.queries, consistencyLevel, requestTime);
             // Note that the only difference between the command in a group must be the partition key on which
             // they applied.
             boolean enforceStrictLiveness = group.queries.get(0).metadata().enforceStrictLiveness();
@@ -1989,7 +2001,9 @@ public class StorageProxy implements StorageProxyMBean
         }
         finally
         {
-            long latency = nanoTime() - start;
+            // We track latency based on request processing time, since the amount of time that request spends in the queue
+            // is not a representative metric of replica performance.
+            long latency = nanoTime() - requestTime.startedAtNanos();
             readMetrics.addNano(latency);
             readMetricsForLevel(consistencyLevel).addNano(latency);
             // TODO avoid giving every command the same latency number.  Can fix this in CASSADRA-5329
@@ -2043,7 +2057,7 @@ public class StorageProxy implements StorageProxyMBean
      * 4. If the digests (if any) match the data return the data
      * 5. else carry out read repair by getting data from all the nodes.
      */
-    private static PartitionIterator fetchRows(List<SinglePartitionReadCommand> commands, ConsistencyLevel consistencyLevel, long queryStartNanoTime)
+    private static PartitionIterator fetchRows(List<SinglePartitionReadCommand> commands, ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime)
     throws UnavailableException, ReadFailureException, ReadTimeoutException
     {
         int cmdCount = commands.size();
@@ -2054,7 +2068,7 @@ public class StorageProxy implements StorageProxyMBean
         // for type of speculation we'll use in this read
         for (int i=0; i<cmdCount; i++)
         {
-            reads[i] = AbstractReadExecutor.getReadExecutor(commands.get(i), consistencyLevel, queryStartNanoTime);
+            reads[i] = AbstractReadExecutor.getReadExecutor(commands.get(i), consistencyLevel, requestTime);
 
             if (reads[i].hasLocalRead())
                 readMetrics.localRequests.mark();
@@ -2117,14 +2131,14 @@ public class StorageProxy implements StorageProxyMBean
         private final ReadCallback handler;
         private final boolean trackRepairedStatus;
 
-        public LocalReadRunnable(ReadCommand command, ReadCallback handler)
+        public LocalReadRunnable(ReadCommand command, ReadCallback handler, Dispatcher.RequestTime requestTime)
         {
-            this(command, handler, false);
+            this(command, handler, requestTime, false);
         }
 
-        public LocalReadRunnable(ReadCommand command, ReadCallback handler, boolean trackRepairedStatus)
+        public LocalReadRunnable(ReadCommand command, ReadCallback handler, Dispatcher.RequestTime requestTime, boolean trackRepairedStatus)
         {
-            super(Verb.READ_REQ);
+            super(Verb.READ_REQ, requestTime);
             this.command = command;
             this.handler = handler;
             this.trackRepairedStatus = trackRepairedStatus;
@@ -2137,7 +2151,8 @@ public class StorageProxy implements StorageProxyMBean
                 MessageParams.reset();
 
                 boolean readRejected = false;
-                command.setMonitoringTime(approxCreationTimeNanos, false, verb.expiresAfterNanos(), DatabaseDescriptor.getSlowQueryTimeout(NANOSECONDS));
+                long deadline = requestTime.computeDeadline(verb.expiresAfterNanos());
+                command.setMonitoringTime(requestTime.startedAtNanos(), false, deadline - requestTime.startedAtNanos(), DatabaseDescriptor.getSlowQueryTimeout(NANOSECONDS));
 
                 ReadResponse response;
                 try (ReadExecutionController controller = command.executionController(trackRepairedStatus);
@@ -2160,12 +2175,14 @@ public class StorageProxy implements StorageProxyMBean
                 }
                 else
                 {
-                    MessagingService.instance().metrics.recordSelfDroppedMessage(verb, MonotonicClock.Global.approxTime.now() - approxCreationTimeNanos, NANOSECONDS);
+                    // We track latency based on request processing time
+                    MessagingService.instance().metrics.recordSelfDroppedMessage(verb, MonotonicClock.Global.preciseTime.now() - requestTime.startedAtNanos(), NANOSECONDS);
                     handler.onFailure(FBUtilities.getBroadcastAddressAndPort(), RequestFailureReason.UNKNOWN);
                 }
 
                 if (!readRejected)
-                    MessagingService.instance().latencySubscribers.add(FBUtilities.getBroadcastAddressAndPort(), MonotonicClock.Global.approxTime.now() - approxCreationTimeNanos, NANOSECONDS);
+                    // We track latency based on request processing time
+                    MessagingService.instance().latencySubscribers.add(FBUtilities.getBroadcastAddressAndPort(), MonotonicClock.Global.preciseTime.now() - requestTime.startedAtNanos(), NANOSECONDS);
             }
             catch (Throwable t)
             {
@@ -2185,13 +2202,13 @@ public class StorageProxy implements StorageProxyMBean
         @Override
         public long creationTimeNanos()
         {
-            return approxCreationTimeNanos;
+            return requestTime.enqueuedAtNanos();
         }
 
         @Override
         public long startTimeNanos()
         {
-            return approxStartTimeNanos;
+            return requestTime.startedAtNanos();
         }
 
         @Override
@@ -2203,7 +2220,7 @@ public class StorageProxy implements StorageProxyMBean
 
     public static PartitionIterator getRangeSlice(PartitionRangeReadCommand command,
                                                   ConsistencyLevel consistencyLevel,
-                                                  long queryStartNanoTime)
+                                                  Dispatcher.RequestTime requestTime)
     {
         if (DatabaseDescriptor.getPartitionDenylistEnabled() && DatabaseDescriptor.getDenylistRangeReadsEnabled())
         {
@@ -2217,7 +2234,7 @@ public class StorageProxy implements StorageProxyMBean
                                                                 tokens));
             }
         }
-        return RangeCommands.partitions(command, consistencyLevel, queryStartNanoTime);
+        return RangeCommands.partitions(command, consistencyLevel, requestTime);
     }
 
     public Map<String, List<String>> getSchemaVersions()
@@ -2485,7 +2502,8 @@ public class StorageProxy implements StorageProxyMBean
         public void apply(IMutation mutation,
                           ReplicaPlan.ForWrite targets,
                           AbstractWriteResponseHandler<IMutation> responseHandler,
-                          String localDataCenter) throws OverloadedException;
+                          String localDataCenter,
+                          Dispatcher.RequestTime requestTime) throws OverloadedException;
     }
 
     /**
@@ -2493,9 +2511,9 @@ public class StorageProxy implements StorageProxyMBean
      */
     private static class ViewWriteMetricsWrapped extends BatchlogResponseHandler<IMutation>
     {
-        public ViewWriteMetricsWrapped(AbstractWriteResponseHandler<IMutation> writeHandler, int i, BatchlogCleanup cleanup, long queryStartNanoTime)
+        public ViewWriteMetricsWrapped(AbstractWriteResponseHandler<IMutation> writeHandler, int i, BatchlogCleanup cleanup, Dispatcher.RequestTime requestTime)
         {
-            super(writeHandler, i, cleanup, queryStartNanoTime);
+            super(writeHandler, i, cleanup, requestTime);
             viewWriteMetrics.viewReplicasAttempted.inc(candidateReplicaCount());
         }
 
@@ -2511,25 +2529,22 @@ public class StorageProxy implements StorageProxyMBean
      */
     private static abstract class DroppableRunnable implements Runnable
     {
-        protected final long approxCreationTimeNanos;
-        protected volatile long approxStartTimeNanos;
-        
         final Verb verb;
-
-        public DroppableRunnable(Verb verb)
+        final Dispatcher.RequestTime requestTime;
+        public DroppableRunnable(Verb verb, Dispatcher.RequestTime requestTime)
         {
-            this.approxCreationTimeNanos = MonotonicClock.Global.approxTime.now();
             this.verb = verb;
+            this.requestTime = requestTime;
         }
 
         public final void run()
         {
-            approxStartTimeNanos = MonotonicClock.Global.approxTime.now();
-            long expirationTimeNanos = verb.expiresAtNanos(approxCreationTimeNanos);
-            if (approxStartTimeNanos > expirationTimeNanos)
+            long nowNanos = MonotonicClock.Global.preciseTime.now();
+            long deadline = requestTime.computeDeadline(verb.expiresAfterNanos());
+            if (nowNanos > deadline)
             {
-                long timeTakenNanos = approxStartTimeNanos - approxCreationTimeNanos;
-                MessagingService.instance().metrics.recordSelfDroppedMessage(verb, timeTakenNanos, NANOSECONDS);
+                long elapsed = nowNanos - requestTime.startedAtNanos();
+                MessagingService.instance().metrics.recordSelfDroppedMessage(verb, elapsed, NANOSECONDS);
                 return;
             }
             try
@@ -2551,35 +2566,37 @@ public class StorageProxy implements StorageProxyMBean
      */
     private static abstract class LocalMutationRunnable implements RunnableDebuggableTask
     {
-        private final long approxCreationTimeNanos = MonotonicClock.Global.approxTime.now();
-        private volatile long approxStartTimeNanos;
-
         private final Replica localReplica;
+        private final Dispatcher.RequestTime requestTime;
 
-        LocalMutationRunnable(Replica localReplica)
+        LocalMutationRunnable(Replica localReplica, Dispatcher.RequestTime requestTime)
         {
             this.localReplica = localReplica;
+            this.requestTime = requestTime;
         }
 
         public final void run()
         {
             final Verb verb = verb();
-            approxStartTimeNanos = MonotonicClock.Global.approxTime.now();
-            long expirationTimeNanos = verb.expiresAtNanos(approxCreationTimeNanos);
-            
-            if (approxStartTimeNanos > expirationTimeNanos)
+            long now = MonotonicClock.Global.approxTime.now();
+            long deadline = requestTime.computeDeadline(verb.expiresAfterNanos());
+
+            if (now > deadline)
             {
-                long timeTakenNanos = approxStartTimeNanos - approxCreationTimeNanos;
+                long timeTakenNanos = now - startTimeNanos();
                 MessagingService.instance().metrics.recordSelfDroppedMessage(Verb.MUTATION_REQ, timeTakenNanos, NANOSECONDS);
 
-                HintRunnable runnable = new HintRunnable(EndpointsForToken.of(localReplica.range().right, localReplica))
+                if (requestTime.shouldSendHints())
                 {
-                    protected void runMayThrow() throws Exception
+                    HintRunnable runnable = new HintRunnable(EndpointsForToken.of(localReplica.range().right, localReplica))
                     {
-                        LocalMutationRunnable.this.runMayThrow();
-                    }
-                };
-                submitHint(runnable);
+                        protected void runMayThrow() throws Exception
+                        {
+                            LocalMutationRunnable.this.runMayThrow();
+                        }
+                    };
+                    submitHint(runnable);
+                }
                 return;
             }
 
@@ -2596,13 +2613,13 @@ public class StorageProxy implements StorageProxyMBean
         @Override
         public long creationTimeNanos()
         {
-            return approxCreationTimeNanos;
+            return requestTime.enqueuedAtNanos();
         }
 
         @Override
         public long startTimeNanos()
         {
-            return approxStartTimeNanos;
+            return requestTime.startedAtNanos();
         }
 
         @Override
@@ -2695,14 +2712,14 @@ public class StorageProxy implements StorageProxyMBean
         }
     }
 
-    public static Future<Void> submitHint(Mutation mutation, Replica target, AbstractWriteResponseHandler<IMutation> responseHandler)
+    public static void submitHint(Mutation mutation, Replica target, AbstractWriteResponseHandler<IMutation> responseHandler)
     {
-        return submitHint(mutation, EndpointsForToken.of(target.range().right, target), responseHandler);
+        submitHint(mutation, EndpointsForToken.of(target.range().right, target), responseHandler);
     }
 
-    public static Future<Void> submitHint(Mutation mutation,
-                                          EndpointsForToken targets,
-                                          AbstractWriteResponseHandler<IMutation> responseHandler)
+    private static void submitHint(Mutation mutation,
+                                   EndpointsForToken targets,
+                                   AbstractWriteResponseHandler<IMutation> responseHandler)
     {
         Replicas.assertFull(targets); // hints should not be written for transient replicas
         HintRunnable runnable = new HintRunnable(targets)
@@ -2731,7 +2748,7 @@ public class StorageProxy implements StorageProxyMBean
             }
         };
 
-        return submitHint(runnable);
+        submitHint(runnable);
     }
 
     private static Future<Void> submitHint(HintRunnable runnable)

src/java/org/apache/cassandra/service/StorageService.java

@@ -6966,4 +6966,83 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
     {
         DatabaseDescriptor.setMinTrackedPartitionTombstoneCount(value);
     }
+
+    @Override
+    public String getCQLStartTime()
+    {
+        return DatabaseDescriptor.getCQLStartTime().toString();
+    }
+
+    @Override
+    public void setCQLStartTime(String value)
+    {
+        DatabaseDescriptor.setCQLStartTime(Config.CQLStartTime.valueOf(value));
+    }
+
+    @Override
+    public double getNativeTransportQueueMaxItemAgeThreshold()
+    {
+        return DatabaseDescriptor.getNativeTransportQueueMaxItemAgeThreshold();
+    }
+
+    @Override
+    public void setNativeTransportQueueMaxItemAgeThreshold(double threshold)
+    {
+        DatabaseDescriptor.setNativeTransportMaxQueueItemAgeThreshold(threshold);
+    }
+
+    @Override
+    public long getNativeTransportMinBackoffOnQueueOverloadInMillis()
+    {
+        return DatabaseDescriptor.getNativeTransportMinBackoffOnQueueOverload(MILLISECONDS);
+    }
+
+    @Override
+    public long getNativeTransportMaxBackoffOnQueueOverloadInMillis()
+    {
+        return DatabaseDescriptor.getNativeTransportMaxBackoffOnQueueOverload(MILLISECONDS);
+    }
+
+    @Override
+    public void setNativeTransportBackoffOnQueueOverloadInMillis(long min, long max)
+    {
+        DatabaseDescriptor.setNativeTransportBackoffOnQueueOverload(min, max, MILLISECONDS);
+    }
+
+    @Override
+    public boolean getNativeTransportThrowOnOverload()
+    {
+        return DatabaseDescriptor.getNativeTransportThrowOnOverload();
+    }
+
+    @Override
+    public void setNativeTransportThrowOnOverload(boolean throwOnOverload)
+    {
+        DatabaseDescriptor.setNativeTransportThrowOnOverload(throwOnOverload);
+    }
+
+    @Override
+    public long getNativeTransportTimeoutMillis()
+    {
+        return DatabaseDescriptor.getNativeTransportTimeout(MILLISECONDS);
+    }
+
+    @Override
+    public void setNativeTransportTimeoutMillis(long deadlineMillis)
+    {
+        DatabaseDescriptor.setNativeTransportTimeout(deadlineMillis, MILLISECONDS);
+    }
+
+    @Override
+    public boolean getEnforceNativeDeadlineForHints()
+    {
+        return DatabaseDescriptor.getEnforceNativeDeadlineForHints();
+    }
+
+    @Override
+    public void setEnforceNativeDeadlineForHints(boolean value)
+    {
+        DatabaseDescriptor.setEnforceNativeDeadlineForHints(value);
+    }
+
 }

src/java/org/apache/cassandra/service/StorageServiceMBean.java

@@ -1050,4 +1050,23 @@ public interface StorageServiceMBean extends NotificationEmitter
     public void setMinTrackedPartitionSize(String value);
     public long getMinTrackedPartitionTombstoneCount();
     public void setMinTrackedPartitionTombstoneCount(long value);
+
+    String getCQLStartTime();
+    void setCQLStartTime(String value);
+
+    double getNativeTransportQueueMaxItemAgeThreshold();
+    void setNativeTransportQueueMaxItemAgeThreshold(double threshold);
+
+    long getNativeTransportMinBackoffOnQueueOverloadInMillis();
+    long getNativeTransportMaxBackoffOnQueueOverloadInMillis();
+    void setNativeTransportBackoffOnQueueOverloadInMillis(long min, long max);
+
+    boolean getNativeTransportThrowOnOverload();
+    void setNativeTransportThrowOnOverload(boolean throwOnOverload);
+
+    long getNativeTransportTimeoutMillis();
+    void setNativeTransportTimeoutMillis(long deadlineMillis);
+
+    boolean getEnforceNativeDeadlineForHints();
+    void setEnforceNativeDeadlineForHints(boolean value);
 }

src/java/org/apache/cassandra/service/WriteResponseHandler.java

@@ -27,6 +27,7 @@ import org.slf4j.LoggerFactory;
 
 import org.apache.cassandra.net.Message;
 import org.apache.cassandra.db.WriteType;
+import org.apache.cassandra.transport.Dispatcher;
 
 /**
  * Handles blocking writes for ONE, ANY, TWO, THREE, QUORUM, and ALL consistency levels.
@@ -43,15 +44,15 @@ public class WriteResponseHandler<T> extends AbstractWriteResponseHandler<T>
                                 Runnable callback,
                                 WriteType writeType,
                                 Supplier<Mutation> hintOnFailure,
-                                long queryStartNanoTime)
+                                Dispatcher.RequestTime requestTime)
     {
-        super(replicaPlan, callback, writeType, hintOnFailure, queryStartNanoTime);
+        super(replicaPlan, callback, writeType, hintOnFailure, requestTime);
         responses = blockFor();
     }
 
-    public WriteResponseHandler(ReplicaPlan.ForWrite replicaPlan, WriteType writeType, Supplier<Mutation> hintOnFailure, long queryStartNanoTime)
+    public WriteResponseHandler(ReplicaPlan.ForWrite replicaPlan, WriteType writeType, Supplier<Mutation> hintOnFailure, Dispatcher.RequestTime requestTime)
     {
-        this(replicaPlan, null, writeType, hintOnFailure, queryStartNanoTime);
+        this(replicaPlan, null, writeType, hintOnFailure, requestTime);
     }
 
     public void onResponse(Message<T> m)

src/java/org/apache/cassandra/service/pager/AbstractQueryPager.java

@@ -24,6 +24,7 @@ import org.apache.cassandra.db.filter.DataLimits;
 import org.apache.cassandra.db.transform.Transformation;
 import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.service.ClientState;
+import org.apache.cassandra.transport.Dispatcher;
 import org.apache.cassandra.transport.ProtocolVersion;
 
 abstract class AbstractQueryPager<T extends ReadQuery> implements QueryPager
@@ -59,7 +60,7 @@ abstract class AbstractQueryPager<T extends ReadQuery> implements QueryPager
         return query.executionController();
     }
 
-    public PartitionIterator fetchPage(int pageSize, ConsistencyLevel consistency, ClientState clientState, long queryStartNanoTime)
+    public PartitionIterator fetchPage(int pageSize, ConsistencyLevel consistency, ClientState clientState, Dispatcher.RequestTime requestTime)
     {
         if (isExhausted())
             return EmptyIterators.partition();
@@ -72,7 +73,7 @@ abstract class AbstractQueryPager<T extends ReadQuery> implements QueryPager
             exhausted = true;
             return EmptyIterators.partition();
         }
-        return Transformation.apply(readQuery.execute(consistency, clientState, queryStartNanoTime), pager);
+        return Transformation.apply(readQuery.execute(consistency, clientState, requestTime), pager);
     }
 
     public PartitionIterator fetchPageInternal(int pageSize, ReadExecutionController executionController)

src/java/org/apache/cassandra/service/pager/AggregationQueryPager.java

@@ -28,8 +28,7 @@ import org.apache.cassandra.db.partitions.PartitionIterator;
 import org.apache.cassandra.db.rows.Row;
 import org.apache.cassandra.db.rows.RowIterator;
 import org.apache.cassandra.service.ClientState;
-
-import static org.apache.cassandra.utils.Clock.Global.nanoTime;
+import org.apache.cassandra.transport.Dispatcher;
 
 /**
  * {@code QueryPager} that takes care of fetching the pages for aggregation queries.
@@ -55,12 +54,12 @@ public final class AggregationQueryPager implements QueryPager
     public PartitionIterator fetchPage(int pageSize,
                                        ConsistencyLevel consistency,
                                        ClientState clientState,
-                                       long queryStartNanoTime)
+                                       Dispatcher.RequestTime requestTime)
     {
         if (limits.isGroupByLimit())
-            return new GroupByPartitionIterator(pageSize, consistency, clientState, queryStartNanoTime);
+            return new GroupByPartitionIterator(pageSize, consistency, clientState, requestTime);
 
-        return new AggregationPartitionIterator(pageSize, consistency, clientState, queryStartNanoTime);
+        return new AggregationPartitionIterator(pageSize, consistency, clientState, requestTime);
     }
 
     @Override
@@ -73,9 +72,9 @@ public final class AggregationQueryPager implements QueryPager
     public PartitionIterator fetchPageInternal(int pageSize, ReadExecutionController executionController)
     {
         if (limits.isGroupByLimit())
-            return new GroupByPartitionIterator(pageSize, executionController, nanoTime());
+            return new GroupByPartitionIterator(pageSize, executionController, Dispatcher.RequestTime.forImmediateExecution());
 
-        return new AggregationPartitionIterator(pageSize, executionController, nanoTime());
+        return new AggregationPartitionIterator(pageSize, executionController, Dispatcher.RequestTime.forImmediateExecution());
     }
 
     @Override
@@ -155,34 +154,34 @@ public final class AggregationQueryPager implements QueryPager
          */
         private int initialMaxRemaining;
 
-        private long queryStartNanoTime;
+        private Dispatcher.RequestTime requestTime;
 
         public GroupByPartitionIterator(int pageSize,
-                                         ConsistencyLevel consistency,
-                                         ClientState clientState,
-                                        long queryStartNanoTime)
+                                        ConsistencyLevel consistency,
+                                        ClientState clientState,
+                                        Dispatcher.RequestTime requestTime)
         {
-            this(pageSize, consistency, clientState, null, queryStartNanoTime);
+            this(pageSize, consistency, clientState, null, requestTime);
         }
 
         public GroupByPartitionIterator(int pageSize,
                                         ReadExecutionController executionController,
-                                        long queryStartNanoTime)
+                                        Dispatcher.RequestTime requestTime)
        {
-           this(pageSize, null, null, executionController, queryStartNanoTime);
+           this(pageSize, null, null, executionController, requestTime);
        }
 
         private GroupByPartitionIterator(int pageSize,
                                          ConsistencyLevel consistency,
                                          ClientState clientState,
                                          ReadExecutionController executionController,
-                                         long queryStartNanoTime)
+                                         Dispatcher.RequestTime requestTime)
         {
             this.pageSize = handlePagingOff(pageSize);
             this.consistency = consistency;
             this.clientState = clientState;
             this.executionController = executionController;
-            this.queryStartNanoTime = queryStartNanoTime;
+            this.requestTime = requestTime;
         }
 
         private int handlePagingOff(int pageSize)
@@ -289,7 +288,7 @@ public final class AggregationQueryPager implements QueryPager
          */
         private final PartitionIterator fetchSubPage(int subPageSize)
         {
-            return consistency != null ? subPager.fetchPage(subPageSize, consistency, clientState, queryStartNanoTime)
+            return consistency != null ? subPager.fetchPage(subPageSize, consistency, clientState, requestTime)
                                        : subPager.fetchPageInternal(subPageSize, executionController);
         }
 
@@ -403,16 +402,16 @@ public final class AggregationQueryPager implements QueryPager
         public AggregationPartitionIterator(int pageSize,
                                             ConsistencyLevel consistency,
                                             ClientState clientState,
-                                            long queryStartNanoTime)
+                                            Dispatcher.RequestTime requestTime)
         {
-            super(pageSize, consistency, clientState, queryStartNanoTime);
+            super(pageSize, consistency, clientState, requestTime);
         }
 
         public AggregationPartitionIterator(int pageSize,
                                             ReadExecutionController executionController,
-                                            long queryStartNanoTime)
+                                            Dispatcher.RequestTime requestTime)
         {
-            super(pageSize, executionController, queryStartNanoTime);
+            super(pageSize, executionController, requestTime);
         }
 
         @Override

src/java/org/apache/cassandra/service/pager/MultiPartitionPager.java

@@ -17,6 +17,7 @@
  */
 package org.apache.cassandra.service.pager;
 
+import org.apache.cassandra.transport.Dispatcher;
 import org.apache.cassandra.transport.ProtocolVersion;
 import org.apache.cassandra.utils.AbstractIterator;
 
@@ -30,8 +31,6 @@ import org.apache.cassandra.exceptions.RequestValidationException;
 import org.apache.cassandra.exceptions.RequestExecutionException;
 import org.apache.cassandra.service.ClientState;
 
-import static org.apache.cassandra.utils.Clock.Global.nanoTime;
-
 /**
  * Pager over a list of SinglePartitionReadQuery.
  *
@@ -150,17 +149,18 @@ public class MultiPartitionPager<T extends SinglePartitionReadQuery> implements
     }
 
     @SuppressWarnings("resource") // iter closed via countingIter
-    public PartitionIterator fetchPage(int pageSize, ConsistencyLevel consistency, ClientState clientState, long queryStartNanoTime) throws RequestValidationException, RequestExecutionException
+    @Override
+    public PartitionIterator fetchPage(int pageSize, ConsistencyLevel consistency, ClientState clientState, Dispatcher.RequestTime requestTime) throws RequestValidationException, RequestExecutionException
     {
         int toQuery = Math.min(remaining, pageSize);
-        return new PagersIterator(toQuery, consistency, clientState, null, queryStartNanoTime);
+        return new PagersIterator(toQuery, consistency, clientState, null, requestTime);
     }
 
     @SuppressWarnings("resource") // iter closed via countingIter
     public PartitionIterator fetchPageInternal(int pageSize, ReadExecutionController executionController) throws RequestValidationException, RequestExecutionException
     {
         int toQuery = Math.min(remaining, pageSize);
-        return new PagersIterator(toQuery, null, null, executionController, nanoTime());
+        return new PagersIterator(toQuery, null, null, executionController, Dispatcher.RequestTime.forImmediateExecution());
     }
 
     private class PagersIterator extends AbstractIterator<RowIterator> implements PartitionIterator
@@ -168,7 +168,7 @@ public class MultiPartitionPager<T extends SinglePartitionReadQuery> implements
         private final int pageSize;
         private PartitionIterator result;
         private boolean closed;
-        private final long queryStartNanoTime;
+        private final Dispatcher.RequestTime requestTime;
 
         // For "normal" queries
         private final ConsistencyLevel consistency;
@@ -180,13 +180,13 @@ public class MultiPartitionPager<T extends SinglePartitionReadQuery> implements
         private int pagerMaxRemaining;
         private int counted;
 
-        public PagersIterator(int pageSize, ConsistencyLevel consistency, ClientState clientState, ReadExecutionController executionController, long queryStartNanoTime)
+        public PagersIterator(int pageSize, ConsistencyLevel consistency, ClientState clientState, ReadExecutionController executionController, Dispatcher.RequestTime requestTime)
         {
             this.pageSize = pageSize;
             this.consistency = consistency;
             this.clientState = clientState;
             this.executionController = executionController;
-            this.queryStartNanoTime = queryStartNanoTime;
+            this.requestTime = requestTime;
         }
 
         protected RowIterator computeNext()
@@ -215,7 +215,7 @@ public class MultiPartitionPager<T extends SinglePartitionReadQuery> implements
                 int toQuery = pageSize - counted;
                 result = consistency == null
                        ? pagers[current].fetchPageInternal(toQuery, executionController)
-                       : pagers[current].fetchPage(toQuery, consistency, clientState, queryStartNanoTime);
+                       : pagers[current].fetchPage(toQuery, consistency, clientState, requestTime);
             }
             return result.next();
         }

src/java/org/apache/cassandra/service/pager/QueryPager.java

@@ -25,6 +25,7 @@ import org.apache.cassandra.db.partitions.PartitionIterator;
 import org.apache.cassandra.exceptions.RequestExecutionException;
 import org.apache.cassandra.exceptions.RequestValidationException;
 import org.apache.cassandra.service.ClientState;
+import org.apache.cassandra.transport.Dispatcher;
 
 /**
  * Perform a query, paging it by page of a given size.
@@ -54,7 +55,7 @@ public interface QueryPager
             return ReadExecutionController.empty();
         }
 
-        public PartitionIterator fetchPage(int pageSize, ConsistencyLevel consistency, ClientState clientState, long queryStartNanoTime) throws RequestValidationException, RequestExecutionException
+        public PartitionIterator fetchPage(int pageSize, ConsistencyLevel consistency, ClientState clientState, Dispatcher.RequestTime requestTime) throws RequestValidationException, RequestExecutionException
         {
             return EmptyIterators.partition();
         }
@@ -94,7 +95,7 @@ public interface QueryPager
      * {@code consistency} is a serial consistency.
      * @return the page of result.
      */
-    public PartitionIterator fetchPage(int pageSize, ConsistencyLevel consistency, ClientState clientState, long queryStartNanoTime) throws RequestValidationException, RequestExecutionException;
+    public PartitionIterator fetchPage(int pageSize, ConsistencyLevel consistency, ClientState clientState, Dispatcher.RequestTime requestTime) throws RequestValidationException, RequestExecutionException;
 
     /**
      * Starts a new read operation.

src/java/org/apache/cassandra/service/paxos/Paxos.java

@@ -88,6 +88,7 @@ import org.apache.cassandra.service.reads.DataResolver;
 import org.apache.cassandra.service.reads.repair.NoopReadRepair;
 import org.apache.cassandra.service.paxos.cleanup.PaxosTableRepairs;
 import org.apache.cassandra.tracing.Tracing;
+import org.apache.cassandra.transport.Dispatcher;
 import org.apache.cassandra.triggers.TriggerExecutor;
 import org.apache.cassandra.utils.CassandraVersion;
 import org.apache.cassandra.utils.CollectionSerializer;
@@ -808,21 +809,20 @@ public class Paxos
         return read.rowIterator();
     }
 
-    public static PartitionIterator read(SinglePartitionReadCommand.Group group, ConsistencyLevel consistencyForConsensus)
+    public static PartitionIterator read(SinglePartitionReadCommand.Group group, ConsistencyLevel consistencyForConsensus, Dispatcher.RequestTime requestTime)
             throws InvalidRequestException, UnavailableException, ReadFailureException, ReadTimeoutException
     {
-        long start = nanoTime();
-        long deadline = start + DatabaseDescriptor.getReadRpcTimeout(NANOSECONDS);
-        return read(group, consistencyForConsensus, start, deadline);
+        long deadline = requestTime.computeDeadline(DatabaseDescriptor.getReadRpcTimeout(NANOSECONDS));
+        return read(group, consistencyForConsensus, requestTime, deadline);
     }
 
     public static PartitionIterator read(SinglePartitionReadCommand.Group group, ConsistencyLevel consistencyForConsensus, long deadline)
             throws InvalidRequestException, UnavailableException, ReadFailureException, ReadTimeoutException
     {
-        return read(group, consistencyForConsensus, nanoTime(), deadline);
+        return read(group, consistencyForConsensus, Dispatcher.RequestTime.forImmediateExecution(), deadline);
     }
 
-    private static PartitionIterator read(SinglePartitionReadCommand.Group group, ConsistencyLevel consistencyForConsensus, long start, long deadline)
+    private static PartitionIterator read(SinglePartitionReadCommand.Group group, ConsistencyLevel consistencyForConsensus, Dispatcher.RequestTime requestTime, long deadline)
             throws InvalidRequestException, UnavailableException, ReadFailureException, ReadTimeoutException
     {
         if (group.queries.size() > 1)
@@ -889,7 +889,9 @@ public class Paxos
         }
         finally
         {
-            long latency = nanoTime() - start;
+            // We track latency based on request processing time, since the amount of time that request spends in the queue
+            // is not a representative metric of replica performance.
+            long latency = nanoTime() - requestTime.startedAtNanos();
             readMetrics.addNano(latency);
             casReadMetrics.addNano(latency);
             readMetricsMap.get(consistencyForConsensus).addNano(latency);
@@ -1028,7 +1030,7 @@ public class Paxos
                     // round's proposal (if any).
                     PaxosPrepare.Success success = prepare.success();
 
-                    DataResolver<?, ?> resolver = new DataResolver(query, success.participants, NoopReadRepair.instance, query.creationTimeNanos());
+                    DataResolver<?, ?> resolver = new DataResolver(query, success.participants, NoopReadRepair.instance, new Dispatcher.RequestTime(query.creationTimeNanos()));
                     for (int i = 0 ; i < success.responses.size() ; ++i)
                         resolver.preprocess(success.responses.get(i));
 

src/java/org/apache/cassandra/service/paxos/v1/AbstractPaxosCallback.java

@@ -17,6 +17,8 @@
  */
 package org.apache.cassandra.service.paxos.v1;
 
+import org.apache.cassandra.transport.Dispatcher;
+import org.apache.cassandra.utils.Clock;
 import org.apache.cassandra.utils.concurrent.CountDownLatch;
 
 import org.apache.cassandra.config.DatabaseDescriptor;
@@ -27,7 +29,6 @@ import org.apache.cassandra.net.RequestCallback;
 import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException;
 
 import static java.util.concurrent.TimeUnit.NANOSECONDS;
-import static org.apache.cassandra.utils.Clock.Global.nanoTime;
 import static org.apache.cassandra.utils.concurrent.CountDownLatch.newCountDownLatch;
 
 public abstract class AbstractPaxosCallback<T> implements RequestCallback<T>
@@ -35,14 +36,14 @@ public abstract class AbstractPaxosCallback<T> implements RequestCallback<T>
     protected final CountDownLatch latch;
     protected final int targets;
     private final ConsistencyLevel consistency;
-    private final long queryStartNanoTime;
+    private final Dispatcher.RequestTime requestTime;
 
-    public AbstractPaxosCallback(int targets, ConsistencyLevel consistency, long queryStartNanoTime)
+    public AbstractPaxosCallback(int targets, ConsistencyLevel consistency, Dispatcher.RequestTime requestTime)
     {
         this.targets = targets;
         this.consistency = consistency;
         latch = newCountDownLatch(targets);
-        this.queryStartNanoTime = queryStartNanoTime;
+        this.requestTime = requestTime;
     }
 
     public int getResponseCount()
@@ -54,7 +55,9 @@ public abstract class AbstractPaxosCallback<T> implements RequestCallback<T>
     {
         try
         {
-            long timeout = DatabaseDescriptor.getWriteRpcTimeout(NANOSECONDS) - (nanoTime() - queryStartNanoTime);
+            long now = Clock.Global.nanoTime();
+            long timeout = requestTime.computeTimeout(now, DatabaseDescriptor.getWriteRpcTimeout(NANOSECONDS));
+
             if (!latch.await(timeout, NANOSECONDS))
                 throw new WriteTimeoutException(WriteType.CAS, consistency, getResponseCount(), targets);
         }

src/java/org/apache/cassandra/service/paxos/v1/PrepareCallback.java

@@ -34,6 +34,7 @@ import org.slf4j.LoggerFactory;
 import org.apache.cassandra.net.Message;
 import org.apache.cassandra.service.paxos.Commit;
 import org.apache.cassandra.service.paxos.PrepareResponse;
+import org.apache.cassandra.transport.Dispatcher;
 
 public class PrepareCallback extends AbstractPaxosCallback<PrepareResponse>
 {
@@ -45,9 +46,9 @@ public class PrepareCallback extends AbstractPaxosCallback<PrepareResponse>
 
     private final Map<InetAddressAndPort, Commit> commitsByReplica = new ConcurrentHashMap<>();
 
-    public PrepareCallback(DecoratedKey key, TableMetadata metadata, int targets, ConsistencyLevel consistency, long queryStartNanoTime)
+    public PrepareCallback(DecoratedKey key, TableMetadata metadata, int targets, ConsistencyLevel consistency, Dispatcher.RequestTime requestTime)
     {
-        super(targets, consistency, queryStartNanoTime);
+        super(targets, consistency, requestTime);
         // need to inject the right key in the empty commit so comparing with empty commits in the response works as expected
         mostRecentCommit = Commit.emptyCommit(key, metadata);
         mostRecentInProgressCommit = Commit.emptyCommit(key, metadata);

src/java/org/apache/cassandra/service/paxos/v1/ProposeCallback.java

@@ -26,6 +26,7 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import org.apache.cassandra.net.Message;
+import org.apache.cassandra.transport.Dispatcher;
 import org.apache.cassandra.utils.Nemesis;
 
 /**
@@ -49,9 +50,9 @@ public class ProposeCallback extends AbstractPaxosCallback<Boolean>
     private final int requiredAccepts;
     private final boolean failFast;
 
-    public ProposeCallback(int totalTargets, int requiredTargets, boolean failFast, ConsistencyLevel consistency, long queryStartNanoTime)
+    public ProposeCallback(int totalTargets, int requiredTargets, boolean failFast, ConsistencyLevel consistency, Dispatcher.RequestTime requestTime)
     {
-        super(totalTargets, consistency, queryStartNanoTime);
+        super(totalTargets, consistency, requestTime);
         this.requiredAccepts = requiredTargets;
         this.failFast = failFast;
     }

src/java/org/apache/cassandra/service/reads/AbstractReadExecutor.java

@@ -45,10 +45,13 @@ import org.apache.cassandra.service.StorageProxy.LocalReadRunnable;
 import org.apache.cassandra.service.reads.repair.ReadRepair;
 import org.apache.cassandra.tracing.TraceState;
 import org.apache.cassandra.tracing.Tracing;
+import org.apache.cassandra.transport.Dispatcher;
 import org.apache.cassandra.utils.FBUtilities;
 
 import static com.google.common.collect.Iterables.all;
 import static java.util.concurrent.TimeUnit.MICROSECONDS;
+import static java.util.concurrent.TimeUnit.NANOSECONDS;
+import static org.apache.cassandra.utils.MonotonicClock.Global.preciseTime;
 
 /**
  * Sends a read request to the replicas needed to satisfy a given ConsistencyLevel.
@@ -69,22 +72,23 @@ public abstract class AbstractReadExecutor
     protected final ReadCallback<EndpointsForToken, ReplicaPlan.ForTokenRead> handler;
     protected final TraceState traceState;
     protected final ColumnFamilyStore cfs;
-    protected final long queryStartNanoTime;
+    protected final Dispatcher.RequestTime requestTime;
+
     private   final int initialDataRequestCount;
     protected volatile PartitionIterator result = null;
 
-    AbstractReadExecutor(ColumnFamilyStore cfs, ReadCommand command, ReplicaPlan.ForTokenRead replicaPlan, int initialDataRequestCount, long queryStartNanoTime)
+    AbstractReadExecutor(ColumnFamilyStore cfs, ReadCommand command, ReplicaPlan.ForTokenRead replicaPlan, int initialDataRequestCount, Dispatcher.RequestTime requestTime)
     {
         this.command = command;
         this.replicaPlan = ReplicaPlan.shared(replicaPlan);
         this.initialDataRequestCount = initialDataRequestCount;
         // the ReadRepair and DigestResolver both need to see our updated
-        this.readRepair = ReadRepair.create(command, this.replicaPlan, queryStartNanoTime);
-        this.digestResolver = new DigestResolver<>(command, this.replicaPlan, queryStartNanoTime);
-        this.handler = new ReadCallback<>(digestResolver, command, this.replicaPlan, queryStartNanoTime);
+        this.readRepair = ReadRepair.create(command, this.replicaPlan, requestTime);
+        this.digestResolver = new DigestResolver<>(command, this.replicaPlan, requestTime);
+        this.handler = new ReadCallback<>(digestResolver, command, this.replicaPlan, requestTime);
         this.cfs = cfs;
         this.traceState = Tracing.instance.get();
-        this.queryStartNanoTime = queryStartNanoTime;
+        this.requestTime = requestTime;
 
 
         // Set the digest version (if we request some digests). This is the smallest version amongst all our target replicas since new nodes
@@ -147,7 +151,9 @@ public abstract class AbstractReadExecutor
                 traceState.trace("reading {} from {}", readCommand.isDigestQuery() ? "digest" : "data", endpoint);
 
             if (null == message)
-                message = readCommand.createMessage(false);
+            {
+                message = readCommand.createMessage(false, requestTime);
+            }
 
             MessagingService.instance().sendWithCallback(message, endpoint, handler);
         }
@@ -156,7 +162,7 @@ public abstract class AbstractReadExecutor
         if (hasLocalEndpoint)
         {
             logger.trace("reading {} locally", readCommand.isDigestQuery() ? "digest" : "data");
-            Stage.READ.maybeExecuteImmediately(new LocalReadRunnable(readCommand, handler));
+            Stage.READ.maybeExecuteImmediately(new LocalReadRunnable(readCommand, handler, requestTime));
         }
     }
 
@@ -181,7 +187,7 @@ public abstract class AbstractReadExecutor
     /**
      * @return an executor appropriate for the configured speculative read policy
      */
-    public static AbstractReadExecutor getReadExecutor(SinglePartitionReadCommand command, ConsistencyLevel consistencyLevel, long queryStartNanoTime) throws UnavailableException
+    public static AbstractReadExecutor getReadExecutor(SinglePartitionReadCommand command, ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime) throws UnavailableException
     {
         Keyspace keyspace = Keyspace.open(command.metadata().keyspace);
         ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(command.metadata().id);
@@ -192,20 +198,20 @@ public abstract class AbstractReadExecutor
         // Speculative retry is disabled *OR*
         // 11980: Disable speculative retry if using EACH_QUORUM in order to prevent miscounting DC responses
         if (retry.equals(NeverSpeculativeRetryPolicy.INSTANCE) || consistencyLevel == ConsistencyLevel.EACH_QUORUM)
-            return new NeverSpeculatingReadExecutor(cfs, command, replicaPlan, queryStartNanoTime, false);
+            return new NeverSpeculatingReadExecutor(cfs, command, replicaPlan, requestTime, false);
 
         // There are simply no extra replicas to speculate.
         // Handle this separately so it can record failed attempts to speculate due to lack of replicas
         if (replicaPlan.contacts().size() == replicaPlan.readCandidates().size())
         {
             boolean recordFailedSpeculation = consistencyLevel != ConsistencyLevel.ALL;
-            return new NeverSpeculatingReadExecutor(cfs, command, replicaPlan, queryStartNanoTime, recordFailedSpeculation);
+            return new NeverSpeculatingReadExecutor(cfs, command, replicaPlan, requestTime, recordFailedSpeculation);
         }
 
         if (retry.equals(AlwaysSpeculativeRetryPolicy.INSTANCE))
-            return new AlwaysSpeculatingReadExecutor(cfs, command, replicaPlan, queryStartNanoTime);
+            return new AlwaysSpeculatingReadExecutor(cfs, command, replicaPlan, requestTime);
         else // PERCENTILE or CUSTOM.
-            return new SpeculatingReadExecutor(cfs, command, replicaPlan, queryStartNanoTime);
+            return new SpeculatingReadExecutor(cfs, command, replicaPlan, requestTime);
     }
 
     public boolean hasLocalRead()
@@ -219,17 +225,27 @@ public abstract class AbstractReadExecutor
      */
     boolean shouldSpeculateAndMaybeWait()
     {
+        long now = preciseTime.now();
+        long sampleLatencyNanos = MICROSECONDS.toNanos(cfs.sampleReadLatencyMicros);
         // no latency information, or we're overloaded
-        if (cfs.sampleReadLatencyMicros > command.getTimeout(MICROSECONDS))
+        if (sampleLatencyNanos > command.getTimeout(NANOSECONDS))
         {
-            if (logger.isTraceEnabled())
-                logger.trace("Decided not to speculate as {} > {}", cfs.sampleReadLatencyMicros, command.getTimeout(MICROSECONDS));
+            logger.trace("Decided not to speculate as {}ns > {}ns", sampleLatencyNanos, command.getTimeout(NANOSECONDS));
             return false;
         }
 
+        if (now + sampleLatencyNanos > requestTime.clientDeadline())
+        {
+            logger.trace("Decided not to speculate as native transport timeout will be reached before speculating");
+            return false;
+        }
+
+        // We track latency based on request processing time, since the amount of time that request spends in the queue
+        // is not a representative metric of replica performance.
         if (logger.isTraceEnabled())
-            logger.trace("Awaiting {} microseconds before speculating", cfs.sampleReadLatencyMicros);
-        return !handler.await(cfs.sampleReadLatencyMicros, MICROSECONDS);
+            logger.trace("Awaiting {}ns before speculating", sampleLatencyNanos);
+
+        return !handler.awaitUntil(requestTime.startedAtNanos() + sampleLatencyNanos);
     }
 
     ReplicaPlan.ForTokenRead replicaPlan()
@@ -248,9 +264,13 @@ public abstract class AbstractReadExecutor
          */
         private final boolean logFailedSpeculation;
 
-        public NeverSpeculatingReadExecutor(ColumnFamilyStore cfs, ReadCommand command, ReplicaPlan.ForTokenRead replicaPlan, long queryStartNanoTime, boolean logFailedSpeculation)
+        public NeverSpeculatingReadExecutor(ColumnFamilyStore cfs,
+                                            ReadCommand command,
+                                            ReplicaPlan.ForTokenRead replicaPlan,
+                                            Dispatcher.RequestTime requestTime,
+                                            boolean logFailedSpeculation)
         {
-            super(cfs, command, replicaPlan, 1, queryStartNanoTime);
+            super(cfs, command, replicaPlan, 1, requestTime);
             this.logFailedSpeculation = logFailedSpeculation;
         }
 
@@ -270,12 +290,12 @@ public abstract class AbstractReadExecutor
         public SpeculatingReadExecutor(ColumnFamilyStore cfs,
                                        ReadCommand command,
                                        ReplicaPlan.ForTokenRead replicaPlan,
-                                       long queryStartNanoTime)
+                                       Dispatcher.RequestTime requestTime)
         {
             // We're hitting additional targets for read repair (??).  Since our "extra" replica is the least-
             // preferred by the snitch, we do an extra data read to start with against a replica more
             // likely to respond; better to let RR fail than the entire query.
-            super(cfs, command, replicaPlan, replicaPlan.readQuorum() < replicaPlan.contacts().size() ? 2 : 1, queryStartNanoTime);
+            super(cfs, command, replicaPlan, replicaPlan.readQuorum() < replicaPlan.contacts().size() ? 2 : 1, requestTime);
         }
 
         public void maybeTryAdditionalReplicas()
@@ -321,7 +341,8 @@ public abstract class AbstractReadExecutor
                 if (traceState != null)
                     traceState.trace("speculating read retry on {}", extraReplica);
                 logger.trace("speculating read retry on {}", extraReplica);
-                MessagingService.instance().sendWithCallback(retryCommand.createMessage(false), extraReplica.endpoint(), handler);
+
+                MessagingService.instance().sendWithCallback(retryCommand.createMessage(false, requestTime), extraReplica.endpoint(), handler);
             }
         }
 
@@ -340,11 +361,11 @@ public abstract class AbstractReadExecutor
         public AlwaysSpeculatingReadExecutor(ColumnFamilyStore cfs,
                                              ReadCommand command,
                                              ReplicaPlan.ForTokenRead replicaPlan,
-                                             long queryStartNanoTime)
+                                             Dispatcher.RequestTime requestTime)
         {
             // presumably, we speculate an extra data request here in case it is our data request that fails to respond,
             // and there are no more nodes to consult
-            super(cfs, command, replicaPlan, replicaPlan.contacts().size() > 1 ? 2 : 1, queryStartNanoTime);
+            super(cfs, command, replicaPlan, replicaPlan.contacts().size() > 1 ? 2 : 1, requestTime);
         }
 
         public void maybeTryAdditionalReplicas()

src/java/org/apache/cassandra/service/reads/DataResolver.java

@@ -57,6 +57,7 @@ import org.apache.cassandra.service.reads.repair.NoopReadRepair;
 import org.apache.cassandra.service.reads.repair.ReadRepair;
 import org.apache.cassandra.service.reads.repair.RepairedDataTracker;
 import org.apache.cassandra.service.reads.repair.RepairedDataVerifier;
+import org.apache.cassandra.transport.Dispatcher;
 
 import static com.google.common.collect.Iterables.*;
 
@@ -66,14 +67,14 @@ public class DataResolver<E extends Endpoints<E>, P extends ReplicaPlan.ForRead<
     private final ReadRepair<E, P> readRepair;
     private final boolean trackRepairedStatus;
 
-    public DataResolver(ReadCommand command, Supplier<? extends P> replicaPlan, ReadRepair<E, P> readRepair, long queryStartNanoTime)
+    public DataResolver(ReadCommand command, Supplier<? extends P> replicaPlan, ReadRepair<E, P> readRepair, Dispatcher.RequestTime requestTime)
     {
-        this(command, replicaPlan, readRepair, queryStartNanoTime, false);
+        this(command, replicaPlan, readRepair, requestTime, false);
     }
 
-    public DataResolver(ReadCommand command, Supplier<? extends P> replicaPlan, ReadRepair<E, P> readRepair, long queryStartNanoTime, boolean trackRepairedStatus)
+    public DataResolver(ReadCommand command, Supplier<? extends P> replicaPlan, ReadRepair<E, P> readRepair, Dispatcher.RequestTime requestTime, boolean trackRepairedStatus)
     {
-        super(command, replicaPlan, queryStartNanoTime);
+        super(command, replicaPlan, requestTime);
         this.enforceStrictLiveness = command.metadata().enforceStrictLiveness();
         this.readRepair = readRepair;
         this.trackRepairedStatus = trackRepairedStatus;
@@ -200,7 +201,7 @@ public class DataResolver<E extends Endpoints<E>, P extends ReplicaPlan.ForRead<
                                             originalResponse,
                                             command,
                                             context.mergedResultCounter,
-                                            queryStartNanoTime,
+                                            requestTime,
                                             enforceStrictLiveness)
                : originalResponse;
     }
@@ -246,7 +247,7 @@ public class DataResolver<E extends Endpoints<E>, P extends ReplicaPlan.ForRead<
         ReplicaFilteringProtection<E> rfp = new ReplicaFilteringProtection<>(replicaPlan().keyspace(),
                                                                              command,
                                                                              replicaPlan().consistencyLevel(),
-                                                                             queryStartNanoTime,
+                                                                             requestTime,
                                                                              firstPhaseContext.replicas,
                                                                              DatabaseDescriptor.getCachedReplicaRowsWarnThreshold(),
                                                                              DatabaseDescriptor.getCachedReplicaRowsFailThreshold());

src/java/org/apache/cassandra/service/reads/DigestResolver.java

@@ -35,6 +35,7 @@ import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.locator.ReplicaPlan;
 import org.apache.cassandra.net.Message;
 import org.apache.cassandra.service.reads.repair.NoopReadRepair;
+import org.apache.cassandra.transport.Dispatcher;
 import org.apache.cassandra.utils.ByteBufferUtil;
 
 import static com.google.common.collect.Iterables.any;
@@ -44,9 +45,9 @@ public class DigestResolver<E extends Endpoints<E>, P extends ReplicaPlan.ForRea
 {
     private volatile Message<ReadResponse> dataResponse;
 
-    public DigestResolver(ReadCommand command, ReplicaPlan.Shared<E, P> replicaPlan, long queryStartNanoTime)
+    public DigestResolver(ReadCommand command, ReplicaPlan.Shared<E, P> replicaPlan, Dispatcher.RequestTime requestTime)
     {
-        super(command, replicaPlan, queryStartNanoTime);
+        super(command, replicaPlan, requestTime);
         Preconditions.checkArgument(command instanceof SinglePartitionReadCommand,
                                     "DigestResolver can only be used with SinglePartitionReadCommand commands");
     }
@@ -86,7 +87,7 @@ public class DigestResolver<E extends Endpoints<E>, P extends ReplicaPlan.ForRea
             // This path can be triggered only if we've got responses from full replicas and they match, but
             // transient replica response still contains data, which needs to be reconciled.
             DataResolver<E, P> dataResolver
-                    = new DataResolver<>(command, replicaPlan, NoopReadRepair.instance, queryStartNanoTime);
+                    = new DataResolver<>(command, replicaPlan, NoopReadRepair.instance, requestTime);
 
             dataResolver.preprocess(dataResponse);
             // Reconcile with transient replicas

src/java/org/apache/cassandra/service/reads/ReadCallback.java

@@ -26,6 +26,7 @@ import java.util.concurrent.atomic.AtomicReferenceFieldUpdater;
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.db.MessageParams;
 import org.apache.cassandra.locator.ReplicaPlan;
+import org.apache.cassandra.transport.Dispatcher;
 import org.apache.cassandra.utils.concurrent.Condition;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -49,10 +50,8 @@ import org.apache.cassandra.tracing.Tracing;
 import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException;
 
 import static java.util.concurrent.TimeUnit.MILLISECONDS;
-import static java.util.concurrent.TimeUnit.NANOSECONDS;
 import static java.util.concurrent.atomic.AtomicIntegerFieldUpdater.newUpdater;
 import static org.apache.cassandra.tracing.Tracing.isTracing;
-import static org.apache.cassandra.utils.Clock.Global.nanoTime;
 import static org.apache.cassandra.utils.concurrent.Condition.newOneTimeCondition;
 
 public class ReadCallback<E extends Endpoints<E>, P extends ReplicaPlan.ForRead<E, P>> implements RequestCallback<ReadResponse>
@@ -61,7 +60,7 @@ public class ReadCallback<E extends Endpoints<E>, P extends ReplicaPlan.ForRead<
 
     public final ResponseResolver<E, P> resolver;
     final Condition condition = newOneTimeCondition();
-    private final long queryStartNanoTime;
+    private final Dispatcher.RequestTime requestTime;
     final int blockFor; // TODO: move to replica plan as well?
     // this uses a plain reference, but is initialised before handoff to any other threads; the later updates
     // may not be visible to the threads immediately, but ReplicaPlan only contains final fields, so they will never see an uninitialised object
@@ -75,11 +74,11 @@ public class ReadCallback<E extends Endpoints<E>, P extends ReplicaPlan.ForRead<
     private static final AtomicReferenceFieldUpdater<ReadCallback, WarningContext> warningsUpdater
         = AtomicReferenceFieldUpdater.newUpdater(ReadCallback.class, WarningContext.class, "warningContext");
 
-    public ReadCallback(ResponseResolver<E, P> resolver, ReadCommand command, ReplicaPlan.Shared<E, P> replicaPlan, long queryStartNanoTime)
+    public ReadCallback(ResponseResolver<E, P> resolver, ReadCommand command, ReplicaPlan.Shared<E, P> replicaPlan, Dispatcher.RequestTime requestTime)
     {
         this.command = command;
         this.resolver = resolver;
-        this.queryStartNanoTime = queryStartNanoTime;
+        this.requestTime = requestTime;
         this.replicaPlan = replicaPlan;
         this.blockFor = replicaPlan.get().readQuorum();
         this.failureReasonByEndpoint = new ConcurrentHashMap<>();
@@ -95,12 +94,28 @@ public class ReadCallback<E extends Endpoints<E>, P extends ReplicaPlan.ForRead<
         return replicaPlan.get();
     }
 
-    public boolean await(long timePastStart, TimeUnit unit)
+    public boolean await(long commandTimeout, TimeUnit unit)
+    {
+        return awaitUntil(requestTime.computeDeadline(unit.toNanos(commandTimeout)));
+    }
+
+    /**
+     * In case of speculation, we want to time out the request immediately if we have _also_ hit a deadline.
+     *
+     * For example, we have a read timeout of 10s, 99% latency of 5 seconds, and time base is QUEUE:
+     *   * Request has spent 3 seconds in the queue. Here, we will wait for 2 seconds and try to speculate
+     *   * Request has spent 10 seconds in the queue. Here, we will wait for 0 seconds and try to speculate
+     *
+     *  If the time base is REQUEST:
+     *   * Request has spent 10 seconds in the queue. Here, we will only wait 2 seconds and then try to speculate
+     *
+     * We should _not_ speculate in all these cases, since by that time we are already past request deadline.
+     */
+    public boolean awaitUntil(long deadline)
     {
-        long time = unit.toNanos(timePastStart) - (nanoTime() - queryStartNanoTime);
         try
         {
-            return condition.await(time, NANOSECONDS);
+            return condition.awaitUntil(deadline);
         }
         catch (InterruptedException e)
         {

src/java/org/apache/cassandra/service/reads/ReplicaFilteringProtection.java

@@ -71,6 +71,7 @@ import org.apache.cassandra.service.ClientWarn;
 import org.apache.cassandra.service.StorageProxy;
 import org.apache.cassandra.service.reads.repair.NoopReadRepair;
 import org.apache.cassandra.tracing.Tracing;
+import org.apache.cassandra.transport.Dispatcher;
 import org.apache.cassandra.utils.NoSpamLogger;
 import org.apache.cassandra.utils.btree.BTreeSet;
 
@@ -96,7 +97,7 @@ public class ReplicaFilteringProtection<E extends Endpoints<E>>
     private final Keyspace keyspace;
     private final ReadCommand command;
     private final ConsistencyLevel consistency;
-    private final long queryStartNanoTime;
+    private final Dispatcher.RequestTime requestTime;
     private final E sources;
     private final TableMetrics tableMetrics;
 
@@ -117,7 +118,7 @@ public class ReplicaFilteringProtection<E extends Endpoints<E>>
     ReplicaFilteringProtection(Keyspace keyspace,
                                ReadCommand command,
                                ConsistencyLevel consistency,
-                               long queryStartNanoTime,
+                               Dispatcher.RequestTime requestTime,
                                E sources,
                                int cachedRowsWarnThreshold,
                                int cachedRowsFailThreshold)
@@ -125,7 +126,7 @@ public class ReplicaFilteringProtection<E extends Endpoints<E>>
         this.keyspace = keyspace;
         this.command = command;
         this.consistency = consistency;
-        this.queryStartNanoTime = queryStartNanoTime;
+        this.requestTime = requestTime;
         this.sources = sources;
         this.originalPartitions = new ArrayList<>(sources.size());
 
@@ -144,19 +145,19 @@ public class ReplicaFilteringProtection<E extends Endpoints<E>>
     {
         @SuppressWarnings("unchecked")
         DataResolver<EndpointsForToken, ReplicaPlan.ForTokenRead> resolver =
-            new DataResolver<>(cmd, replicaPlan, (NoopReadRepair<EndpointsForToken, ReplicaPlan.ForTokenRead>) NoopReadRepair.instance, queryStartNanoTime);
+            new DataResolver<>(cmd, replicaPlan, (NoopReadRepair<EndpointsForToken, ReplicaPlan.ForTokenRead>) NoopReadRepair.instance, requestTime);
 
-        ReadCallback<EndpointsForToken, ReplicaPlan.ForTokenRead> handler = new ReadCallback<>(resolver, cmd, replicaPlan, queryStartNanoTime);
+        ReadCallback<EndpointsForToken, ReplicaPlan.ForTokenRead> handler = new ReadCallback<>(resolver, cmd, replicaPlan, requestTime);
 
         if (source.isSelf())
         {
-            Stage.READ.maybeExecuteImmediately(new StorageProxy.LocalReadRunnable(cmd, handler));
+            Stage.READ.maybeExecuteImmediately(new StorageProxy.LocalReadRunnable(cmd, handler, requestTime));
         }
         else
         {
             if (source.isTransient())
                 cmd = cmd.copyAsTransientQuery(source);
-            MessagingService.instance().sendWithCallback(cmd.createMessage(false), source.endpoint(), handler);
+            MessagingService.instance().sendWithCallback(cmd.createMessage(false, requestTime), source.endpoint(), handler);
         }
 
         // We don't call handler.get() because we want to preserve tombstones since we're still in the middle of merging node results.

src/java/org/apache/cassandra/service/reads/ResponseResolver.java

@@ -27,6 +27,7 @@ import org.apache.cassandra.db.ReadResponse;
 import org.apache.cassandra.locator.Endpoints;
 import org.apache.cassandra.locator.ReplicaPlan;
 import org.apache.cassandra.net.Message;
+import org.apache.cassandra.transport.Dispatcher;
 import org.apache.cassandra.utils.concurrent.Accumulator;
 
 public abstract class ResponseResolver<E extends Endpoints<E>, P extends ReplicaPlan.ForRead<E, P>>
@@ -39,14 +40,14 @@ public abstract class ResponseResolver<E extends Endpoints<E>, P extends Replica
 
     // Accumulator gives us non-blocking thread-safety with optimal algorithmic constraints
     protected final Accumulator<Message<ReadResponse>> responses;
-    protected final long queryStartNanoTime;
+    protected final Dispatcher.RequestTime requestTime;
 
-    public ResponseResolver(ReadCommand command, Supplier<? extends P> replicaPlan, long queryStartNanoTime)
+    public ResponseResolver(ReadCommand command, Supplier<? extends P> replicaPlan, Dispatcher.RequestTime requestTime)
     {
         this.command = command;
         this.replicaPlan = replicaPlan;
         this.responses = new Accumulator<>(replicaPlan.get().readCandidates().size());
-        this.queryStartNanoTime = queryStartNanoTime;
+        this.requestTime = requestTime;
     }
 
     protected P replicaPlan()

src/java/org/apache/cassandra/service/reads/ShortReadPartitionsProtection.java

@@ -46,6 +46,7 @@ import org.apache.cassandra.net.MessagingService;
 import org.apache.cassandra.service.reads.repair.NoopReadRepair;
 import org.apache.cassandra.service.StorageProxy;
 import org.apache.cassandra.tracing.Tracing;
+import org.apache.cassandra.transport.Dispatcher;
 
 public class ShortReadPartitionsProtection extends Transformation<UnfilteredRowIterator> implements MorePartitions<UnfilteredPartitionIterator>
 {
@@ -62,21 +63,21 @@ public class ShortReadPartitionsProtection extends Transformation<UnfilteredRowI
 
     private boolean partitionsFetched; // whether we've seen any new partitions since iteration start or last moreContents() call
 
-    private final long queryStartNanoTime;
+    private final Dispatcher.RequestTime requestTime;
 
     public ShortReadPartitionsProtection(ReadCommand command,
                                          Replica source,
                                          Runnable preFetchCallback,
                                          DataLimits.Counter singleResultCounter,
                                          DataLimits.Counter mergedResultCounter,
-                                         long queryStartNanoTime)
+                                         Dispatcher.RequestTime requestTime)
     {
         this.command = command;
         this.source = source;
         this.preFetchCallback = preFetchCallback;
         this.singleResultCounter = singleResultCounter;
         this.mergedResultCounter = mergedResultCounter;
-        this.queryStartNanoTime = queryStartNanoTime;
+        this.requestTime = requestTime;
     }
 
     @Override
@@ -177,18 +178,18 @@ public class ShortReadPartitionsProtection extends Transformation<UnfilteredRowI
     private <E extends Endpoints<E>, P extends ReplicaPlan.ForRead<E, P>>
     UnfilteredPartitionIterator executeReadCommand(ReadCommand cmd, ReplicaPlan.Shared<E, P> replicaPlan)
     {
-        DataResolver<E, P> resolver = new DataResolver<>(cmd, replicaPlan, (NoopReadRepair<E, P>)NoopReadRepair.instance, queryStartNanoTime);
-        ReadCallback<E, P> handler = new ReadCallback<>(resolver, cmd, replicaPlan, queryStartNanoTime);
+        DataResolver<E, P> resolver = new DataResolver<>(cmd, replicaPlan, (NoopReadRepair<E, P>)NoopReadRepair.instance, requestTime);
+        ReadCallback<E, P> handler = new ReadCallback<>(resolver, cmd, replicaPlan, requestTime);
 
         if (source.isSelf())
         {
-            Stage.READ.maybeExecuteImmediately(new StorageProxy.LocalReadRunnable(cmd, handler));
+            Stage.READ.maybeExecuteImmediately(new StorageProxy.LocalReadRunnable(cmd, handler, requestTime));
         }
         else
         {
             if (source.isTransient())
                 cmd = cmd.copyAsTransientQuery(source);
-            MessagingService.instance().sendWithCallback(cmd.createMessage(false), source.endpoint(), handler);
+            MessagingService.instance().sendWithCallback(cmd.createMessage(false, requestTime), source.endpoint(), handler);
         }
 
         // We don't call handler.get() because we want to preserve tombstones since we're still in the middle of merging node results.

src/java/org/apache/cassandra/service/reads/ShortReadProtection.java

@@ -25,6 +25,7 @@ import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
 import org.apache.cassandra.db.transform.MorePartitions;
 import org.apache.cassandra.db.transform.Transformation;
 import org.apache.cassandra.locator.Replica;
+import org.apache.cassandra.transport.Dispatcher;
 
 /**
  * We have a potential short read if the result from a given node contains the requested number of rows
@@ -44,7 +45,7 @@ public class ShortReadProtection
                                                      UnfilteredPartitionIterator partitions,
                                                      ReadCommand command,
                                                      DataLimits.Counter mergedResultCounter,
-                                                     long queryStartNanoTime,
+                                                     Dispatcher.RequestTime requestTime,
                                                      boolean enforceStrictLiveness)
     {
         DataLimits.Counter singleResultCounter = command.limits().newCounter(command.nowInSec(),
@@ -57,7 +58,7 @@ public class ShortReadProtection
                                                                                      preFetchCallback,
                                                                                      singleResultCounter,
                                                                                      mergedResultCounter,
-                                                                                     queryStartNanoTime);
+                                                                                     requestTime);
 
         /*
          * The order of extention and transformations is important here. Extending with more partitions has to happen

src/java/org/apache/cassandra/service/reads/range/RangeCommandIterator.java

@@ -50,6 +50,7 @@ import org.apache.cassandra.service.reads.DataResolver;
 import org.apache.cassandra.service.reads.ReadCallback;
 import org.apache.cassandra.service.reads.repair.ReadRepair;
 import org.apache.cassandra.tracing.Tracing;
+import org.apache.cassandra.transport.Dispatcher;
 import org.apache.cassandra.utils.AbstractIterator;
 import org.apache.cassandra.utils.CloseableIterator;
 
@@ -67,8 +68,7 @@ public class RangeCommandIterator extends AbstractIterator<RowIterator> implemen
     private final PartitionRangeReadCommand command;
     private final boolean enforceStrictLiveness;
 
-    private final long startTime;
-    private final long queryStartNanoTime;
+    private final Dispatcher.RequestTime requestTime;
     private DataLimits.Counter counter;
     private PartitionIterator sentQueryIterator;
 
@@ -85,16 +85,14 @@ public class RangeCommandIterator extends AbstractIterator<RowIterator> implemen
                          int concurrencyFactor,
                          int maxConcurrencyFactor,
                          int totalRangeCount,
-                         long queryStartNanoTime)
+                         Dispatcher.RequestTime requestTime)
     {
         this.replicaPlans = replicaPlans;
         this.command = command;
         this.concurrencyFactor = concurrencyFactor;
         this.maxConcurrencyFactor = maxConcurrencyFactor;
         this.totalRangeCount = totalRangeCount;
-        this.queryStartNanoTime = queryStartNanoTime;
-
-        startTime = nanoTime();
+        this.requestTime = requestTime;
         enforceStrictLiveness = command.metadata().enforceStrictLiveness();
     }
 
@@ -196,15 +194,15 @@ public class RangeCommandIterator extends AbstractIterator<RowIterator> implemen
 
         ReplicaPlan.SharedForRangeRead sharedReplicaPlan = ReplicaPlan.shared(replicaPlan);
         ReadRepair<EndpointsForRange, ReplicaPlan.ForRangeRead> readRepair =
-                ReadRepair.create(command, sharedReplicaPlan, queryStartNanoTime);
+                ReadRepair.create(command, sharedReplicaPlan, requestTime);
         DataResolver<EndpointsForRange, ReplicaPlan.ForRangeRead> resolver =
-                new DataResolver<>(rangeCommand, sharedReplicaPlan, readRepair, queryStartNanoTime, trackRepairedStatus);
+                new DataResolver<>(rangeCommand, sharedReplicaPlan, readRepair, requestTime, trackRepairedStatus);
         ReadCallback<EndpointsForRange, ReplicaPlan.ForRangeRead> handler =
-                new ReadCallback<>(resolver, rangeCommand, sharedReplicaPlan, queryStartNanoTime);
+                new ReadCallback<>(resolver, rangeCommand, sharedReplicaPlan, requestTime);
 
         if (replicaPlan.contacts().size() == 1 && replicaPlan.contacts().get(0).isSelf())
         {
-            Stage.READ.execute(new StorageProxy.LocalReadRunnable(rangeCommand, handler, trackRepairedStatus));
+            Stage.READ.execute(new StorageProxy.LocalReadRunnable(rangeCommand, handler, requestTime, trackRepairedStatus));
         }
         else
         {
@@ -212,7 +210,7 @@ public class RangeCommandIterator extends AbstractIterator<RowIterator> implemen
             {
                 Tracing.trace("Enqueuing request to {}", replica);
                 ReadCommand command = replica.isFull() ? rangeCommand : rangeCommand.copyAsTransientQuery(replica);
-                Message<ReadCommand> message = command.createMessage(trackRepairedStatus && replica.isFull());
+                Message<ReadCommand> message = command.createMessage(trackRepairedStatus && replica.isFull(), requestTime);
                 MessagingService.instance().sendWithCallback(message, replica.endpoint(), handler);
             }
         }
@@ -268,7 +266,9 @@ public class RangeCommandIterator extends AbstractIterator<RowIterator> implemen
         }
         finally
         {
-            long latency = nanoTime() - startTime;
+            // We track latency based on request processing time, since the amount of time that request spends in the queue
+            // is not a representative metric of replica performance.
+            long latency = nanoTime() - requestTime.startedAtNanos();
             rangeMetrics.addNano(latency);
             Keyspace.openAndGetStore(command.metadata()).metric.coordinatorScanLatency.update(latency, TimeUnit.NANOSECONDS);
         }

src/java/org/apache/cassandra/service/reads/range/RangeCommands.java

@@ -34,6 +34,7 @@ import org.apache.cassandra.index.Index;
 import org.apache.cassandra.locator.ReplicaPlans;
 import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.tracing.Tracing;
+import org.apache.cassandra.transport.Dispatcher;
 import org.apache.cassandra.utils.FBUtilities;
 
 public class RangeCommands
@@ -55,10 +56,10 @@ public class RangeCommands
     @SuppressWarnings("resource") // created iterators will be closed in CQL layer through the chain of transformations
     public static PartitionIterator partitions(PartitionRangeReadCommand command,
                                                ConsistencyLevel consistencyLevel,
-                                               long queryStartNanoTime)
+                                               Dispatcher.RequestTime requestTime)
     {
         // Note that in general, a RangeCommandIterator will honor the command limit for each range, but will not enforce it globally.
-        RangeCommandIterator rangeCommands = rangeCommandIterator(command, consistencyLevel, queryStartNanoTime);
+        RangeCommandIterator rangeCommands = rangeCommandIterator(command, consistencyLevel, requestTime);
         return command.limits().filter(command.postReconciliationProcessing(rangeCommands),
                                        command.nowInSec(),
                                        command.selectsFullPartition(),
@@ -69,7 +70,7 @@ public class RangeCommands
     @SuppressWarnings("resource") // created iterators will be closed in CQL layer through the chain of transformations
     static RangeCommandIterator rangeCommandIterator(PartitionRangeReadCommand command,
                                                      ConsistencyLevel consistencyLevel,
-                                                     long queryStartNanoTime)
+                                                     Dispatcher.RequestTime requestTime)
     {
         Tracing.trace("Computing ranges to query");
 
@@ -96,7 +97,7 @@ public class RangeCommands
                                         concurrencyFactor,
                                         maxConcurrencyFactor,
                                         replicaPlans.size(),
-                                        queryStartNanoTime);
+                                        requestTime);
     }
 
     /**

src/java/org/apache/cassandra/service/reads/repair/AbstractReadRepair.java

@@ -20,12 +20,11 @@ package org.apache.cassandra.service.reads.repair;
 
 import java.util.function.Consumer;
 
-import com.codahale.metrics.Meter;
 import com.google.common.base.Preconditions;
-
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.codahale.metrics.Meter;
 import org.apache.cassandra.concurrent.Stage;
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.db.ColumnFamilyStore;
@@ -46,6 +45,7 @@ import org.apache.cassandra.service.reads.DataResolver;
 import org.apache.cassandra.service.reads.DigestResolver;
 import org.apache.cassandra.service.reads.ReadCallback;
 import org.apache.cassandra.tracing.Tracing;
+import org.apache.cassandra.transport.Dispatcher;
 
 import static java.util.concurrent.TimeUnit.MICROSECONDS;
 
@@ -55,7 +55,7 @@ public abstract class AbstractReadRepair<E extends Endpoints<E>, P extends Repli
     protected static final Logger logger = LoggerFactory.getLogger(AbstractReadRepair.class);
 
     protected final ReadCommand command;
-    protected final long queryStartNanoTime;
+    protected final Dispatcher.RequestTime requestTime;
     protected final ReplicaPlan.Shared<E, P> replicaPlan;
     protected final ColumnFamilyStore cfs;
 
@@ -77,10 +77,10 @@ public abstract class AbstractReadRepair<E extends Endpoints<E>, P extends Repli
 
     public AbstractReadRepair(ReadCommand command,
                               ReplicaPlan.Shared<E, P> replicaPlan,
-                              long queryStartNanoTime)
+                              Dispatcher.RequestTime requestTime)
     {
         this.command = command;
-        this.queryStartNanoTime = queryStartNanoTime;
+        this.requestTime = requestTime;
         this.replicaPlan = replicaPlan;
         this.cfs = Keyspace.openAndGetStore(command.metadata());
     }
@@ -96,7 +96,7 @@ public abstract class AbstractReadRepair<E extends Endpoints<E>, P extends Repli
         
         if (to.isSelf())
         {
-            Stage.READ.maybeExecuteImmediately(new StorageProxy.LocalReadRunnable(command, readCallback, trackRepairedStatus));
+            Stage.READ.maybeExecuteImmediately(new StorageProxy.LocalReadRunnable(command, readCallback, requestTime, trackRepairedStatus));
             return;
         }
 
@@ -116,7 +116,7 @@ public abstract class AbstractReadRepair<E extends Endpoints<E>, P extends Repli
             Tracing.trace("Enqueuing {} data read to {}", type, to);
         }
 
-        Message<ReadCommand> message = command.createMessage(trackRepairedStatus && to.isFull());
+        Message<ReadCommand> message = command.createMessage(trackRepairedStatus && to.isFull(), requestTime);
         MessagingService.instance().sendWithCallback(message, to.endpoint(), readCallback);
     }
 
@@ -139,8 +139,8 @@ public abstract class AbstractReadRepair<E extends Endpoints<E>, P extends Repli
         boolean trackRepairedStatus = DatabaseDescriptor.getRepairedDataTrackingForPartitionReadsEnabled();
 
         // Do a full data read to resolve the correct response (and repair node that need be)
-        DataResolver<E, P> resolver = new DataResolver<>(command, replicaPlan, this, queryStartNanoTime, trackRepairedStatus);
-        ReadCallback<E, P> readCallback = new ReadCallback<>(resolver, command, replicaPlan, queryStartNanoTime);
+        DataResolver<E, P> resolver = new DataResolver<>(command, replicaPlan, this, requestTime, trackRepairedStatus);
+        ReadCallback<E, P> readCallback = new ReadCallback<>(resolver, command, replicaPlan, requestTime);
 
         digestRepair = new DigestRepair<>(resolver, readCallback, resultConsumer);
 
@@ -190,7 +190,7 @@ public abstract class AbstractReadRepair<E extends Endpoints<E>, P extends Repli
         if (repair == null)
             return;
 
-        if (shouldSpeculate() && !repair.readCallback.await(cfs.sampleReadLatencyMicros, MICROSECONDS))
+        if (shouldSpeculate() && !repair.readCallback.awaitUntil(requestTime.startedAtNanos() + MICROSECONDS.toNanos(cfs.sampleReadLatencyMicros)))
         {
             Replica uncontacted = replicaPlan().firstUncontactedCandidate(replica -> true);
             if (uncontacted == null)

src/java/org/apache/cassandra/service/reads/repair/BlockingReadRepair.java

@@ -37,6 +37,7 @@ import org.apache.cassandra.locator.Replica;
 import org.apache.cassandra.locator.ReplicaPlan;
 import org.apache.cassandra.metrics.ReadRepairMetrics;
 import org.apache.cassandra.tracing.Tracing;
+import org.apache.cassandra.transport.Dispatcher;
 
 import static java.util.concurrent.TimeUnit.MICROSECONDS;
 import static java.util.concurrent.TimeUnit.NANOSECONDS;
@@ -53,9 +54,9 @@ public class BlockingReadRepair<E extends Endpoints<E>, P extends ReplicaPlan.Fo
 
     protected final Queue<BlockingPartitionRepair> repairs = new ConcurrentLinkedQueue<>();
 
-    BlockingReadRepair(ReadCommand command, ReplicaPlan.Shared<E, P> replicaPlan, long queryStartNanoTime)
+    BlockingReadRepair(ReadCommand command, ReplicaPlan.Shared<E, P> replicaPlan, Dispatcher.RequestTime requestTime)
     {
-        super(command, replicaPlan, queryStartNanoTime);
+        super(command, replicaPlan, requestTime);
     }
 
     public UnfilteredPartitionIterators.MergeListener getMergeListener(P replicaPlan)
@@ -84,7 +85,9 @@ public class BlockingReadRepair<E extends Endpoints<E>, P extends ReplicaPlan.Fo
         BlockingPartitionRepair timedOut = null;
         for (BlockingPartitionRepair repair : repairs)
         {
-            if (!repair.awaitRepairsUntil(DatabaseDescriptor.getReadRpcTimeout(NANOSECONDS) + queryStartNanoTime, NANOSECONDS))
+            long deadline = requestTime.computeDeadline(DatabaseDescriptor.getReadRpcTimeout(NANOSECONDS));
+
+            if (!repair.awaitRepairsUntil(deadline, NANOSECONDS))
             {
                 timedOut = repair;
                 break;

src/java/org/apache/cassandra/service/reads/repair/ReadOnlyReadRepair.java

@@ -29,6 +29,7 @@ import org.apache.cassandra.locator.Endpoints;
 import org.apache.cassandra.locator.Replica;
 import org.apache.cassandra.locator.ReplicaPlan;
 import org.apache.cassandra.metrics.ReadRepairMetrics;
+import org.apache.cassandra.transport.Dispatcher;
 
 /**
  * Only performs the collection of data responses and reconciliation of them, doesn't send repair mutations
@@ -37,9 +38,9 @@ import org.apache.cassandra.metrics.ReadRepairMetrics;
 public class ReadOnlyReadRepair<E extends Endpoints<E>, P extends ReplicaPlan.ForRead<E, P>>
         extends AbstractReadRepair<E, P>
 {
-    ReadOnlyReadRepair(ReadCommand command, ReplicaPlan.Shared<E, P> replicaPlan, long queryStartNanoTime)
+    ReadOnlyReadRepair(ReadCommand command, ReplicaPlan.Shared<E, P> replicaPlan, Dispatcher.RequestTime requestTime)
     {
-        super(command, replicaPlan, queryStartNanoTime);
+        super(command, replicaPlan, requestTime);
     }
 
     @Override

src/java/org/apache/cassandra/service/reads/repair/ReadRepair.java

@@ -31,19 +31,20 @@ import org.apache.cassandra.exceptions.ReadTimeoutException;
 import org.apache.cassandra.locator.Replica;
 import org.apache.cassandra.locator.ReplicaPlan;
 import org.apache.cassandra.service.reads.DigestResolver;
+import org.apache.cassandra.transport.Dispatcher;
 
 public interface ReadRepair<E extends Endpoints<E>, P extends ReplicaPlan.ForRead<E, P>>
 {
     public interface Factory
     {
         <E extends Endpoints<E>, P extends ReplicaPlan.ForRead<E, P>>
-        ReadRepair<E, P> create(ReadCommand command, ReplicaPlan.Shared<E, P> replicaPlan, long queryStartNanoTime);
+        ReadRepair<E, P> create(ReadCommand command, ReplicaPlan.Shared<E, P> replicaPlan, Dispatcher.RequestTime requestTime);
     }
 
     static <E extends Endpoints<E>, P extends ReplicaPlan.ForRead<E, P>>
-    ReadRepair<E, P> create(ReadCommand command, ReplicaPlan.Shared<E, P> replicaPlan, long queryStartNanoTime)
+    ReadRepair<E, P> create(ReadCommand command, ReplicaPlan.Shared<E, P> replicaPlan, Dispatcher.RequestTime requestTime)
     {
-        return command.metadata().params.readRepair.create(command, replicaPlan, queryStartNanoTime);
+        return command.metadata().params.readRepair.create(command, replicaPlan, requestTime);
     }
 
     /**

src/java/org/apache/cassandra/service/reads/repair/ReadRepairStrategy.java

@@ -21,24 +21,25 @@ package org.apache.cassandra.service.reads.repair;
 import org.apache.cassandra.db.ReadCommand;
 import org.apache.cassandra.locator.Endpoints;
 import org.apache.cassandra.locator.ReplicaPlan;
+import org.apache.cassandra.transport.Dispatcher;
 
 public enum ReadRepairStrategy implements ReadRepair.Factory
 {
     NONE
     {
         public <E extends Endpoints<E>, P extends ReplicaPlan.ForRead<E, P>>
-        ReadRepair<E, P> create(ReadCommand command, ReplicaPlan.Shared<E, P> replicaPlan, long queryStartNanoTime)
+        ReadRepair<E, P> create(ReadCommand command, ReplicaPlan.Shared<E, P> replicaPlan, Dispatcher.RequestTime requestTime)
         {
-            return new ReadOnlyReadRepair<>(command, replicaPlan, queryStartNanoTime);
+            return new ReadOnlyReadRepair<>(command, replicaPlan, requestTime);
         }
     },
 
     BLOCKING
     {
         public <E extends Endpoints<E>, P extends ReplicaPlan.ForRead<E, P>>
-        ReadRepair<E, P> create(ReadCommand command, ReplicaPlan.Shared<E, P> replicaPlan, long queryStartNanoTime)
+        ReadRepair<E, P> create(ReadCommand command, ReplicaPlan.Shared<E, P> replicaPlan, Dispatcher.RequestTime requestTime)
         {
-            return new BlockingReadRepair<>(command, replicaPlan, queryStartNanoTime);
+            return new BlockingReadRepair<>(command, replicaPlan, requestTime);
         }
     };
 

src/java/org/apache/cassandra/tracing/TraceStateImpl.java

@@ -32,6 +32,7 @@ import org.apache.cassandra.db.Mutation;
 import org.apache.cassandra.exceptions.OverloadedException;
 import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.service.StorageProxy;
+import org.apache.cassandra.transport.Dispatcher;
 import org.apache.cassandra.utils.JVMStabilityInspector;
 import org.apache.cassandra.utils.TimeUUID;
 import org.apache.cassandra.utils.concurrent.Future;
@@ -39,7 +40,6 @@ import org.apache.cassandra.utils.concurrent.FutureCombiner;
 
 import static java.util.Collections.singletonList;
 import static org.apache.cassandra.db.ConsistencyLevel.ANY;
-import static org.apache.cassandra.utils.Clock.Global.nanoTime;
 
 /**
  * ThreadLocal state for a tracing session. The presence of an instance of this class as a ThreadLocal denotes that an
@@ -113,7 +113,7 @@ public class TraceStateImpl extends TraceState
     {
         try
         {
-            StorageProxy.mutate(singletonList(mutation), ANY, nanoTime());
+            StorageProxy.mutate(singletonList(mutation), ANY, Dispatcher.RequestTime.forImmediateExecution());
         }
         catch (OverloadedException e)
         {

src/java/org/apache/cassandra/transport/CQLMessageHandler.java

@@ -23,8 +23,6 @@ import java.nio.ByteBuffer;
 import java.util.concurrent.TimeUnit;
 
 import com.google.common.primitives.Ints;
-import org.apache.cassandra.transport.ClientResourceLimits.Overload;
-import org.apache.cassandra.utils.concurrent.NonBlockingRateLimiter;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -33,8 +31,8 @@ import io.netty.buffer.Unpooled;
 import io.netty.channel.Channel;
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.exceptions.OverloadedException;
-import org.apache.cassandra.metrics.ClientMetrics;
 import org.apache.cassandra.metrics.ClientMessageSizeMetrics;
+import org.apache.cassandra.metrics.ClientMetrics;
 import org.apache.cassandra.net.AbstractMessageHandler;
 import org.apache.cassandra.net.FrameDecoder;
 import org.apache.cassandra.net.FrameDecoder.IntactFrame;
@@ -42,9 +40,11 @@ import org.apache.cassandra.net.FrameEncoder;
 import org.apache.cassandra.net.ResourceLimits;
 import org.apache.cassandra.net.ResourceLimits.Limit;
 import org.apache.cassandra.net.ShareableBytes;
+import org.apache.cassandra.transport.ClientResourceLimits.Overload;
 import org.apache.cassandra.transport.Flusher.FlushItem.Framed;
 import org.apache.cassandra.transport.messages.ErrorMessage;
 import org.apache.cassandra.utils.NoSpamLogger;
+import org.apache.cassandra.utils.concurrent.NonBlockingRateLimiter;
 
 import static org.apache.cassandra.utils.MonotonicClock.Global.approxTime;
 
@@ -81,6 +81,7 @@ public class CQLMessageHandler<M extends Message> extends AbstractMessageHandler
     public static final int LARGE_MESSAGE_THRESHOLD = FrameEncoder.Payload.MAX_SIZE - 1;
     public static final TimeUnit RATE_LIMITER_DELAY_UNIT = TimeUnit.NANOSECONDS;
 
+    private final QueueBackpressure queueBackpressure;
     private final Envelope.Decoder envelopeDecoder;
     private final Message.Decoder<M> messageDecoder;
     private final FrameEncoder.PayloadAllocator payloadAllocator;
@@ -95,7 +96,8 @@ public class CQLMessageHandler<M extends Message> extends AbstractMessageHandler
 
     interface MessageConsumer<M extends Message>
     {
-        void accept(Channel channel, M message, Dispatcher.FlushItemConverter toFlushItem, Overload backpressure);
+        void dispatch(Channel channel, M message, Dispatcher.FlushItemConverter toFlushItem, Overload backpressure);
+        boolean hasQueueCapacity();
     }
 
     interface ErrorHandler
@@ -111,6 +113,7 @@ public class CQLMessageHandler<M extends Message> extends AbstractMessageHandler
                       MessageConsumer<M> dispatcher,
                       FrameEncoder.PayloadAllocator payloadAllocator,
                       int queueCapacity,
+                      QueueBackpressure queueBackpressure,
                       ClientResourceLimits.ResourceProvider resources,
                       OnHandlerClosed onClosed,
                       ErrorHandler errorHandler,
@@ -129,6 +132,7 @@ public class CQLMessageHandler<M extends Message> extends AbstractMessageHandler
         this.messageDecoder     = messageDecoder;
         this.payloadAllocator   = payloadAllocator;
         this.dispatcher         = dispatcher;
+        this.queueBackpressure  = queueBackpressure;
         this.errorHandler       = errorHandler;
         this.throwOnOverload    = throwOnOverload;
         this.version            = version;
@@ -178,7 +182,8 @@ public class CQLMessageHandler<M extends Message> extends AbstractMessageHandler
 
         // max CQL message size defaults to 256mb, so should be safe to downcast
         int messageSize = Ints.checkedCast(header.bodySizeInBytes);
-        
+
+        Overload backpressure = Overload.NONE;
         if (throwOnOverload)
         {
             if (!acquireCapacity(header, endpointReserve, globalReserve))
@@ -188,56 +193,67 @@ public class CQLMessageHandler<M extends Message> extends AbstractMessageHandler
             }
 
             if (DatabaseDescriptor.getNativeTransportRateLimitingEnabled() && !requestRateLimiter.tryReserve())
+                backpressure = Overload.REQUESTS;
+            else if (!dispatcher.hasQueueCapacity())
+                backpressure = Overload.QUEUE_TIME;
+
+            if (backpressure != Overload.NONE)
             {
                 // We've already allocated against the bytes-in-flight limits, so release those resources.
                 release(header);
-                discardAndThrow(endpointReserve, globalReserve, buf, header, messageSize, Overload.REQUESTS);
+                discardAndThrow(endpointReserve, globalReserve, buf, header, messageSize, backpressure);
                 return true;
             }
         }
         else
         {
-            Overload backpressure = Overload.NONE;
+            long delay = -1;
 
             if (!acquireCapacityAndQueueOnFailure(header, endpointReserve, globalReserve))
-            {
-                if (processRequestAndUpdateMetrics(bytes, header, messageSize, Overload.BYTES_IN_FLIGHT))
-                {
-                    if (decoder.isActive())
-                        ClientMetrics.instance.pauseConnection();
-                }
-
                 backpressure = Overload.BYTES_IN_FLIGHT;
-            }
-            
+
+            // Apply rate limiting, if enabled
             if (DatabaseDescriptor.getNativeTransportRateLimitingEnabled())
             {
                 // Reserve a permit even if we've already triggered backpressure on bytes in flight.
-                long delay = requestRateLimiter.reserveAndGetDelay(RATE_LIMITER_DELAY_UNIT);
-                
-                if (backpressure == Overload.NONE && delay > 0)
-                {
-                    if (processRequestAndUpdateMetrics(bytes, header, messageSize, Overload.REQUESTS))
-                    {
-                        if (decoder.isActive())
-                            ClientMetrics.instance.pauseConnection();
+                delay = requestRateLimiter.reserveAndGetDelay(RATE_LIMITER_DELAY_UNIT);
 
-                        // Schedule a wakup here if we process successfully. The connection should be closing otherwise.  
+                if (backpressure == Overload.NONE && delay > 0)
+                    backpressure = Overload.REQUESTS;
+            }
+
+            // Check queue time, if enabled
+            if (backpressure == Overload.NONE && !dispatcher.hasQueueCapacity())
+            {
+                delay = queueBackpressure.markAndGetDelay(RATE_LIMITER_DELAY_UNIT);
+
+                if (delay > 0)
+                    backpressure = Overload.QUEUE_TIME;
+            }
+
+            if (backpressure != Overload.NONE)
+            {
+                if (processRequestAndUpdateMetrics(bytes, header, messageSize, backpressure))
+                {
+                    if (decoder.isActive())
+                        ClientMetrics.instance.pauseConnection();
+
+                    if (delay > 0)
+                    {
+                        // Schedule a wakeup here if we process successfully. The connection should be closing otherwise.
                         scheduleConnectionWakeupTask(delay, RATE_LIMITER_DELAY_UNIT);
                     }
-                    
-                    backpressure = Overload.REQUESTS;
                 }
-            }
-            
-            // If we triggered backpressure, make sure the caller stops processing frames after the request completes.
-            if (backpressure != Overload.NONE)
+
+                // If we triggered backpressure, make sure the caller stops processing frames after the request completes.
                 return false;
+            }
         }
 
         return processRequestAndUpdateMetrics(bytes, header, messageSize, Overload.NONE);
     }
 
+
     private boolean processRequestAndUpdateMetrics(ShareableBytes bytes, Envelope.Header header, int messageSize, Overload backpressure)
     {
         channelPayloadBytesInFlight += messageSize;
@@ -263,13 +279,24 @@ public class CQLMessageHandler<M extends Message> extends AbstractMessageHandler
     }
 
     private OverloadedException buildOverloadedException(Limit endpointReserve, Limit globalReserve, Overload overload) {
-        return overload == Overload.REQUESTS
-                ? new OverloadedException(String.format("Request breached global limit of %d requests/second. Server is " +
-                                                        "currently in an overloaded state and cannot accept more requests.", 
-                                                        requestRateLimiter.getRate()))
-                : new OverloadedException(String.format("Request breached limit on bytes in flight. (Endpoint: %d/%d bytes, Global: %d/%d bytes.) " +
-                                                        "Server is currently in an overloaded state and cannot accept more requests.",
-                                                        endpointReserve.using(), endpointReserve.limit(), globalReserve.using(), globalReserve.limit()));
+        switch (overload)
+        {
+            case REQUESTS:
+                return new OverloadedException(String.format("Request breached global limit of %d requests/second. Server is " +
+                                                             "currently in an overloaded state and cannot accept more requests.",
+                                                             requestRateLimiter.getRate()));
+            case BYTES_IN_FLIGHT:
+                return new OverloadedException(String.format("Request breached limit on bytes in flight. (Endpoint: %d/%d bytes, Global: %d/%d bytes.) " +
+                                                             "Server is currently in an overloaded state and cannot accept more requests.",
+                                                             endpointReserve.using(), endpointReserve.limit(), globalReserve.using(), globalReserve.limit()));
+            case QUEUE_TIME:
+                return new OverloadedException(String.format("Request has spent over %s time of the maximum timeout %dms in the queue",
+                                                             DatabaseDescriptor.getNativeTransportQueueMaxItemAgeThreshold(),
+                                                             DatabaseDescriptor.getNativeTransportTimeout(TimeUnit.MILLISECONDS)));
+            default:
+                throw new IllegalArgumentException("Overload exception should not have been thrown with " + overload);
+
+        }
     }
 
     private void logOverload(Limit endpointReserve, Limit globalReserve, Envelope.Header header, int messageSize)
@@ -348,7 +375,7 @@ public class CQLMessageHandler<M extends Message> extends AbstractMessageHandler
         try
         {
             message = messageDecoder.decode(channel, request);
-            dispatcher.accept(channel, message, this::toFlushItem, backpressure);
+            dispatcher.dispatch(channel, message, this::toFlushItem, backpressure);
             
             // sucessfully delivered a CQL message to the execution
             // stage, so reset the counter of consecutive errors
@@ -494,55 +521,79 @@ public class CQLMessageHandler<M extends Message> extends AbstractMessageHandler
             
             LargeMessage largeMessage = new LargeMessage(header);
 
-            if (!acquireCapacity(header, endpointReserve, globalReserve))
+            if (throwOnOverload)
             {
-                // In the case of large messages, never stop processing incoming frames
-                // as this will halt the client meaning no further frames will be sent,
-                // leading to starvation.
-                // If the throwOnOverload option is set, don't process the message once
-                // read, return an error response to notify the client that resource
-                // limits have been exceeded. If the option isn't set, the only thing we
-                // can do is to consume the subsequent frames and process the message.
-                // Large and small messages are never interleaved for a single client, so
-                // we know that this client will finish sending the large message before
-                // anything else. Other clients sending small messages concurrently will
-                // be backpressured by the global resource limits. The server is still
-                // vulnerable to overload by multiple clients sending large messages
-                // concurrently.
-                if (throwOnOverload)
+                if (!acquireCapacity(header, endpointReserve, globalReserve))
                 {
+                    // In the case of large messages, never stop processing incoming frames
+                    // as this will halt the client meaning no further frames will be sent,
+                    // leading to starvation.
+                    // If the throwOnOverload option is set, don't process the message once
+                    // read, return an error response to notify the client that resource
+                    // limits have been exceeded. If the option isn't set, the only thing we
+                    // can do is to consume the subsequent frames and process the message.
+                    // Large and small messages are never interleaved for a single client, so
+                    // we know that this client will finish sending the large message before
+                    // anything else. Other clients sending small messages concurrently will
+                    // be backpressured by the global resource limits. The server is still
+                    // vulnerable to overload by multiple clients sending large messages
+                    // concurrently.
+
                     // Mark as overloaded so that discard the message after consuming any subsequent frames.
                     ClientMetrics.instance.markRequestDiscarded();
                     logOverload(endpointReserve, globalReserve, header, messageSize);
                     largeMessage.markOverloaded(Overload.BYTES_IN_FLIGHT);
                 }
-            }
-            else if (DatabaseDescriptor.getNativeTransportRateLimitingEnabled())
-            {
-                if (throwOnOverload)
+                else
                 {
-                    if (!requestRateLimiter.tryReserve())
+                    Overload backpressure = Overload.NONE;
+                    if (DatabaseDescriptor.getNativeTransportRateLimitingEnabled() && !requestRateLimiter.tryReserve())
+                        backpressure = Overload.REQUESTS;
+                    else if (!dispatcher.hasQueueCapacity())
+                        backpressure = Overload.QUEUE_TIME;
+
+                    if (backpressure != Overload.NONE)
                     {
                         ClientMetrics.instance.markRequestDiscarded();
                         logOverload(endpointReserve, globalReserve, header, messageSize);
-                        
+
                         // Mark as overloaded so that we discard the message after consuming any subsequent frames.
                         // (i.e. Request resources we may already have acquired above will be released.)
-                        largeMessage.markOverloaded(Overload.REQUESTS);
-                        
+                        largeMessage.markOverloaded(backpressure);
+
                         this.largeMessage = largeMessage;
                         largeMessage.supply(frame);
                         return true;
                     }
                 }
-                else
+            }
+            else
+            {
+                if (acquireCapacity(header, endpointReserve, globalReserve))
                 {
-                    long delay = requestRateLimiter.reserveAndGetDelay(RATE_LIMITER_DELAY_UNIT);
+                    long delay = -1;
+                    Overload backpressure = Overload.NONE;
+
+                    if (DatabaseDescriptor.getNativeTransportRateLimitingEnabled())
+                    {
+                        delay = requestRateLimiter.reserveAndGetDelay(RATE_LIMITER_DELAY_UNIT);
+
+                        if (delay > 0)
+                            backpressure = Overload.REQUESTS;
+                    }
+
+                    if (backpressure == Overload.NONE && !dispatcher.hasQueueCapacity())
+                    {
+                        delay = queueBackpressure.markAndGetDelay(RATE_LIMITER_DELAY_UNIT);
+
+                        if (delay > 0)
+                            backpressure = Overload.QUEUE_TIME;
+                    }
 
                     if (delay > 0)
                     {
                         this.largeMessage = largeMessage;
-                        largeMessage.markBackpressure(Overload.REQUESTS);
+                        largeMessage.markBackpressure(backpressure);
                         largeMessage.supply(frame);
 
                         if (decoder.isActive())
@@ -552,8 +603,12 @@ public class CQLMessageHandler<M extends Message> extends AbstractMessageHandler
                         return false;
                     }
                 }
+                else
+                {
+                    noSpamLogger.error("Could not aquire capacity while processing native protocol message");
+                }
             }
-            
+
             this.largeMessage = largeMessage;
             largeMessage.supply(frame);
             return true;

src/java/org/apache/cassandra/transport/ClientResourceLimits.java

@@ -45,7 +45,7 @@ public class ClientResourceLimits
 
     public static final NonBlockingRateLimiter GLOBAL_REQUEST_LIMITER = new NonBlockingRateLimiter(getNativeTransportMaxRequestsPerSecond());
 
-    public enum Overload { NONE, REQUESTS, BYTES_IN_FLIGHT }
+    public enum Overload { NONE, REQUESTS, BYTES_IN_FLIGHT, QUEUE_TIME }
     
     public static Allocator getAllocatorForEndpoint(InetAddress endpoint)
     {

src/java/org/apache/cassandra/transport/Dispatcher.java

@@ -30,9 +30,10 @@ import org.slf4j.LoggerFactory;
 import io.netty.channel.Channel;
 import io.netty.channel.EventLoop;
 import io.netty.util.AttributeKey;
-import org.apache.cassandra.concurrent.DebuggableTask.RunnableDebuggableTask;
+import org.apache.cassandra.concurrent.DebuggableTask;
 import org.apache.cassandra.concurrent.LocalAwareExecutorPlus;
 import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.exceptions.OverloadedException;
 import org.apache.cassandra.metrics.ClientMetrics;
 import org.apache.cassandra.net.FrameEncoder;
 import org.apache.cassandra.service.ClientWarn;
@@ -48,7 +49,7 @@ import org.apache.cassandra.utils.NoSpamLogger;
 
 import static org.apache.cassandra.concurrent.SharedExecutorPool.SHARED;
 
-public class Dispatcher
+public class Dispatcher implements CQLMessageHandler.MessageConsumer<Message.Request>
 {
     private static final Logger logger = LoggerFactory.getLogger(Dispatcher.class);
     
@@ -78,22 +79,176 @@ public class Dispatcher
         this.useLegacyFlusher = useLegacyFlusher;
     }
 
+    @Override
     public void dispatch(Channel channel, Message.Request request, FlushItemConverter forFlusher, Overload backpressure)
     {
         requestExecutor.submit(new RequestProcessor(channel, request, forFlusher, backpressure));
         ClientMetrics.instance.markRequestDispatched();
     }
 
-    public class RequestProcessor implements RunnableDebuggableTask
+    public static class RequestTime
+    {
+        private final long enqueuedAtNanos;
+        private final long startedAtNanos;
+
+        public RequestTime(long createdAtNanos)
+        {
+            this(createdAtNanos, createdAtNanos);
+        }
+
+        public RequestTime(long enqueuedAtNanos, long startedAtNanos)
+        {
+            this.enqueuedAtNanos = enqueuedAtNanos;
+            this.startedAtNanos = startedAtNanos;
+        }
+
+        public static RequestTime forImmediateExecution()
+        {
+            return new RequestTime(MonotonicClock.Global.preciseTime.now());
+        }
+
+        public long startedAtNanos()
+        {
+            return startedAtNanos;
+        }
+
+        public long enqueuedAtNanos()
+        {
+            return enqueuedAtNanos;
+        }
+
+        /**
+         * Base time is used by timeouts, and can be set to either when the request was added to the queue,
+         * or when the processing has started. Since client read/write timeouts are usually aligned with
+         * server-side timeouts, it is desireable to use enqueue time as a base. However, since client removes
+         * the handler `readTimeoutMillis` (which is 12 seconds by default), the upper bound for any execution on
+         * the coordinator is 12 seconds (thanks to CASSANDRA-7392, any replica-side query is capped by the verb timeout),
+         * if REQUEST option is used. But even simply allowing such long timeouts also implicitly allows queues to grow
+         * large, since our queues are currently unbounded.
+         *
+         * Latency, however, is _always_ based on request processing time, since the amount of time that request spends
+         * in the queue is not a representative metric of replica performance.
+         */
+        public long baseTimeNanos()
+        {
+            switch (DatabaseDescriptor.getCQLStartTime())
+            {
+                case REQUEST:
+                    return startedAtNanos();
+                case QUEUE:
+                    return enqueuedAtNanos();
+                default:
+                    throw new IllegalArgumentException("Unknown start time: " + DatabaseDescriptor.getCQLStartTime());
+            }
+        }
+
+        /**
+         * Given the current time and a base timeout for the verb return a request's expiration deadline,
+         * the time at which it becomes eligible for load shedding.
+         * The two factors to consider are the per-verb and client timeouts. Both are calculated by subtracting the
+         * time already elapsed during the lifetime of the request from some base value.
+         *
+         * When deriving verb timeout, two alternative semantics are available. This timeout may represent either:
+         *  * the total time available for a coordinator to process a client request and return its response
+         *  * a time bound for a coordinator to send internode requests and gather responses from replicas
+         *
+         * The point from which elapsed time is measured here is configurable to accommodate these two different
+         * options. For the former, the clock starts when a client request is received and enqueued by the coordinator.
+         * For the latter, it starts when the request is dequeued by the coordinator and processing is started.
+         * See {@link #baseTimeNanos()} for details.
+         *
+         * The client timeout represents how long the sender of a request is prepared to wait for a response. By
+         * implication, after this window has passed any further work done on the server side is wasted effort. Ideally,
+         * the base for this timeout would be set on a per-request basis but as this not currently supported in the
+         * protocol, it is configured uniformly for all requests. See {@DatabaseDescriptor#getNativeTransportTimeout}.
+         * For this calculation, elapsed time is always measured from the point when a request is received and enqueued.
+         *
+         * Where verb timeout is based on queue admission, deadline computation is straightforward. The expiration
+         * deadline is simply the current time plus the smaller of the verb and client timeouts.
+         *
+         * However, if verb timeout is based on when the request is dequeued, the implications are more nuanced.
+         * In this scenario, while there may still be "headroom" available within the verb timeout, using it could
+         * exceed the client timeout (which is always based on admission time).
+         *
+         * For example:
+         *
+         * * Client timeout base is 10 (cb), verb timeout base is 5 (vb)
+         * * Request is enqueued at t1 (e)
+         * * Request is dequeued at t8 (d)
+         * * computeDeadline is called at t9 (n)
+         *
+         * If verb timeout is based on dequeuing, there would still some time remaining before a verb-based deadline.
+         * elapsed  = (n - d)        ;  1
+         * timeout  = (vb - elapsed) ;  4
+         * deadline = (n + timeout)  ;  t13
+         * ostensibly, the coordinator has until t13 to complete processing
+         *
+         * But as client timeout is measured from admission time, the request may exceeded the maximum wait period for
+         * the client sooner.
+         * elapsed  = (n - e)        ;  8
+         * timeout  = (cb - elapsed) ;  2
+         * deadline = (n + timeout)  ;  t11
+         * So the coordinator actually only has until t11 to complete processing, beyond then the client will not accept
+         * any response.
+         *
+         * @param verbExpiresAfterNanos the base timeout value for the verb being executed
+         * @return the point in time after which no further processing should occur
+         */
+        public long computeDeadline(long verbExpiresAfterNanos)
+        {
+            long clientDeadline = clientDeadline();
+
+            long verbDeadline = baseTimeNanos() + verbExpiresAfterNanos;
+            // Whichever one is closer
+            return Math.min(verbDeadline, clientDeadline);
+        }
+
+        public long computeTimeout(long now, long verbExpiresAfterNanos)
+        {
+            return computeDeadline(verbExpiresAfterNanos) - now;
+        }
+
+        /**
+         * No request should survive native request deadline, but in order to err on the side of caution, we have this
+         * swtich that allows hints to be submitted to mutation stage when cluster is potentially overloaded. Allowing
+         * hints to be not bound by deadline can exacerbate overload, but since there are also correctness implications,
+         * this seemed like a reasonable configuration option.
+         */
+        public boolean shouldSendHints()
+        {
+            if (!DatabaseDescriptor.getEnforceNativeDeadlineForHints())
+                return true;
+
+            long now = MonotonicClock.Global.preciseTime.now();
+            long clientDeadline = clientDeadline();
+            return now < clientDeadline;
+        }
+
+        public long clientDeadline()
+        {
+            return enqueuedAtNanos() + DatabaseDescriptor.getNativeTransportTimeout(TimeUnit.NANOSECONDS);
+        }
+
+        public long timeSpentInQueueNanos()
+        {
+            return startedAtNanos - enqueuedAtNanos;
+        }
+    }
+
+    /**
+     * It is important to keep this class an instance of {@link DebuggableTask}, either runnable or callable since this
+     * is the only way we can keep it not wrapped into a callable on SEPExecutor submission path. And we need this
+     * functionality for tracking time purposes.
+     */
+    public class RequestProcessor implements DebuggableTask.RunnableDebuggableTask
     {
         private final Channel channel;
         private final Message.Request request;
         private final FlushItemConverter forFlusher;
         private final Overload backpressure;
-        
-        private final long approxCreationTimeNanos = MonotonicClock.Global.approxTime.now();
-        private volatile long approxStartTimeNanos;
-        
+
+        private volatile long startTimeNanos;
+
         public RequestProcessor(Channel channel, Message.Request request, FlushItemConverter forFlusher, Overload backpressure)
         {
             this.channel = channel;
@@ -105,20 +260,20 @@ public class Dispatcher
         @Override
         public void run()
         {
-            approxStartTimeNanos = MonotonicClock.Global.approxTime.now();
-            processRequest(channel, request, forFlusher, backpressure, approxStartTimeNanos);
+            startTimeNanos = MonotonicClock.Global.preciseTime.now();
+            processRequest(channel, request, forFlusher, backpressure, new RequestTime(request.createdAtNanos, startTimeNanos));
         }
 
         @Override
         public long creationTimeNanos()
         {
-            return approxCreationTimeNanos;
+            return request.createdAtNanos;
         }
 
         @Override
         public long startTimeNanos()
         {
-            return approxStartTimeNanos;
+            return startTimeNanos;
         }
 
         @Override
@@ -126,14 +281,49 @@ public class Dispatcher
         {
             return request.toString();
         }
+
+        @Override
+        public String toString()
+        {
+            return "RequestProcessor{" +
+                   "request=" + request +
+                   ", approxStartTimeNanos=" + startTimeNanos +
+                   '}';
+        }
+    }
+
+    /**
+     * Checks if the item in the head of the queue has spent more than allowed time in the queue.
+     */
+    @Override
+    public boolean hasQueueCapacity()
+    {
+        double threshold = DatabaseDescriptor.getNativeTransportQueueMaxItemAgeThreshold();
+        if (threshold <= 0)
+            return true;
+
+        return requestExecutor.oldestTaskQueueTime() < (DatabaseDescriptor.getNativeTransportTimeout(TimeUnit.NANOSECONDS) * threshold);
     }
 
     /**
      * Note: this method may be executed on the netty event loop, during initial protocol negotiation; the caller is
      * responsible for cleaning up any global or thread-local state. (ex. tracing, client warnings, etc.).
      */
-    private static Message.Response processRequest(ServerConnection connection, Message.Request request, Overload backpressure, long startTimeNanos)
+    private static Message.Response processRequest(ServerConnection connection, Message.Request request, Overload backpressure, RequestTime requestTime)
     {
+        long queueTime = requestTime.timeSpentInQueueNanos();
+
+        // If we have already crossed the max timeout for all possible RPCs, we time out the query immediately.
+        // We do not differentiate between query types here, since if we got into a situation when, say, we have a PREPARE
+        // query that is stuck behind the EXECUTE query, we would rather time it out and catch up with a backlog, expecting
+        // that the bursts are going to be short-lived.
+        ClientMetrics.instance.queueTime(queueTime, TimeUnit.NANOSECONDS);
+        if (queueTime > DatabaseDescriptor.getNativeTransportTimeout(TimeUnit.NANOSECONDS))
+        {
+            ClientMetrics.instance.markTimedOutBeforeProcessing();
+            return ErrorMessage.fromException(new OverloadedException("Query timed out before it could start"));
+        }
+
         if (connection.getVersion().isGreaterOrEqualTo(ProtocolVersion.V4))
             ClientWarn.instance.captureWarnings();
 
@@ -142,28 +332,45 @@ public class Dispatcher
         if (request.isTrackable())
             CoordinatorWarnings.init();
 
-        if (backpressure == Overload.REQUESTS)
+        switch (backpressure)
         {
-            String message = String.format("Request breached global limit of %d requests/second and triggered backpressure.",
-                                           ClientResourceLimits.getNativeTransportMaxRequestsPerSecond());
+            case NONE:
+                break;
+            case REQUESTS:
+            {
+                String message = String.format("Request breached global limit of %d requests/second and triggered backpressure.",
+                                               ClientResourceLimits.getNativeTransportMaxRequestsPerSecond());
 
-            NoSpamLogger.log(logger, NoSpamLogger.Level.INFO, 1, TimeUnit.MINUTES, message);
-            ClientWarn.instance.warn(message);
-        }
-        else if (backpressure == Overload.BYTES_IN_FLIGHT)
-        {
-            String message = String.format("Request breached limit(s) on bytes in flight (Endpoint: %d, Global: %d) and triggered backpressure.",
-                                           ClientResourceLimits.getEndpointLimit(), ClientResourceLimits.getGlobalLimit());
+                NoSpamLogger.log(logger, NoSpamLogger.Level.INFO, 1, TimeUnit.MINUTES, message);
+                ClientWarn.instance.warn(message);
+                break;
+            }
+            case BYTES_IN_FLIGHT:
+            {
+                String message = String.format("Request breached limit(s) on bytes in flight (Endpoint: %d, Global: %d) and triggered backpressure.",
+                                               ClientResourceLimits.getEndpointLimit(), ClientResourceLimits.getGlobalLimit());
 
-            NoSpamLogger.log(logger, NoSpamLogger.Level.INFO, 1, TimeUnit.MINUTES, message);
-            ClientWarn.instance.warn(message);
+                NoSpamLogger.log(logger, NoSpamLogger.Level.INFO, 1, TimeUnit.MINUTES, message);
+                ClientWarn.instance.warn(message);
+                break;
+            }
+            case QUEUE_TIME:
+            {
+                String message = String.format("Request has spent over %s time of the maximum timeout %dms in the queue",
+                                               DatabaseDescriptor.getNativeTransportQueueMaxItemAgeThreshold(),
+                                               DatabaseDescriptor.getNativeTransportTimeout(TimeUnit.MILLISECONDS));
+
+                NoSpamLogger.log(logger, NoSpamLogger.Level.INFO, 1, TimeUnit.MINUTES, message);
+                ClientWarn.instance.warn(message);
+                break;
+            }
         }
 
         QueryState qstate = connection.validateNewMessage(request.type, connection.getVersion());
 
         Message.logger.trace("Received: {}, v={}", request, connection.getVersion());
         connection.requests.inc();
-        Message.Response response = request.execute(qstate, startTimeNanos);
+        Message.Response response = request.execute(qstate, requestTime);
 
         if (request.isTrackable())
             CoordinatorWarnings.done();
@@ -178,11 +385,11 @@ public class Dispatcher
     /**
      * Note: this method may be executed on the netty event loop.
      */
-    static Message.Response processRequest(Channel channel, Message.Request request, Overload backpressure, long approxStartTimeNanos)
+    static Message.Response processRequest(Channel channel, Message.Request request, Overload backpressure, RequestTime requestTime)
     {
         try
         {
-            return processRequest((ServerConnection) request.connection(), request, backpressure, approxStartTimeNanos);
+            return processRequest((ServerConnection) request.connection(), request, backpressure, requestTime);
         }
         catch (Throwable t)
         {
@@ -207,9 +414,9 @@ public class Dispatcher
     /**
      * Note: this method is not expected to execute on the netty event loop.
      */
-    void processRequest(Channel channel, Message.Request request, FlushItemConverter forFlusher, Overload backpressure, long approxStartTimeNanos)
+    void processRequest(Channel channel, Message.Request request, FlushItemConverter forFlusher, Overload backpressure, RequestTime requestTime)
     {
-        Message.Response response = processRequest(channel, request, backpressure, approxStartTimeNanos);
+        Message.Response response = processRequest(channel, request, backpressure, requestTime);
         FlushItem<?> toFlush = forFlusher.toFlushItem(channel, request, response);
         Message.logger.trace("Responding: {}, v={}", response, request.connection().getVersion());
         flush(toFlush);
@@ -245,7 +452,7 @@ public class Dispatcher
      * for delivering events to registered clients is dependent on protocol version and the configuration
      * of the pipeline. For v5 and newer connections, the event message is encoded into an Envelope,
      * wrapped in a FlushItem and then delivered via the pipeline's flusher, in a similar way to
-     * a Response returned from {@link #processRequest(Channel, Message.Request, FlushItemConverter, Overload, long)}.
+     * a Response returned from {@link #processRequest(Channel, Message.Request, FlushItemConverter, Overload, RequestTime)}.
      * It's worth noting that events are not generally fired as a direct response to a client request,
      * so this flush item has a null request attribute. The dispatcher itself is created when the
      * pipeline is first configured during protocol negotiation and is attached to the channel for

src/java/org/apache/cassandra/transport/InitialConnectionHandler.java

@@ -26,7 +26,6 @@ import java.util.List;
 import java.util.Map;
 
 import org.apache.cassandra.transport.ClientResourceLimits.Overload;
-import org.apache.cassandra.utils.MonotonicClock;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -149,8 +148,7 @@ public class InitialConnectionHandler extends ByteToMessageDecoder
                         promise = new VoidChannelPromise(ctx.channel(), false);
                     }
 
-                    long approxStartTimeNanos = MonotonicClock.Global.approxTime.now();
-                    final Message.Response response = Dispatcher.processRequest(ctx.channel(), startup, Overload.NONE, approxStartTimeNanos);
+                    final Message.Response response = Dispatcher.processRequest(ctx.channel(), startup, Overload.NONE, Dispatcher.RequestTime.forImmediateExecution());
 
                     outbound = response.encode(inbound.header.version);
                     ctx.writeAndFlush(outbound, promise);

src/java/org/apache/cassandra/transport/Message.java

@@ -36,6 +36,7 @@ import org.apache.cassandra.service.StorageService;
 import org.apache.cassandra.tracing.Tracing;
 import org.apache.cassandra.transport.messages.*;
 import org.apache.cassandra.service.QueryState;
+import org.apache.cassandra.utils.MonotonicClock;
 import org.apache.cassandra.utils.TimeUUID;
 
 import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID;
@@ -202,11 +203,14 @@ public abstract class Message
     public static abstract class Request extends Message
     {
         private boolean tracingRequested;
+        public final long createdAtNanos;
 
         protected Request(Type type)
         {
             super(type);
 
+            createdAtNanos = MonotonicClock.Global.preciseTime.now();
+
             if (type.direction != Direction.REQUEST)
                 throw new IllegalArgumentException();
         }
@@ -227,9 +231,9 @@ public abstract class Message
             return false;
         }
 
-        protected abstract Response execute(QueryState queryState, long queryStartNanoTime, boolean traceRequest);
+        protected abstract Response execute(QueryState queryState, Dispatcher.RequestTime requestTime, boolean traceRequest);
 
-        public final Response execute(QueryState queryState, long queryStartNanoTime)
+        public final Response execute(QueryState queryState, Dispatcher.RequestTime requestTime)
         {
             boolean shouldTrace = false;
             TimeUUID tracingSessionId = null;
@@ -252,7 +256,7 @@ public abstract class Message
             Response response;
             try
             {
-                response = execute(queryState, queryStartNanoTime, shouldTrace);
+                response = execute(queryState, requestTime, shouldTrace);
             }
             finally
             {

src/java/org/apache/cassandra/transport/PipelineConfigurator.java

@@ -24,6 +24,7 @@ import java.util.Map;
 import java.util.concurrent.TimeUnit;
 
 import com.google.common.base.Strings;
+import com.google.common.annotations.VisibleForTesting;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -94,16 +95,33 @@ public class PipelineConfigurator
     private final boolean keepAlive;
     private final EncryptionOptions.TlsEncryptionPolicy tlsEncryptionPolicy;
     private final Dispatcher dispatcher;
+    // Shared between pre-v5 and CQLMessage handlers
+    private final QueueBackpressure queueBackpressure;
 
     public PipelineConfigurator(boolean epoll,
                                 boolean keepAlive,
-                                boolean legacyFlusher,
+                                EncryptionOptions.TlsEncryptionPolicy encryptionPolicy,
+                                Dispatcher dispatcher)
+    {
+        this.epoll               = epoll;
+        this.keepAlive           = keepAlive;
+        this.tlsEncryptionPolicy = encryptionPolicy;
+        this.dispatcher          = dispatcher;
+        this.queueBackpressure   = QueueBackpressure.DEFAULT;
+    }
+
+    @Deprecated
+    @VisibleForTesting
+    public PipelineConfigurator(boolean epoll,
+                                boolean keepAlive,
+                                boolean useLegacyFlusher,
                                 EncryptionOptions.TlsEncryptionPolicy encryptionPolicy)
     {
         this.epoll               = epoll;
         this.keepAlive           = keepAlive;
         this.tlsEncryptionPolicy = encryptionPolicy;
-        this.dispatcher          = dispatcher(legacyFlusher);
+        this.dispatcher          = new Dispatcher(useLegacyFlusher);
+        this.queueBackpressure   = QueueBackpressure.DEFAULT;
     }
 
     public ChannelFuture initializeChannel(final EventLoopGroup workerGroup,
@@ -283,7 +301,12 @@ public class PipelineConfigurator
         int queueCapacity = DatabaseDescriptor.getNativeTransportReceiveQueueCapacityInBytes();
         ClientResourceLimits.ResourceProvider resourceProvider = resourceProvider(resourceAllocator);
         AbstractMessageHandler.OnHandlerClosed onClosed = handler -> resourceProvider.release();
-        boolean throwOnOverload = "1".equals(options.get(StartupMessage.THROW_ON_OVERLOAD));
+        String fromOptions = options.get(StartupMessage.THROW_ON_OVERLOAD);
+        boolean throwOnOverload;
+        if (fromOptions == null)
+            throwOnOverload = DatabaseDescriptor.getNativeTransportThrowOnOverload();
+        else
+            throwOnOverload = "1".equals(fromOptions);
 
         CQLMessageHandler.MessageConsumer<Message.Request> messageConsumer = messageConsumer();
         CQLMessageHandler<Message.Request> processor =
@@ -295,6 +318,7 @@ public class PipelineConfigurator
                                     messageConsumer,
                                     payloadAllocator,
                                     queueCapacity,
+                                    queueBackpressure,
                                     resourceProvider,
                                     onClosed,
                                     errorHandler,
@@ -329,7 +353,7 @@ public class PipelineConfigurator
 
     protected CQLMessageHandler.MessageConsumer<Message.Request> messageConsumer()
     {
-        return dispatcher::dispatch;
+        return dispatcher;
     }
 
     protected Message.Decoder<Message.Request> messageDecoder()
@@ -363,7 +387,7 @@ public class PipelineConfigurator
         pipeline.addBefore(INITIAL_HANDLER, MESSAGE_COMPRESSOR, Envelope.Compressor.instance);
         pipeline.addBefore(INITIAL_HANDLER, MESSAGE_DECODER, PreV5Handlers.ProtocolDecoder.instance);
         pipeline.addBefore(INITIAL_HANDLER, MESSAGE_ENCODER, PreV5Handlers.ProtocolEncoder.instance);
-        pipeline.addBefore(INITIAL_HANDLER, LEGACY_MESSAGE_PROCESSOR, new PreV5Handlers.LegacyDispatchHandler(dispatcher, limits));
+        pipeline.addBefore(INITIAL_HANDLER, LEGACY_MESSAGE_PROCESSOR, new PreV5Handlers.LegacyDispatchHandler(dispatcher, queueBackpressure, limits));
         pipeline.remove(INITIAL_HANDLER);
         onNegotiationComplete(pipeline);
     }

src/java/org/apache/cassandra/transport/PreV5Handlers.java

@@ -19,6 +19,7 @@
 package org.apache.cassandra.transport;
 
 import java.util.List;
+import java.util.concurrent.TimeUnit;
 
 import com.google.common.base.Predicate;
 
@@ -59,6 +60,8 @@ public class PreV5Handlers
         private final Dispatcher dispatcher;
         private final ClientResourceLimits.Allocator endpointPayloadTracker;
 
+        private final QueueBackpressure queueBackpressure;
+
         /**
          * Current count of *request* bytes that are live on the channel.
          * <p>
@@ -69,9 +72,10 @@ public class PreV5Handlers
         /** The cause of the current connection pause, or {@link Overload#NONE} if it is unpaused. */
         private Overload backpressure = Overload.NONE;
 
-        LegacyDispatchHandler(Dispatcher dispatcher, ClientResourceLimits.Allocator endpointPayloadTracker)
+        LegacyDispatchHandler(Dispatcher dispatcher, QueueBackpressure queueBackpressure, ClientResourceLimits.Allocator endpointPayloadTracker)
         {
             this.dispatcher = dispatcher;
+            this.queueBackpressure = queueBackpressure;
             this.endpointPayloadTracker = endpointPayloadTracker;
         }
 
@@ -152,12 +156,19 @@ public class PreV5Handlers
                     discardAndThrow(request, requestSize, Overload.BYTES_IN_FLIGHT);
                 }
 
+                Overload backpressure = Overload.NONE;
                 if (DatabaseDescriptor.getNativeTransportRateLimitingEnabled() && !GLOBAL_REQUEST_LIMITER.tryReserve())
+                    backpressure = Overload.REQUESTS;
+                else if (!dispatcher.hasQueueCapacity())
+                    backpressure = Overload.QUEUE_TIME;
+
+                if (backpressure != Overload.NONE)
                 {
                     // We've already allocated against the payload tracker here, so release those resources.
                     endpointPayloadTracker.release(requestSize);
-                    discardAndThrow(request, requestSize, Overload.REQUESTS);
+                    discardAndThrow(request, requestSize, backpressure);
                 }
+
             }
             else
             {
@@ -172,20 +183,33 @@ public class PreV5Handlers
                     backpressure = Overload.BYTES_IN_FLIGHT;
                 }
 
+                long delay = -1;
+
                 if (DatabaseDescriptor.getNativeTransportRateLimitingEnabled())
                 {
                     // Reserve a permit even if we've already triggered backpressure on bytes in flight.
-                    long delay = GLOBAL_REQUEST_LIMITER.reserveAndGetDelay(RATE_LIMITER_DELAY_UNIT);
+                    delay = GLOBAL_REQUEST_LIMITER.reserveAndGetDelay(RATE_LIMITER_DELAY_UNIT);
                     
                     // If we've already triggered backpressure on bytes in flight, no further action is necessary.
                     if (backpressure == Overload.NONE && delay > 0)
-                    {
-                        pauseConnection(ctx);
-                        
-                        // A permit isn't immediately available, so schedule an unpause for when it is.
-                        ctx.channel().eventLoop().schedule(() -> unpauseConnection(ctx.channel().config()), delay, RATE_LIMITER_DELAY_UNIT);
                         backpressure = Overload.REQUESTS;
-                    }
+                }
+
+                if (backpressure == Overload.NONE && !dispatcher.hasQueueCapacity())
+                {
+                    delay = queueBackpressure.markAndGetDelay(RATE_LIMITER_DELAY_UNIT);
+
+                    if (delay > 0)
+                        backpressure = Overload.QUEUE_TIME;
+                }
+
+                if (delay > 0)
+                {
+                    assert backpressure != Overload.NONE;
+                    pauseConnection(ctx);
+
+                    // A permit isn't immediately available, so schedule an unpause for when it is.
+                    ctx.channel().eventLoop().schedule(() -> unpauseConnection(ctx.channel().config()), delay, RATE_LIMITER_DELAY_UNIT);
                 }
             }
         }
@@ -219,15 +243,30 @@ public class PreV5Handlers
                          requestSize, channelPayloadBytesInFlight, endpointPayloadTracker,
                          GLOBAL_REQUEST_LIMITER, request);
 
-            OverloadedException exception = overload == Overload.REQUESTS
-                    ? new OverloadedException(String.format("Request breached global limit of %d requests/second. Server is " +
-                                                            "currently in an overloaded state and cannot accept more requests.",
-                                                            GLOBAL_REQUEST_LIMITER.getRate()))
-                    : new OverloadedException(String.format("Request breached limit on bytes in flight. (%s)) " +
-                                                            "Server is currently in an overloaded state and cannot accept more requests.",
 
-                    endpointPayloadTracker));
-            
+            OverloadedException exception;
+            switch (overload)
+            {
+                case REQUESTS:
+                    exception = new OverloadedException(String.format("Request breached global limit of %d requests/second. Server is " +
+                                                                      "currently in an overloaded state and cannot accept more requests.",
+                                                                      GLOBAL_REQUEST_LIMITER.getRate()));
+                    break;
+                case BYTES_IN_FLIGHT:
+                    exception = new OverloadedException(String.format("Request breached limit on bytes in flight. (%s)) " +
+                                                                      "Server is currently in an overloaded state and cannot accept more requests.",
+
+                                                                      endpointPayloadTracker));
+                    break;
+                case QUEUE_TIME:
+                    exception = new OverloadedException(String.format("Request has spent over %s time of the maximum timeout %dms in the queue",
+                                                                      DatabaseDescriptor.getNativeTransportQueueMaxItemAgeThreshold(),
+                                                                      DatabaseDescriptor.getNativeTransportTimeout(TimeUnit.MILLISECONDS)));
+                    break;
+                default:
+                    throw new IllegalArgumentException(String.format("Can't create an exception from %s", overload));
+            }
+
             throw ErrorMessage.wrap(exception, request.getSource().header.streamId);
         }
 

src/java/org/apache/cassandra/transport/QueueBackpressure.java

@@ -0,0 +1,148 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.transport;
+
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.LongSupplier;
+
+import com.google.common.annotations.VisibleForTesting;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+
+import static org.apache.cassandra.utils.MonotonicClock.Global.preciseTime;
+
+/**
+ * Native Queue Backpressure mechanism. If the queue fills up above {@link DatabaseDescriptor#getNativeTransportQueueMaxItemAgeThreshold()}.
+ * In other words, request has been sitting more than a % of {@link DatabaseDescriptor#getNativeTransportQueueMaxItemAgeThreshold()} in the queue,
+ * we start an incident.
+ * <p>
+ * Incident starts by marking the incident time and raising severity level to 1. Each time we observe an old item in the head of
+ * the queue, we first bump then number of times we have applied the backpressure. After bumping it 10 times, we increase severity level by 1.
+ * <p>
+ * Backpressure delay applied to the client socket is computed by multiplying the severity level by the minimum delay.
+ * <p>
+ * If we have not seen old requests in the head of the queue for 1 second, we close the incident.
+ * <p>
+ * If the queue remains saturated for a prolonged period (meaning {@link Dispatcher#hasQueueCapacity returns false), the amount of delay
+ * will increase in proportion to the request rate as appliedTimes & severityLevel are incremented. If no new requests are considered
+ * overloaded in this way for a second, the incident will be reset and so the delay will drop back down to {@link Impl#minDelayNanos()}.
+ */
+public interface QueueBackpressure
+{
+    QueueBackpressure NO_OP = timeUnit -> 0;
+
+    QueueBackpressure DEFAULT = new QueueBackpressure()
+    {
+        private final AtomicReference<Impl> state = new AtomicReference<>(noBackpressure(() -> DatabaseDescriptor.getNativeTransportMinBackoffOnQueueOverload(TimeUnit.NANOSECONDS),
+                                                                                         () -> DatabaseDescriptor.getNativeTransportMaxBackoffOnQueueOverload(TimeUnit.NANOSECONDS)));
+
+        public long markAndGetDelay(TimeUnit timeUnit)
+        {
+            return state.updateAndGet(Impl::mark).delay(timeUnit);
+        }
+    };
+
+    long markAndGetDelay(TimeUnit timeUnit);
+
+
+    static Impl noBackpressure(LongSupplier minDelayNanos, LongSupplier maxDelayNanos)
+    {
+        return new Impl(minDelayNanos, maxDelayNanos,
+                        -1, 0, 0);
+    }
+
+    @VisibleForTesting
+    class Impl
+    {
+        private final long appliedAt;
+
+        private final int severityLevel;
+        private final int appliedTimes;
+
+        private final LongSupplier minDelayNanos;
+        private final LongSupplier maxDelayNanos;
+
+        @VisibleForTesting
+        public Impl(LongSupplier minDelayNanos, LongSupplier maxDelayNanos, long appliedAt, int severityLevel, int appliedTimes)
+        {
+            this.minDelayNanos = minDelayNanos;
+            this.maxDelayNanos = maxDelayNanos;
+            this.appliedAt = appliedAt;
+            this.severityLevel = severityLevel;
+            this.appliedTimes = appliedTimes;
+        }
+
+
+        public Impl mark()
+        {
+            return mark(preciseTime.now());
+        }
+
+        @VisibleForTesting
+        public Impl mark(long now)
+        {
+            // Last time we have applied backpressure was over a second ago, consider this a new incident
+            if (appliedAt > 0 && now - appliedAt >= TimeUnit.SECONDS.toNanos(1))
+            {
+                return new Impl(minDelayNanos, maxDelayNanos, now, 1, 1);
+            }
+            // Continuing incident: apply backpressure but do not bump severity level yet
+            else if (appliedTimes < 10)
+            {
+                return new Impl(minDelayNanos, maxDelayNanos, now, severityLevel == 0 ? 1 : severityLevel, appliedTimes + 1);
+            }
+            //
+            else
+            {
+                return new Impl(minDelayNanos, maxDelayNanos, now, severityLevel + 1, 1);
+            }
+        }
+
+        public long appliedAt()
+        {
+            return appliedAt;
+        }
+
+        public long delay(TimeUnit timeUnit)
+        {
+            return timeUnit.convert(Math.min(maxDelayNanos(), severityLevel * minDelayNanos()), TimeUnit.NANOSECONDS);
+        }
+
+        public long minDelayNanos()
+        {
+            return minDelayNanos.getAsLong();
+        }
+
+        public long maxDelayNanos()
+        {
+            return maxDelayNanos.getAsLong();
+        }
+
+        public String toString()
+        {
+            return "QueueBackpressure{" +
+                   "appliedAgo=" + (appliedAt == -1 ? "never" : TimeUnit.NANOSECONDS.toMillis(preciseTime.now() - appliedAt)) +
+                   ", severityLevel=" + severityLevel +
+                   ", appliedTimes=" + appliedTimes +
+                   ", currentDelay=" + delay(TimeUnit.MILLISECONDS) + "ms" +
+                   '}';
+        }
+    }
+}

src/java/org/apache/cassandra/transport/SimpleClient.java

@@ -446,8 +446,17 @@ public class SimpleClient implements Closeable
             FrameEncoder frameEncoder = frameEncoder(ctx);
             FrameEncoder.PayloadAllocator payloadAllocator = frameEncoder.allocator();
 
-            CQLMessageHandler.MessageConsumer<Message.Response> responseConsumer = (c, message, converter, backpressured) -> {
-                responseHandler.handleResponse(c, message);
+            CQLMessageHandler.MessageConsumer<Message.Response> responseConsumer = new CQLMessageHandler.MessageConsumer<Message.Response>()
+            {
+                public void dispatch(Channel channel, Message.Response message, Dispatcher.FlushItemConverter toFlushItem, Overload backpressure)
+                {
+                    responseHandler.handleResponse(channel, message);
+                }
+
+                public boolean hasQueueCapacity()
+                {
+                    return true;
+                }
             };
 
             CQLMessageHandler.ErrorHandler errorHandler = (error) -> {
@@ -502,6 +511,7 @@ public class SimpleClient implements Closeable
                                         responseConsumer,
                                         payloadAllocator,
                                         queueCapacity,
+                                        QueueBackpressure.NO_OP,
                                         resources,
                                         handler -> {},
                                         errorHandler,
@@ -540,7 +550,7 @@ public class SimpleClient implements Closeable
             pipeline.remove(this);
 
             Message.Response message = messageDecoder.decode(ctx.channel(), response);
-            responseConsumer.accept(channel, message, (ch, req, resp) -> null, Overload.NONE);
+            responseConsumer.dispatch(channel, message, (ch, req, resp) -> null, Overload.NONE);
         }
 
         private FrameDecoder frameDecoder(ChannelHandlerContext ctx, BufferPoolAllocator allocator)

src/java/org/apache/cassandra/transport/messages/AuthResponse.java

@@ -69,7 +69,7 @@ public class AuthResponse extends Message.Request
     }
 
     @Override
-    protected Response execute(QueryState queryState, long queryStartNanoTime, boolean traceRequest)
+    protected Response execute(QueryState queryState, Dispatcher.RequestTime requestTime, boolean traceRequest)
     {
         try
         {

src/java/org/apache/cassandra/transport/messages/BatchMessage.java

@@ -40,6 +40,7 @@ import org.apache.cassandra.service.ClientState;
 import org.apache.cassandra.service.QueryState;
 import org.apache.cassandra.tracing.Tracing;
 import org.apache.cassandra.transport.CBUtil;
+import org.apache.cassandra.transport.Dispatcher;
 import org.apache.cassandra.transport.Message;
 import org.apache.cassandra.transport.ProtocolException;
 import org.apache.cassandra.transport.ProtocolVersion;
@@ -169,7 +170,7 @@ public class BatchMessage extends Message.Request
     }
 
     @Override
-    protected Message.Response execute(QueryState state, long queryStartNanoTime, boolean traceRequest)
+    protected Message.Response execute(QueryState state, Dispatcher.RequestTime requestTime, boolean traceRequest)
     {
         List<QueryHandler.Prepared> prepared = null;
         try
@@ -226,7 +227,7 @@ public class BatchMessage extends Message.Request
             BatchStatement batch = new BatchStatement(batchType, VariableSpecifications.empty(), statements, Attributes.none());
 
             long queryTime = currentTimeMillis();
-            Message.Response response = handler.processBatch(batch, state, batchOptions, getCustomPayload(), queryStartNanoTime);
+            Message.Response response = handler.processBatch(batch, state, batchOptions, getCustomPayload(), requestTime);
             if (queries != null)
                 QueryEvents.instance.notifyBatchSuccess(batchType, statements, queries, values, options, state, queryTime, response);
             return response;

src/java/org/apache/cassandra/transport/messages/ExecuteMessage.java

@@ -35,6 +35,7 @@ import org.apache.cassandra.service.ClientState;
 import org.apache.cassandra.service.QueryState;
 import org.apache.cassandra.tracing.Tracing;
 import org.apache.cassandra.transport.CBUtil;
+import org.apache.cassandra.transport.Dispatcher;
 import org.apache.cassandra.transport.Message;
 import org.apache.cassandra.transport.ProtocolException;
 import org.apache.cassandra.transport.ProtocolVersion;
@@ -125,7 +126,7 @@ public class ExecuteMessage extends Message.Request
     }
 
     @Override
-    protected Message.Response execute(QueryState state, long queryStartNanoTime, boolean traceRequest)
+    protected Message.Response execute(QueryState state, Dispatcher.RequestTime requestTime, boolean traceRequest)
     {
         QueryHandler.Prepared prepared = null;
         try
@@ -148,7 +149,6 @@ public class ExecuteMessage extends Message.Request
                 nospam.error(msg);
             }
 
-
             CQLStatement statement = prepared.statement;
             options.prepare(statement.getBindVariables());
 
@@ -164,7 +164,7 @@ public class ExecuteMessage extends Message.Request
 
             long requestStartTime = currentTimeMillis();
 
-            Message.Response response = handler.processPrepared(statement, state, queryOptions, getCustomPayload(), queryStartNanoTime);
+            Message.Response response = handler.processPrepared(statement, state, queryOptions, getCustomPayload(), requestTime);
 
             QueryEvents.instance.notifyExecuteSuccess(prepared.statement, prepared.rawCQLStatement, options, state, requestStartTime, response);
 

src/java/org/apache/cassandra/transport/messages/OptionsMessage.java

@@ -27,6 +27,7 @@ import io.netty.buffer.ByteBuf;
 import org.apache.cassandra.cql3.QueryProcessor;
 import org.apache.cassandra.service.QueryState;
 import org.apache.cassandra.transport.Compressor;
+import org.apache.cassandra.transport.Dispatcher;
 import org.apache.cassandra.transport.Message;
 import org.apache.cassandra.transport.ProtocolVersion;
 
@@ -58,7 +59,7 @@ public class OptionsMessage extends Message.Request
     }
 
     @Override
-    protected Message.Response execute(QueryState state, long queryStartNanoTime, boolean traceRequest)
+    protected Message.Response execute(QueryState state, Dispatcher.RequestTime requestTime, boolean traceRequest)
     {
         List<String> cqlVersions = new ArrayList<String>();
         cqlVersions.add(QueryProcessor.CQL_VERSION.toString());

src/java/org/apache/cassandra/transport/messages/PrepareMessage.java

@@ -31,6 +31,7 @@ import org.apache.cassandra.service.ClientState;
 import org.apache.cassandra.service.QueryState;
 import org.apache.cassandra.tracing.Tracing;
 import org.apache.cassandra.transport.CBUtil;
+import org.apache.cassandra.transport.Dispatcher;
 import org.apache.cassandra.transport.Message;
 import org.apache.cassandra.transport.ProtocolVersion;
 import org.apache.cassandra.utils.JVMStabilityInspector;
@@ -114,7 +115,7 @@ public class PrepareMessage extends Message.Request
     }
 
     @Override
-    protected Message.Response execute(QueryState state, long queryStartNanoTime, boolean traceRequest)
+    protected Message.Response execute(QueryState state, Dispatcher.RequestTime requestTime, boolean traceRequest)
     {
         try
         {

src/java/org/apache/cassandra/transport/messages/QueryMessage.java

@@ -30,6 +30,7 @@ import org.apache.cassandra.service.ClientState;
 import org.apache.cassandra.service.QueryState;
 import org.apache.cassandra.tracing.Tracing;
 import org.apache.cassandra.transport.CBUtil;
+import org.apache.cassandra.transport.Dispatcher;
 import org.apache.cassandra.transport.Message;
 import org.apache.cassandra.transport.ProtocolException;
 import org.apache.cassandra.transport.ProtocolVersion;
@@ -98,7 +99,7 @@ public class QueryMessage extends Message.Request
     }
 
     @Override
-    protected Message.Response execute(QueryState state, long queryStartNanoTime, boolean traceRequest)
+    protected Message.Response execute(QueryState state, Dispatcher.RequestTime requestTime, boolean traceRequest)
     {
         CQLStatement statement = null;
         try
@@ -113,7 +114,7 @@ public class QueryMessage extends Message.Request
 
             QueryHandler queryHandler = ClientState.getCQLQueryHandler();
             statement = queryHandler.parse(query, state, options);
-            Message.Response response = queryHandler.process(statement, state, options, getCustomPayload(), queryStartNanoTime);
+            Message.Response response = queryHandler.process(statement, state, options, getCustomPayload(), requestTime);
             QueryEvents.instance.notifyQuerySuccess(statement, query, options, state, queryStartTime, response);
 
             if (options.skipMetadata() && response instanceof ResultMessage.Rows)

src/java/org/apache/cassandra/transport/messages/RegisterMessage.java

@@ -63,7 +63,7 @@ public class RegisterMessage extends Message.Request
     }
 
     @Override
-    protected Response execute(QueryState state, long queryStartNanoTime, boolean traceRequest)
+    protected Response execute(QueryState state, Dispatcher.RequestTime requestTime, boolean traceRequest)
     {
         assert connection instanceof ServerConnection;
         Connection.Tracker tracker = connection.getTracker();

src/java/org/apache/cassandra/transport/messages/StartupMessage.java

@@ -68,7 +68,7 @@ public class StartupMessage extends Message.Request
     }
 
     @Override
-    protected Message.Response execute(QueryState state, long queryStartNanoTime, boolean traceRequest)
+    protected Message.Response execute(QueryState state, Dispatcher.RequestTime requestTime, boolean traceRequest)
     {
         String cqlVersion = options.get(CQL_VERSION);
         if (cqlVersion == null)

test/burn/org/apache/cassandra/transport/DriverBurnTest.java

@@ -78,7 +78,7 @@ public class DriverBurnTest extends CQLTester
             {
                 QueryMessage queryMessage = QueryMessage.codec.decode(body, version);
                 return new QueryMessage(queryMessage.query, queryMessage.options) {
-                    protected Message.Response execute(QueryState state, long queryStartNanoTime, boolean traceRequest)
+                    protected Message.Response execute(QueryState state, Dispatcher.RequestTime requestTime, boolean traceRequest)
                     {
                         try
                         {
@@ -89,7 +89,7 @@ public class DriverBurnTest extends CQLTester
                         catch (NumberFormatException e)
                         {
                             // for the requests driver issues under the hood
-                            return super.execute(state, queryStartNanoTime, traceRequest);
+                            return super.execute(state, requestTime, traceRequest);
                         }
                     }
                 };
@@ -338,7 +338,7 @@ public class DriverBurnTest extends CQLTester
             {
                 QueryMessage queryMessage = QueryMessage.codec.decode(body, version);
                 return new QueryMessage(queryMessage.query, queryMessage.options) {
-                    protected Message.Response execute(QueryState state, long queryStartNanoTime, boolean traceRequest)
+                    protected Message.Response execute(QueryState state, Dispatcher.RequestTime requestTime, boolean traceRequest)
                     {
                         try
                         {
@@ -348,7 +348,7 @@ public class DriverBurnTest extends CQLTester
                         catch (NumberFormatException e)
                         {
                             // for the requests driver issues under the hood
-                            return super.execute(state, queryStartNanoTime, traceRequest);
+                            return super.execute(state, requestTime, traceRequest);
                         }
                     }
                 };

test/burn/org/apache/cassandra/transport/SimpleClientBurnTest.java

@@ -113,7 +113,8 @@ public class SimpleClientBurnTest
                 QueryMessage queryMessage = QueryMessage.codec.decode(body, version);
                 return new QueryMessage(queryMessage.query, queryMessage.options)
                 {
-                    protected Message.Response execute(QueryState state, long queryStartNanoTime, boolean traceRequest)
+                    @Override
+                    protected Message.Response execute(QueryState state, Dispatcher.RequestTime requestTime, boolean traceRequest)
                     {
                         int idx = Integer.parseInt(queryMessage.query);
                         SizeCaps caps = idx % largeMessageFrequency == 0 ? largeMessageCap : smallMessageCap;

test/burn/org/apache/cassandra/transport/SimpleClientPerfTest.java

@@ -173,7 +173,8 @@ public class SimpleClientPerfTest
                 QueryMessage queryMessage = QueryMessage.codec.decode(body, version);
                 return new QueryMessage(queryMessage.query, queryMessage.options)
                 {
-                    protected Message.Response execute(QueryState state, long queryStartNanoTime, boolean traceRequest)
+                    @Override
+                    protected Message.Response execute(QueryState state, Dispatcher.RequestTime requestTime, boolean traceRequest)
                     {
                         int idx = Integer.parseInt(queryMessage.query); // unused
                         return generateRows(idx, responseCaps);

test/distributed/org/apache/cassandra/distributed/impl/Coordinator.java

@@ -18,7 +18,6 @@
 
 package org.apache.cassandra.distributed.impl;
 
-import java.net.InetSocketAddress;
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.Collections;
@@ -43,14 +42,14 @@ import org.apache.cassandra.service.ClientState;
 import org.apache.cassandra.service.ClientWarn;
 import org.apache.cassandra.service.QueryState;
 import org.apache.cassandra.service.reads.thresholds.CoordinatorWarnings;
+import org.apache.cassandra.transport.Dispatcher;
 import org.apache.cassandra.transport.ProtocolVersion;
 import org.apache.cassandra.tracing.Tracing;
 import org.apache.cassandra.transport.messages.ResultMessage;
 import org.apache.cassandra.utils.ByteBufferUtil;
-import org.apache.cassandra.utils.FBUtilities;
 import org.apache.cassandra.utils.TimeUUID;
 
-import static org.apache.cassandra.utils.Clock.Global.nanoTime;
+import static org.apache.cassandra.distributed.impl.CoordinatorHelper.toCassandraSerialCL;
 
 public class Coordinator implements ICoordinator
 {
@@ -93,10 +92,6 @@ public class Coordinator implements ICoordinator
         }
     }
 
-    protected static org.apache.cassandra.db.ConsistencyLevel toCassandraSerialCL(ConsistencyLevel cl)
-    {
-        return toCassandraCL(cl == null ? ConsistencyLevel.SERIAL : cl);
-    }
 
     public static SimpleQueryResult unsafeExecuteInternal(String query, ConsistencyLevel consistencyLevel, Object[] boundValues)
     {
@@ -105,7 +100,7 @@ public class Coordinator implements ICoordinator
 
     public static SimpleQueryResult unsafeExecuteInternal(String query, ConsistencyLevel serialConsistencyLevel, ConsistencyLevel commitConsistencyLevel, Object[] boundValues)
     {
-        ClientState clientState = makeFakeClientState();
+        ClientState clientState = CoordinatorHelper.makeFakeClientState();
         CQLStatement prepared = QueryProcessor.getStatement(query, clientState);
         List<ByteBuffer> boundBBValues = new ArrayList<>();
         for (Object boundValue : boundValues)
@@ -128,7 +123,7 @@ public class Coordinator implements ICoordinator
                                                        toCassandraSerialCL(serialConsistencyLevel),
                                                        ProtocolVersion.CURRENT,
                                                        null),
-                                   nanoTime());
+                                   Dispatcher.RequestTime.forImmediateExecution());
             // Collect warnings reported during the query.
             CoordinatorWarnings.done();
             if (res != null)
@@ -161,7 +156,7 @@ public class Coordinator implements ICoordinator
     @Override
     public SimpleQueryResult executeWithResult(String query, ConsistencyLevel serialConsistencyLevel, ConsistencyLevel commitConsistencyLevel, Object... boundValues)
     {
-        return instance.sync(() -> unsafeExecuteInternal(query, serialConsistencyLevel, commitConsistencyLevel, boundValues)).call();
+        return instance.sync(() -> CoordinatorHelper.unsafeExecuteInternal(query, serialConsistencyLevel, commitConsistencyLevel, boundValues)).call();
     }
 
     @Override
@@ -171,7 +166,7 @@ public class Coordinator implements ICoordinator
             throw new IllegalArgumentException("Page size should be strictly positive but was " + pageSize);
 
         return instance.sync(() -> {
-            ClientState clientState = makeFakeClientState();
+            ClientState clientState = CoordinatorHelper.makeFakeClientState();
             ConsistencyLevel consistencyLevel = ConsistencyLevel.valueOf(consistencyLevelOrigin.name());
             CQLStatement prepared = QueryProcessor.getStatement(query, clientState);
             final List<ByteBuffer> boundBBValues = new ArrayList<>();
@@ -181,7 +176,7 @@ public class Coordinator implements ICoordinator
             prepared.validate(clientState);
             assert prepared instanceof SelectStatement : "Only SELECT statements can be executed with paging";
 
-            long nanoTime = nanoTime();
+            Dispatcher.RequestTime requestTime = Dispatcher.RequestTime.forImmediateExecution();
             SelectStatement selectStatement = (SelectStatement) prepared;
 
             QueryState queryState = new QueryState(clientState);
@@ -195,9 +190,9 @@ public class Coordinator implements ICoordinator
                                                               selectStatement.keyspace());
 
 
-            ResultMessage.Rows initialRows = selectStatement.execute(queryState, initialOptions, nanoTime);
+            ResultMessage.Rows initialRows = selectStatement.execute(queryState, initialOptions, requestTime);
             Iterator<Object[]> iter = new Iterator<Object[]>() {
-                ResultMessage.Rows rows = selectStatement.execute(queryState, initialOptions, nanoTime);
+                ResultMessage.Rows rows = selectStatement.execute(queryState, initialOptions, requestTime);
                 Iterator<Object[]> iter = RowUtil.toIter(rows);
 
                 public boolean hasNext()
@@ -217,7 +212,7 @@ public class Coordinator implements ICoordinator
                                                                    ProtocolVersion.CURRENT,
                                                                    selectStatement.keyspace());
 
-                    rows = selectStatement.execute(queryState, nextOptions, nanoTime);
+                    rows = selectStatement.execute(queryState, nextOptions, requestTime);
                     iter = Iterators.forArray(RowUtil.toObjects(initialRows.result.metadata.names, rows.result.rows));
 
                     return hasNext();
@@ -233,8 +228,4 @@ public class Coordinator implements ICoordinator
         }).call();
     }
 
-    public static ClientState makeFakeClientState()
-    {
-        return ClientState.forExternalCalls(new InetSocketAddress(FBUtilities.getJustLocalAddress(), 9042));
-    }
 }

test/distributed/org/apache/cassandra/distributed/impl/CoordinatorHelper.java

@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.impl;
+
+import java.net.InetSocketAddress;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.cassandra.cql3.CQLStatement;
+import org.apache.cassandra.cql3.QueryOptions;
+import org.apache.cassandra.cql3.QueryProcessor;
+import org.apache.cassandra.distributed.api.ConsistencyLevel;
+import org.apache.cassandra.distributed.api.SimpleQueryResult;
+import org.apache.cassandra.service.ClientState;
+import org.apache.cassandra.service.ClientWarn;
+import org.apache.cassandra.service.QueryState;
+import org.apache.cassandra.service.reads.thresholds.CoordinatorWarnings;
+import org.apache.cassandra.transport.Dispatcher;
+import org.apache.cassandra.transport.ProtocolVersion;
+import org.apache.cassandra.transport.messages.ResultMessage;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.FBUtilities;
+
+import static org.apache.cassandra.distributed.impl.Coordinator.toCassandraCL;
+
+public class CoordinatorHelper
+{
+    public static SimpleQueryResult unsafeExecuteInternal(String query, ConsistencyLevel serialConsistencyLevel, ConsistencyLevel commitConsistencyLevel, Object[] boundValues)
+    {
+        return unsafeExecuteInternal(query, serialConsistencyLevel, commitConsistencyLevel, Dispatcher.RequestTime.forImmediateExecution(), boundValues);
+    }
+
+    public static SimpleQueryResult unsafeExecuteInternal(String query, ConsistencyLevel serialConsistencyLevel, ConsistencyLevel commitConsistencyLevel, Dispatcher.RequestTime requestTime, Object... boundValues)
+    {
+        ClientState clientState =  makeFakeClientState();
+        CQLStatement prepared = QueryProcessor.getStatement(query, clientState);
+        List<ByteBuffer> boundBBValues = new ArrayList<>();
+        for (Object boundValue : boundValues)
+            boundBBValues.add(ByteBufferUtil.objectToBytes(boundValue));
+
+        prepared.validate(QueryState.forInternalCalls().getClientState());
+        prepared.validate(clientState);
+
+        // Start capturing warnings on this thread. Note that this will implicitly clear out any previous
+        // warnings as it sets a new State instance on the ThreadLocal.
+        ClientWarn.instance.captureWarnings();
+        CoordinatorWarnings.init();
+        try
+        {
+            ResultMessage res = prepared.execute(QueryState.forInternalCalls(),
+                                                 QueryOptions.create(toCassandraCL(commitConsistencyLevel),
+                                                                     boundBBValues,
+                                                                     false,
+                                                                     Integer.MAX_VALUE,
+                                                                     null,
+                                                                     toCassandraSerialCL(serialConsistencyLevel),
+                                                                     ProtocolVersion.CURRENT,
+                                                                     null),
+                                                 requestTime);
+            // Collect warnings reported during the query.
+            CoordinatorWarnings.done();
+            if (res != null)
+                res.setWarnings(ClientWarn.instance.getWarnings());
+
+            return RowUtil.toQueryResult(res);
+        }
+        catch (Exception | Error e)
+        {
+            CoordinatorWarnings.done();
+            throw e;
+        }
+        finally
+        {
+            CoordinatorWarnings.reset();
+            ClientWarn.instance.resetWarnings();
+        }
+    }
+
+    public static ClientState makeFakeClientState()
+    {
+        return ClientState.forExternalCalls(new InetSocketAddress(FBUtilities.getJustLocalAddress(), 9042));
+    }
+
+
+    protected static org.apache.cassandra.db.ConsistencyLevel toCassandraSerialCL(ConsistencyLevel cl)
+    {
+        return toCassandraCL(cl == null ? ConsistencyLevel.SERIAL : cl);
+    }
+}

test/distributed/org/apache/cassandra/distributed/impl/Query.java

@@ -30,13 +30,12 @@ import org.apache.cassandra.distributed.api.IIsolatedExecutor;
 import org.apache.cassandra.service.ClientState;
 import org.apache.cassandra.service.ClientWarn;
 import org.apache.cassandra.service.QueryState;
+import org.apache.cassandra.transport.Dispatcher;
 import org.apache.cassandra.transport.ProtocolVersion;
 import org.apache.cassandra.transport.messages.ResultMessage;
 import org.apache.cassandra.utils.ByteBufferUtil;
 import org.apache.cassandra.utils.FBUtilities;
 
-import static org.apache.cassandra.utils.Clock.Global.nanoTime;
-
 public class Query implements IIsolatedExecutor.SerializableCallable<Object[][]>
 {
     private static final long serialVersionUID = 1L;
@@ -60,7 +59,7 @@ public class Query implements IIsolatedExecutor.SerializableCallable<Object[][]>
     {
         ConsistencyLevel commitConsistency = toCassandraCL(commitConsistencyOrigin);
         ConsistencyLevel serialConsistency = serialConsistencyOrigin == null ? null : toCassandraCL(serialConsistencyOrigin);
-        ClientState clientState = Coordinator.makeFakeClientState();
+        ClientState clientState = CoordinatorHelper.makeFakeClientState();
         CQLStatement prepared = QueryProcessor.getStatement(query, clientState);
         List<ByteBuffer> boundBBValues = new ArrayList<>();
         for (Object boundValue : boundValues)
@@ -83,7 +82,7 @@ public class Query implements IIsolatedExecutor.SerializableCallable<Object[][]>
                                                                  null,
                                                                  timestamp,
                                                                  FBUtilities.nowInSeconds()),
-                                             nanoTime());
+                                             Dispatcher.RequestTime.forImmediateExecution());
 
         // Collect warnings reported during the query.
         if (res != null)

test/distributed/org/apache/cassandra/distributed/test/ByteBuddyExamplesTest.java

@@ -36,6 +36,7 @@ import org.apache.cassandra.cql3.statements.SelectStatement;
 import org.apache.cassandra.distributed.Cluster;
 import org.apache.cassandra.distributed.api.ConsistencyLevel;
 import org.apache.cassandra.service.QueryState;
+import org.apache.cassandra.transport.Dispatcher;
 import org.apache.cassandra.transport.messages.ResultMessage;
 import org.apache.cassandra.transport.messages.ResultMessage.Rows;
 
@@ -116,7 +117,7 @@ public class ByteBuddyExamplesTest extends TestBaseImpl
                            .load(cl, ClassLoadingStrategy.Default.INJECTION);
         }
 
-        public static ResultMessage.Rows execute(QueryState state, QueryOptions options, long queryStartNanoTime, @SuperCall Callable<ResultMessage.Rows> r) throws Exception
+        public static ResultMessage.Rows execute(QueryState state, QueryOptions options, Dispatcher.RequestTime requestTime, @SuperCall Callable<ResultMessage.Rows> r) throws Exception
         {
             Rows res = r.call();
 

test/distributed/org/apache/cassandra/distributed/test/FailingResponseDoesNotLogTest.java

@@ -44,6 +44,7 @@ import org.apache.cassandra.exceptions.RequestExecutionException;
 import org.apache.cassandra.exceptions.RequestValidationException;
 import org.apache.cassandra.service.ClientState;
 import org.apache.cassandra.service.QueryState;
+import org.apache.cassandra.transport.Dispatcher;
 import org.apache.cassandra.transport.SimpleClient;
 import org.apache.cassandra.transport.messages.ResultMessage;
 import org.apache.cassandra.utils.MD5Digest;
@@ -107,7 +108,7 @@ public class FailingResponseDoesNotLogTest extends TestBaseImpl
         }
 
         @Override
-        public ResultMessage process(CQLStatement statement, QueryState state, QueryOptions options, Map<String, ByteBuffer> customPayload, long queryStartNanoTime) throws RequestExecutionException, RequestValidationException
+        public ResultMessage process(CQLStatement statement, QueryState state, QueryOptions options, Map<String, ByteBuffer> customPayload, Dispatcher.RequestTime requestTime) throws RequestExecutionException, RequestValidationException
         {
             throw new AssertionError("reject");
         }
@@ -125,13 +126,13 @@ public class FailingResponseDoesNotLogTest extends TestBaseImpl
         }
 
         @Override
-        public ResultMessage processPrepared(CQLStatement statement, QueryState state, QueryOptions options, Map<String, ByteBuffer> customPayload, long queryStartNanoTime) throws RequestExecutionException, RequestValidationException
+        public ResultMessage processPrepared(CQLStatement statement, QueryState state, QueryOptions options, Map<String, ByteBuffer> customPayload, Dispatcher.RequestTime requestTime) throws RequestExecutionException, RequestValidationException
         {
             throw new AssertionError("reject");
         }
 
         @Override
-        public ResultMessage processBatch(BatchStatement statement, QueryState state, BatchQueryOptions options, Map<String, ByteBuffer> customPayload, long queryStartNanoTime) throws RequestExecutionException, RequestValidationException
+        public ResultMessage processBatch(BatchStatement statement, QueryState state, BatchQueryOptions options, Map<String, ByteBuffer> customPayload, Dispatcher.RequestTime requestTime) throws RequestExecutionException, RequestValidationException
         {
             throw new AssertionError("reject");
         }

test/distributed/org/apache/cassandra/distributed/test/FailureLoggingTest.java

@@ -40,6 +40,7 @@ import org.apache.cassandra.exceptions.UnavailableException;
 import org.apache.cassandra.service.StorageProxy;
 import org.apache.cassandra.service.StorageService;
 import org.apache.cassandra.service.reads.range.RangeCommandIterator;
+import org.apache.cassandra.transport.Dispatcher;
 
 import static net.bytebuddy.matcher.ElementMatchers.named;
 import static org.junit.Assert.assertEquals;
@@ -168,8 +169,8 @@ public class FailureLoggingTest extends TestBaseImpl
 
         @SuppressWarnings("unused")
         public static PartitionIterator fetchRows(List<SinglePartitionReadCommand> commands, 
-                                                  org.apache.cassandra.db.ConsistencyLevel consistencyLevel, 
-                                                  long queryStartNanoTime)
+                                                  org.apache.cassandra.db.ConsistencyLevel consistencyLevel,
+                                                  Dispatcher.RequestTime requestTime)
         {
             throw UnavailableException.create(org.apache.cassandra.db.ConsistencyLevel.ALL, 1, 0);
         }

test/distributed/org/apache/cassandra/distributed/test/MessageTimestampTest.java

@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.test;
+
+import java.util.concurrent.Callable;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import net.bytebuddy.ByteBuddy;
+import net.bytebuddy.dynamic.loading.ClassLoadingStrategy;
+import net.bytebuddy.implementation.MethodDelegation;
+import net.bytebuddy.implementation.bind.annotation.SuperCall;
+import org.apache.cassandra.config.Config;
+import org.apache.cassandra.distributed.Cluster;
+import org.apache.cassandra.distributed.api.ConsistencyLevel;
+import org.apache.cassandra.metrics.InternodeOutboundMetrics;
+import org.apache.cassandra.transport.Dispatcher;
+
+import static net.bytebuddy.matcher.ElementMatchers.named;
+import static org.apache.cassandra.distributed.api.Feature.GOSSIP;
+import static org.apache.cassandra.distributed.api.Feature.NATIVE_PROTOCOL;
+import static org.apache.cassandra.distributed.api.Feature.NETWORK;
+
+public class MessageTimestampTest extends TestBaseImpl
+{
+    @Test
+    public void testFinishInProgressQueries() throws Throwable
+    {
+        ExecutorService executor = Executors.newCachedThreadPool();
+        try (Cluster cluster = init(Cluster.build().withNodes(2)
+                                           .withInstanceInitializer(EnqueuedInThePast::install)
+                                           .withConfig(config -> config.with(GOSSIP, NETWORK, NATIVE_PROTOCOL)
+                                                                       .set("native_transport_max_threads", 2)
+                                                                       .set("read_request_timeout_in_ms", 5000)
+                                                                       .set("cql_start_time", Config.CQLStartTime.QUEUE)
+                                           )
+                                           .start()))
+        {
+            cluster.schemaChange(withKeyspace("CREATE TABLE %s.tbl (pk int PRIMARY KEY, v int)"));
+
+            long expiredBefore = cluster.get(1).callsOnInstance(() -> InternodeOutboundMetrics.totalExpiredCallbacks.getCount()).call();
+
+            cluster.get(1).runOnInstance(() -> {
+                Assert.assertTrue(EnqueuedInThePast.enabled.compareAndSet(false, true));
+            });
+            CompletableFuture.supplyAsync(() -> cluster.coordinator(1).execute("select * from " + KEYSPACE + ".tbl where pk = " + 1, ConsistencyLevel.ALL), executor);
+            CompletableFuture.supplyAsync(() -> cluster.coordinator(1).execute("select * from " + KEYSPACE + ".tbl where pk = " + 2, ConsistencyLevel.ALL), executor) ;
+
+            long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(10);
+            long expiredAfter = 0;
+            while (System.nanoTime() < deadline) {
+                long computed = cluster.get(1).callsOnInstance(() -> InternodeOutboundMetrics.totalExpiredCallbacks.getCount()).call();
+                if (computed > expiredBefore)
+                {
+                    expiredAfter = computed;
+                    break;
+                }
+            }
+
+            Assert.assertTrue(String.format("%d should be larger than %d", expiredAfter, expiredBefore),
+                              expiredAfter > expiredBefore);
+
+
+        }
+    }
+
+    public static class EnqueuedInThePast
+    {
+        static AtomicBoolean enabled = new AtomicBoolean(false);
+
+        static void install(ClassLoader cl, int nodeNumber)
+        {
+            new ByteBuddy().rebase(Dispatcher.RequestTime.class)
+                           .method(named("enqueuedAtNanos"))
+                           .intercept(MethodDelegation.to(EnqueuedInThePast.class))
+                           .make()
+                           .load(cl, ClassLoadingStrategy.Default.INJECTION);
+        }
+
+        public static long enqueuedAtNanos(@SuperCall Callable<Long> r) throws Exception
+        {
+            if (enabled.get())
+            {
+                return System.nanoTime() - TimeUnit.SECONDS.toNanos(30);
+            }
+            return r.call();
+        }
+    }
+}

test/distributed/org/apache/cassandra/distributed/test/ReadSpeculationTest.java

@@ -0,0 +1,209 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.test;
+
+import java.util.concurrent.TimeUnit;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import org.apache.cassandra.concurrent.ScheduledExecutors;
+import org.apache.cassandra.concurrent.ScheduledThreadPoolExecutorPlus;
+import org.apache.cassandra.config.Config;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.distributed.Cluster;
+import org.apache.cassandra.distributed.api.ConsistencyLevel;
+import org.apache.cassandra.distributed.impl.CoordinatorHelper;
+import org.apache.cassandra.exceptions.ReadTimeoutException;
+import org.apache.cassandra.service.CassandraDaemon;
+import org.apache.cassandra.transport.Dispatcher;
+
+import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
+
+public class ReadSpeculationTest extends TestBaseImpl
+{
+
+    @Test
+    public void speculateTest() throws Throwable
+    {
+        try (Cluster cluster = builder().withNodes(3)
+                                        .start())
+        {
+            cluster.get(1).runOnInstance(() -> {
+                // Disable updater since we will force time
+                ((ScheduledThreadPoolExecutorPlus) ScheduledExecutors.optionalTasks).remove(CassandraDaemon.SPECULATION_THRESHOLD_UPDATER);
+            });
+            cluster.schemaChange("CREATE KEYSPACE IF NOT EXISTS " + KEYSPACE + " WITH replication = {'class': 'SimpleStrategy', 'replication_factor': " + 3 + "}");
+            cluster.schemaChange("CREATE TABLE IF NOT EXISTS " + KEYSPACE + ".tbl (pk int, ck int, v int, PRIMARY KEY (pk, ck)) WITH speculative_retry = '2000ms';");
+
+            // force speculation; rely on IP order
+            cluster.filters().allVerbs().from(1).to(2).drop();
+
+
+            cluster.get(1).runOnInstance(() -> {
+
+
+                // Will speculate: have enough time till RPC timeout and client deadline
+                // Request has been submitted 2 seconds ago, and spent 1 second in the queue. Execution has started 1 second ago, and
+                // tracked latency is 3 seconds. Speculation timeout is always computed against the start time, so we will speculate
+                // exactly once, and after 3 - 1 = 2 seconds.
+                new TestScenario(12_000,
+                                 12_000,
+                                 TimeUnit.SECONDS.toMicros(3),
+                                 Config.CQLStartTime.REQUEST,
+                                 TimeUnit.SECONDS.toNanos(2),
+                                 TimeUnit.SECONDS.toNanos(1))
+                .assertWillSpeculate();
+                new TestScenario(12_000,
+                                 12_000,
+                                 TimeUnit.SECONDS.toMicros(3),
+                                 Config.CQLStartTime.QUEUE,
+                                 TimeUnit.SECONDS.toNanos(2),
+                                 TimeUnit.SECONDS.toNanos(1))
+                .assertWillSpeculate();
+
+                // Will not speculate: latency is greater than RPC timeout
+                // Request has been submitted 2 seconds ago, and spent 1 second in the queue. Execution has started 1 second ago, and
+                // tracked latency is 2 seconds. Speculation timeout is always computed against the start time, so we will not speculate
+                // since we only have 1.5 seconds to execute request.
+                new TestScenario(12_000,
+                                 1_500,
+                                 TimeUnit.SECONDS.toMicros(2),
+                                 Config.CQLStartTime.REQUEST,
+                                 TimeUnit.SECONDS.toNanos(3),
+                                 TimeUnit.SECONDS.toNanos(1))
+                .assertWillNotSpeculate();
+                new TestScenario(12_000,
+                                 1_500,
+                                 TimeUnit.SECONDS.toMicros(2),
+                                 Config.CQLStartTime.QUEUE,
+                                 TimeUnit.SECONDS.toNanos(3),
+                                 TimeUnit.SECONDS.toNanos(1))
+                .assertWillNotSpeculate();
+                // Will not speculate: speculating will put us over the client deadline, even though RPC timeout is far enough
+                // Request has been submitted 11 seconds ago, and spent 10 second in the queue. Execution has started 1 second ago, and
+                // tracked latency is 2 seconds. Speculation timeout is always computed against the start time, so we will not speculate
+                // since there is only
+                new TestScenario(12_000,
+                                 10_000,
+                                 TimeUnit.SECONDS.toMicros(2),
+                                 Config.CQLStartTime.REQUEST,
+                                 TimeUnit.SECONDS.toNanos(11),
+                                 TimeUnit.SECONDS.toNanos(1))
+                .assertWillNotSpeculate();
+                new TestScenario(12_000,
+                                 1_500,
+                                 TimeUnit.SECONDS.toMicros(2),
+                                 Config.CQLStartTime.QUEUE,
+                                 TimeUnit.SECONDS.toNanos(11),
+                                 TimeUnit.SECONDS.toNanos(1))
+                .assertWillNotSpeculate();
+                new TestScenario(12_000,
+                                 2000,
+                                 TimeUnit.SECONDS.toMicros(3),
+                                 Config.CQLStartTime.QUEUE,
+                                 TimeUnit.SECONDS.toNanos(9),
+                                 TimeUnit.SECONDS.toNanos(0))
+                .assertWillNotSpeculate();
+            });
+        }
+    }
+
+    private static class TestScenario
+    {
+        final long nativeTimeoutMs;
+        final long rpcTimeoutMs;
+        // After how much time speculation should be triggered
+        final long speculationTimeoutMicros;
+
+        final Config.CQLStartTime cqlStartTime;
+
+        final long enqueuedNsAgo;
+        final long startedNsAgo;
+
+        TestScenario(long nativeTimeoutMs, long rpcTimeoutMs, long speculationTimeoutMicros, Config.CQLStartTime cqlStartTime, long enqueuedNsAgo, long startedNsAgo)
+        {
+            this.nativeTimeoutMs = nativeTimeoutMs;
+            this.rpcTimeoutMs = rpcTimeoutMs;
+            this.speculationTimeoutMicros = speculationTimeoutMicros;
+
+            this.cqlStartTime = cqlStartTime;
+
+            this.enqueuedNsAgo = enqueuedNsAgo;
+            this.startedNsAgo = startedNsAgo;
+        }
+
+        private void assertWillSpeculate()
+        {
+            DatabaseDescriptor.setNativeTransportTimeout(nativeTimeoutMs, TimeUnit.MILLISECONDS);
+            DatabaseDescriptor.setReadRpcTimeout(rpcTimeoutMs);
+            DatabaseDescriptor.setCQLStartTime(cqlStartTime);
+
+            ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore("tbl");
+            long speculatedBefore = cfs.metric.speculativeRetries.getCount();
+            long before = System.nanoTime();
+            cfs.sampleReadLatencyMicros = speculationTimeoutMicros;
+
+            CoordinatorHelper.unsafeExecuteInternal("SELECT * FROM " + KEYSPACE + ".tbl WHERE pk = 1",
+                                                    ConsistencyLevel.QUORUM,
+                                                    ConsistencyLevel.QUORUM,
+                                                    new Dispatcher.RequestTime(before - enqueuedNsAgo,
+                                                                               before - startedNsAgo));
+            long after = System.nanoTime();
+            long elapsed = after - before;
+            assertThat(elapsed).isGreaterThan(TimeUnit.MICROSECONDS.toNanos(speculationTimeoutMicros) - startedNsAgo);
+            long speculatedAfter = cfs.metric.speculativeRetries.getCount();
+            Assert.assertEquals(speculatedAfter, speculatedBefore + 1);
+        }
+
+        private void assertWillNotSpeculate()
+        {
+            DatabaseDescriptor.setNativeTransportIdleTimeout(nativeTimeoutMs);
+            DatabaseDescriptor.setReadRpcTimeout(rpcTimeoutMs);
+            DatabaseDescriptor.setCQLStartTime(cqlStartTime);
+
+            ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore("tbl");
+            long speculatedBefore = cfs.metric.speculativeRetries.getCount();
+            long before = System.nanoTime();
+            cfs.sampleReadLatencyMicros = speculationTimeoutMicros;
+
+            try
+            {
+                CoordinatorHelper.unsafeExecuteInternal("SELECT * FROM " + KEYSPACE + ".tbl WHERE pk = 1",
+                                                        ConsistencyLevel.QUORUM,
+                                                        ConsistencyLevel.QUORUM,
+                                                        new Dispatcher.RequestTime(before - enqueuedNsAgo,
+                                                                                   before - startedNsAgo));
+                throw new AssertionError("Should have timed out");
+            }
+            catch (ReadTimeoutException t)
+            {
+                // Expected
+            }
+            long after = System.nanoTime();
+            long elapsed = after - before;
+            assertThat(TimeUnit.NANOSECONDS.toMicros(elapsed)).isLessThan(speculationTimeoutMicros);
+            long speculatedAfter = cfs.metric.speculativeRetries.getCount();
+            Assert.assertEquals(speculatedAfter, speculatedBefore);
+        }
+    }
+
+}