Merge branch 'cassandra-3.0' into trunk

CHANGES.txt

@@ -3,6 +3,7 @@
 
 
 3.0.0-beta1
+ * Rewrite hinted handoff (CASSANDRA-6230)
  * Fix query on static compact tables (CASSANDRA-10093)
  * Fix race during construction of commit log (CASSANDRA-10049)
  * Add option to only purge repaired tombstones (CASSANDRA-6434)

NEWS.txt

@@ -23,6 +23,9 @@ New features
      for non-primary key queries, and perform much better for indexing high
      cardinality columns.
      See http://www.datastax.com/dev/blog/new-in-cassandra-3-0-materialized-views
+   - Hinted handoff has been completely rewritten. Hints are now stored in flat
+     files, with less overhead for storage and more efficient dispatch.
+     See CASSANDRA-6230 for full details.
    - Option to not purge unrepaired tombstones. To avoid users having data resurrected
      if repair has not been run within gc_grace_seconds, an option has been added to
      only allow tombstones from repaired sstables to be purged. To enable, set the
@@ -30,10 +33,11 @@ New features
      you do not run repair for a long time, you will keep all tombstones around which
      can cause other problems.
 
-
 Upgrading
 ---------
-   - 3.0 requires Java 8u20 or later.
+   - Max mutation size is now configurable via max_mutation_size_in_kb setting in
+     cassandra.yaml; the default is half the size commitlog_segment_size_in_mb * 1024.
+   - 3.0 requires Java 8u40 or later.
    - The default JVM GC has been changed to G1GC.
    - The default JVM flag -XX:+PerfDisableSharedMem will cause the following tools JVM
      to stop working: jps, jstack, jinfo, jmc, jcmd as well as 3rd party tools like Jolokia.
@@ -91,6 +95,7 @@ Upgrading
      set/getCompactionParameters or set/getCompactionParametersJson instead.
    - SizeTieredCompactionStrategy parameter cold_reads_to_omit has been removed.
 
+
 2.2
 ===
 

conf/cassandra.yaml

@@ -53,17 +53,30 @@ hinted_handoff_enabled: true
 # generated.  After it has been dead this long, new hints for it will not be
 # created until it has been seen alive and gone down again.
 max_hint_window_in_ms: 10800000 # 3 hours
+
 # Maximum throttle in KBs per second, per delivery thread.  This will be
 # reduced proportionally to the number of nodes in the cluster.  (If there
 # are two nodes in the cluster, each delivery thread will use the maximum
 # rate; if there are three, each will throttle to half of the maximum,
 # since we expect two nodes to be delivering hints simultaneously.)
 hinted_handoff_throttle_in_kb: 1024
+
 # Number of threads with which to deliver hints;
 # Consider increasing this number when you have multi-dc deployments, since
 # cross-dc handoff tends to be slower
 max_hints_delivery_threads: 2
 
+# Directory where Cassandra should store hints.
+# If not set, the default directory is $CASSANDRA_HOME/data/hints.
+# hints_directory: /var/lib/cassandra/hints
+
+# How often hints should be flushed from the internal buffers to disk.
+# Will *not* trigger fsync.
+hints_flush_period_in_ms: 10000
+
+# Maximum size for a single hints file, in megabytes.
+max_hints_file_size_in_mb: 128
+
 # Maximum throttle in KBs per second, total. This will be
 # reduced proportionally to the number of nodes in the cluster.
 batchlog_replay_throttle_in_kb: 1024

src/java/org/apache/cassandra/config/Config.java

@@ -56,6 +56,7 @@ public class Config
     public volatile boolean hinted_handoff_enabled = true;
     public Set<String> hinted_handoff_disabled_datacenters = Sets.newConcurrentHashSet();
     public volatile Integer max_hint_window_in_ms = 3 * 3600 * 1000; // three hours
+    public String hints_directory;
 
     public ParameterizedClass seed_provider;
     public DiskAccessMode disk_access_mode = DiskAccessMode.auto;
@@ -171,6 +172,8 @@ public class Config
     public int commitlog_max_compression_buffers_in_pool = 3;
     public TransparentDataEncryptionOptions transparent_data_encryption_options = new TransparentDataEncryptionOptions();
 
+    public Integer max_mutation_size_in_kb;
+
     @Deprecated
     public int commitlog_periodic_queue_size = -1;
 
@@ -196,7 +199,9 @@ public class Config
 
     public int hinted_handoff_throttle_in_kb = 1024;
     public int batchlog_replay_throttle_in_kb = 1024;
-    public int max_hints_delivery_threads = 1;
+    public int max_hints_delivery_threads = 2;
+    public int hints_flush_period_in_ms = 10000;
+    public int max_hints_file_size_in_mb = 128;
     public int sstable_preemptive_open_interval_in_mb = 50;
 
     public volatile boolean incremental_backups = false;

src/java/org/apache/cassandra/config/DatabaseDescriptor.java

@@ -484,6 +484,15 @@ public class DatabaseDescriptor
                 throw new ConfigurationException("commitlog_directory is missing and -Dcassandra.storagedir is not set", false);
             conf.commitlog_directory += File.separator + "commitlog";
         }
+
+        if (conf.hints_directory == null)
+        {
+            conf.hints_directory = System.getProperty("cassandra.storagedir", null);
+            if (conf.hints_directory == null)
+                throw new ConfigurationException("hints_directory is missing and -Dcassandra.storagedir is not set", false);
+            conf.hints_directory += File.separator + "hints";
+        }
+
         if (conf.saved_caches_directory == null)
         {
             conf.saved_caches_directory = System.getProperty("cassandra.storagedir", null);
@@ -504,12 +513,18 @@ public class DatabaseDescriptor
         {
             if (datadir.equals(conf.commitlog_directory))
                 throw new ConfigurationException("commitlog_directory must not be the same as any data_file_directories", false);
+            if (datadir.equals(conf.hints_directory))
+                throw new ConfigurationException("hints_directory must not be the same as any data_file_directories", false);
             if (datadir.equals(conf.saved_caches_directory))
                 throw new ConfigurationException("saved_caches_directory must not be the same as any data_file_directories", false);
         }
 
         if (conf.commitlog_directory.equals(conf.saved_caches_directory))
             throw new ConfigurationException("saved_caches_directory must not be the same as the commitlog_directory", false);
+        if (conf.commitlog_directory.equals(conf.hints_directory))
+            throw new ConfigurationException("hints_directory must not be the same as the commitlog_directory", false);
+        if (conf.hints_directory.equals(conf.saved_caches_directory))
+            throw new ConfigurationException("saved_caches_directory must not be the same as the hints_directory", false);
 
         if (conf.memtable_flush_writers == null)
             conf.memtable_flush_writers = Math.min(8, Math.max(2, Math.min(FBUtilities.getAvailableProcessors(), conf.data_file_directories.length)));
@@ -619,6 +634,11 @@ public class DatabaseDescriptor
         // always attempt to load the cipher factory, as we could be in the situation where the user has disabled encryption,
         // but has existing commitlogs and sstables on disk that are still encrypted (and still need to be read)
         encryptionContext = new EncryptionContext(config.transparent_data_encryption_options);
+
+        if (conf.max_mutation_size_in_kb == null)
+            conf.max_mutation_size_in_kb = conf.commitlog_segment_size_in_mb * 1024 / 2;
+        else if (conf.commitlog_segment_size_in_mb * 1024 < 2 * conf.max_mutation_size_in_kb)
+            throw new ConfigurationException("commitlog_segment_size_in_mb must be at least twice the size of max_mutation_size_in_kb / 1024", false);
     }
 
     private static IEndpointSnitch createEndpointSnitch(String snitchClassName) throws ConfigurationException
@@ -714,18 +734,18 @@ public class DatabaseDescriptor
                 throw new ConfigurationException("At least one DataFileDirectory must be specified", false);
 
             for (String dataFileDirectory : conf.data_file_directories)
-            {
                 FileUtils.createDirectory(dataFileDirectory);
-            }
 
             if (conf.commitlog_directory == null)
                 throw new ConfigurationException("commitlog_directory must be specified", false);
-
             FileUtils.createDirectory(conf.commitlog_directory);
 
+            if (conf.hints_directory == null)
+                throw new ConfigurationException("hints_directory must be specified", false);
+            FileUtils.createDirectory(conf.hints_directory);
+
             if (conf.saved_caches_directory == null)
                 throw new ConfigurationException("saved_caches_directory must be specified", false);
-
             FileUtils.createDirectory(conf.saved_caches_directory);
         }
         catch (ConfigurationException e)
@@ -998,6 +1018,7 @@ public class DatabaseDescriptor
             case PAXOS_PREPARE:
             case PAXOS_PROPOSE:
             case BATCHLOG_MUTATION:
+            case HINT:
                 return getWriteRpcTimeout();
             case COUNTER_MUTATION:
                 return getCounterWriteRpcTimeout();
@@ -1125,6 +1146,11 @@ public class DatabaseDescriptor
         return conf.commitlog_max_compression_buffers_in_pool;
     }
 
+    public static int getMaxMutationSize()
+    {
+        return conf.max_mutation_size_in_kb * 1024;
+    }
+
     public static int getTombstoneWarnThreshold()
     {
         return conf.tombstone_warn_threshold;
@@ -1421,6 +1447,11 @@ public class DatabaseDescriptor
         return conf.max_hint_window_in_ms;
     }
 
+    public static File getHintsDirectory()
+    {
+        return new File(conf.hints_directory);
+    }
+
     public static File getSerializedCachePath(String ksName,
                                               String cfName,
                                               UUID cfId,
@@ -1490,11 +1521,21 @@ public class DatabaseDescriptor
         conf.hinted_handoff_throttle_in_kb = throttleInKB;
     }
 
-    public static int getMaxHintsThread()
+    public static int getMaxHintsDeliveryThreads()
     {
         return conf.max_hints_delivery_threads;
     }
 
+    public static int getHintsFlushPeriodInMS()
+    {
+        return conf.hints_flush_period_in_ms;
+    }
+
+    public static long getMaxHintsFileSize()
+    {
+        return conf.max_hints_file_size_in_mb * 1024 * 1024;
+    }
+
     public static boolean isIncrementalBackupsEnabled()
     {
         return conf.incremental_backups;

src/java/org/apache/cassandra/db/BatchlogManager.java

@@ -41,18 +41,20 @@ import org.apache.cassandra.dht.Token;
 import org.apache.cassandra.exceptions.WriteFailureException;
 import org.apache.cassandra.exceptions.WriteTimeoutException;
 import org.apache.cassandra.gms.FailureDetector;
+import org.apache.cassandra.hints.Hint;
+import org.apache.cassandra.hints.HintsService;
 import org.apache.cassandra.io.util.DataInputBuffer;
 import org.apache.cassandra.io.util.DataInputPlus;
 import org.apache.cassandra.io.util.DataOutputBuffer;
 import org.apache.cassandra.net.MessageIn;
 import org.apache.cassandra.net.MessageOut;
 import org.apache.cassandra.net.MessagingService;
-import org.apache.cassandra.service.StorageProxy;
 import org.apache.cassandra.service.StorageService;
 import org.apache.cassandra.service.WriteResponseHandler;
 import org.apache.cassandra.utils.FBUtilities;
 import org.apache.cassandra.utils.UUIDGen;
 
+import static com.google.common.collect.Iterables.transform;
 import static org.apache.cassandra.cql3.QueryProcessor.executeInternal;
 import static org.apache.cassandra.cql3.QueryProcessor.executeInternalWithPaging;
 
@@ -210,6 +212,9 @@ public class BatchlogManager implements BatchlogManagerMBean
         int positionInPage = 0;
         ArrayList<Batch> unfinishedBatches = new ArrayList<>(pageSize);
 
+        Set<InetAddress> hintedNodes = new HashSet<>();
+        Set<UUID> replayedBatches = new HashSet<>();
+
         // Sending out batches for replay without waiting for them, so that one stuck batch doesn't affect others
         for (UntypedResultSet.Row row : batches)
         {
@@ -218,7 +223,7 @@ public class BatchlogManager implements BatchlogManagerMBean
             Batch batch = new Batch(id, row.getBytes("data"), version);
             try
             {
-                if (batch.replay(rateLimiter) > 0)
+                if (batch.replay(rateLimiter, hintedNodes) > 0)
                 {
                     unfinishedBatches.add(batch);
                 }
@@ -238,21 +243,29 @@ public class BatchlogManager implements BatchlogManagerMBean
             {
                 // We have reached the end of a batch. To avoid keeping more than a page of mutations in memory,
                 // finish processing the page before requesting the next row.
-                finishAndClearBatches(unfinishedBatches);
+                finishAndClearBatches(unfinishedBatches, hintedNodes, replayedBatches);
                 positionInPage = 0;
             }
         }
-        finishAndClearBatches(unfinishedBatches);
+
+        finishAndClearBatches(unfinishedBatches, hintedNodes, replayedBatches);
+
+        // to preserve batch guarantees, we must ensure that hints (if any) have made it to disk, before deleting the batches
+        HintsService.instance.flushAndFsyncBlockingly(transform(hintedNodes, StorageService.instance::getHostIdForEndpoint));
+
+        // once all generated hints are fsynced, actually delete the batches
+        replayedBatches.forEach(BatchlogManager::deleteBatch);
     }
 
-    private void finishAndClearBatches(ArrayList<Batch> batches)
+    private void finishAndClearBatches(ArrayList<Batch> batches, Set<InetAddress> hintedNodes, Set<UUID> replayedBatches)
     {
         // schedule hints for timed out deliveries
         for (Batch batch : batches)
         {
-            batch.finish();
-            deleteBatch(batch.id);
+            batch.finish(hintedNodes);
+            replayedBatches.add(batch.id);
         }
+
         totalBatchesReplayed += batches.size();
         batches.clear();
     }
@@ -279,7 +292,7 @@ public class BatchlogManager implements BatchlogManagerMBean
             this.version = version;
         }
 
-        public int replay(RateLimiter rateLimiter) throws IOException
+        public int replay(RateLimiter rateLimiter, Set<InetAddress> hintedNodes) throws IOException
         {
             logger.debug("Replaying batch {}", id);
 
@@ -288,18 +301,18 @@ public class BatchlogManager implements BatchlogManagerMBean
             if (mutations.isEmpty())
                 return 0;
 
-            int ttl = calculateHintTTL(mutations);
-            if (ttl <= 0)
+            int gcgs = gcgs(mutations);
+            if (TimeUnit.MILLISECONDS.toSeconds(writtenAt) + gcgs <= FBUtilities.nowInSeconds())
                 return 0;
 
-            replayHandlers = sendReplays(mutations, writtenAt, ttl);
+            replayHandlers = sendReplays(mutations, writtenAt, hintedNodes);
 
             rateLimiter.acquire(data.remaining()); // acquire afterwards, to not mess up ttl calculation.
 
             return replayHandlers.size();
         }
 
-        public void finish()
+        public void finish(Set<InetAddress> hintedNodes)
         {
             for (int i = 0; i < replayHandlers.size(); i++)
             {
@@ -313,7 +326,7 @@ public class BatchlogManager implements BatchlogManagerMBean
                     logger.debug("Failed replaying a batched mutation to a node, will write a hint");
                     logger.debug("Failure was : {}", e.getMessage());
                     // writing hints for the rest to hints, starting from i
-                    writeHintsForUndeliveredEndpoints(i);
+                    writeHintsForUndeliveredEndpoints(i, hintedNodes);
                     return;
                 }
             }
@@ -341,7 +354,7 @@ public class BatchlogManager implements BatchlogManagerMBean
             return mutations;
         }
 
-        private void writeHintsForUndeliveredEndpoints(int startFrom)
+        private void writeHintsForUndeliveredEndpoints(int startFrom, Set<InetAddress> hintedNodes)
         {
             try
             {
@@ -353,12 +366,15 @@ public class BatchlogManager implements BatchlogManagerMBean
                 for (int i = startFrom; i < replayHandlers.size(); i++)
                 {
                     Mutation undeliveredMutation = replayingMutations.get(i);
-                    int ttl = calculateHintTTL(replayingMutations);
+                    int gcgs = gcgs(replayingMutations);
                     ReplayWriteResponseHandler<Mutation> handler = replayHandlers.get(i);
 
-                    if (ttl > 0 && handler != null)
-                        for (InetAddress endpoint : handler.undelivered)
-                            StorageProxy.writeHintForMutation(undeliveredMutation, writtenAt, ttl, endpoint);
+                    if (TimeUnit.MILLISECONDS.toSeconds(writtenAt) + gcgs > FBUtilities.nowInSeconds() && handler != null)
+                    {
+                        hintedNodes.addAll(handler.undelivered);
+                        HintsService.instance.write(transform(handler.undelivered, StorageService.instance::getHostIdForEndpoint),
+                                                    Hint.create(undeliveredMutation, writtenAt));
+                    }
                 }
             }
             catch (IOException e)
@@ -367,12 +383,14 @@ public class BatchlogManager implements BatchlogManagerMBean
             }
         }
 
-        private static List<ReplayWriteResponseHandler<Mutation>> sendReplays(List<Mutation> mutations, long writtenAt, int ttl)
+        private static List<ReplayWriteResponseHandler<Mutation>> sendReplays(List<Mutation> mutations,
+                                                                              long writtenAt,
+                                                                              Set<InetAddress> hintedNodes)
         {
             List<ReplayWriteResponseHandler<Mutation>> handlers = new ArrayList<>(mutations.size());
             for (Mutation mutation : mutations)
             {
-                ReplayWriteResponseHandler<Mutation> handler = sendSingleReplayMutation(mutation, writtenAt, ttl);
+                ReplayWriteResponseHandler<Mutation> handler = sendSingleReplayMutation(mutation, writtenAt, hintedNodes);
                 if (handler != null)
                     handlers.add(handler);
             }
@@ -385,7 +403,9 @@ public class BatchlogManager implements BatchlogManagerMBean
          *
          * @return direct delivery handler to wait on or null, if no live nodes found
          */
-        private static ReplayWriteResponseHandler<Mutation> sendSingleReplayMutation(final Mutation mutation, long writtenAt, int ttl)
+        private static ReplayWriteResponseHandler<Mutation> sendSingleReplayMutation(final Mutation mutation,
+                                                                                     long writtenAt,
+                                                                                     Set<InetAddress> hintedNodes)
         {
             Set<InetAddress> liveEndpoints = new HashSet<>();
             String ks = mutation.getKeyspaceName();
@@ -395,11 +415,19 @@ public class BatchlogManager implements BatchlogManagerMBean
                                                          StorageService.instance.getTokenMetadata().pendingEndpointsFor(tk, ks)))
             {
                 if (endpoint.equals(FBUtilities.getBroadcastAddress()))
+                {
                     mutation.apply();
+                }
                 else if (FailureDetector.instance.isAlive(endpoint))
+                {
                     liveEndpoints.add(endpoint); // will try delivering directly instead of writing a hint.
+                }
                 else
-                    StorageProxy.writeHintForMutation(mutation, writtenAt, ttl, endpoint);
+                {
+                    hintedNodes.add(endpoint);
+                    HintsService.instance.write(StorageService.instance.getHostIdForEndpoint(endpoint),
+                                                Hint.create(mutation, writtenAt));
+                }
             }
 
             if (liveEndpoints.isEmpty())
@@ -412,16 +440,12 @@ public class BatchlogManager implements BatchlogManagerMBean
             return handler;
         }
 
-        /*
-         * Calculate ttl for the mutations' hints (and reduce ttl by the time the mutations spent in the batchlog).
-         * This ensures that deletes aren't "undone" by an old batch replay.
-         */
-        private int calculateHintTTL(Collection<Mutation> mutations)
+        private static int gcgs(Collection<Mutation> mutations)
         {
-            int unadjustedTTL = Integer.MAX_VALUE;
+            int gcgs = Integer.MAX_VALUE;
             for (Mutation mutation : mutations)
-                unadjustedTTL = Math.min(unadjustedTTL, HintedHandOffManager.calculateHintTTL(mutation));
-            return unadjustedTTL - (int) TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis() - writtenAt);
+                gcgs = Math.min(gcgs, mutation.smallestGCGS());
+            return gcgs;
         }
 
         /**

src/java/org/apache/cassandra/db/HintedHandOffManager.java

@@ -17,156 +17,32 @@
  */
 package org.apache.cassandra.db;
 
-import java.io.IOException;
 import java.lang.management.ManagementFactory;
-import java.net.InetAddress;
-import java.net.UnknownHostException;
-import java.nio.ByteBuffer;
-import java.util.*;
-import java.util.concurrent.*;
-import java.util.concurrent.atomic.AtomicInteger;
+import java.util.List;
 
 import javax.management.MBeanServer;
 import javax.management.ObjectName;
 
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.collect.Lists;
-import com.google.common.util.concurrent.RateLimiter;
-import com.google.common.util.concurrent.Uninterruptibles;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.cassandra.concurrent.DebuggableScheduledThreadPoolExecutor;
-import org.apache.cassandra.concurrent.JMXEnabledThreadPoolExecutor;
-import org.apache.cassandra.concurrent.NamedThreadFactory;
-import org.apache.cassandra.config.ColumnDefinition;
-import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.db.compaction.CompactionManager;
-
-import org.apache.cassandra.db.rows.*;
-import org.apache.cassandra.db.partitions.*;
-import org.apache.cassandra.db.filter.*;
-import org.apache.cassandra.db.marshal.Int32Type;
-import org.apache.cassandra.db.marshal.UUIDType;
-import org.apache.cassandra.exceptions.WriteTimeoutException;
-import org.apache.cassandra.gms.ApplicationState;
-import org.apache.cassandra.gms.FailureDetector;
-import org.apache.cassandra.gms.Gossiper;
-import org.apache.cassandra.io.sstable.Descriptor;
-import org.apache.cassandra.io.sstable.SSTable;
-import org.apache.cassandra.io.util.DataInputBuffer;
-import org.apache.cassandra.io.util.DataInputPlus;
-import org.apache.cassandra.metrics.HintedHandoffMetrics;
-import org.apache.cassandra.net.MessageOut;
-import org.apache.cassandra.net.MessagingService;
-import org.apache.cassandra.service.StorageProxy;
-import org.apache.cassandra.service.StorageService;
-import org.apache.cassandra.service.WriteResponseHandler;
-import org.apache.cassandra.utils.*;
-import org.apache.cassandra.utils.concurrent.OpOrder;
-import org.cliffc.high_scale_lib.NonBlockingHashSet;
+import org.apache.cassandra.hints.HintsService;
 
 /**
- * The hint schema looks like this:
+ * A proxy class that implement the deprecated legacy HintedHandoffManagerMBean interface.
  *
- * CREATE TABLE hints (
- *   target_id uuid,
- *   hint_id timeuuid,
- *   message_version int,
- *   mutation blob,
- *   PRIMARY KEY (target_id, hint_id, message_version)
- * ) WITH COMPACT STORAGE;
- *
- * Thus, for each node in the cluster we treat its uuid as the partition key; each hint is a logical row
- * (physical composite column) containing the mutation to replay and associated metadata.
- *
- * When FailureDetector signals that a node that was down is back up, we page through
- * the hinted mutations and send them over one at a time, waiting for
- * hinted_handoff_throttle_delay in between each.
- *
- * deliverHints is also exposed to JMX so it can be run manually if FD ever misses
- * its cue somehow.
+ * TODO: remove in 4.0.
  */
-
-public class HintedHandOffManager implements HintedHandOffManagerMBean
+@SuppressWarnings("deprecation")
+@Deprecated
+public final class HintedHandOffManager implements HintedHandOffManagerMBean
 {
-    public static final String MBEAN_NAME = "org.apache.cassandra.db:type=HintedHandoffManager";
     public static final HintedHandOffManager instance = new HintedHandOffManager();
 
-    private static final Logger logger = LoggerFactory.getLogger(HintedHandOffManager.class);
+    public static final String MBEAN_NAME = "org.apache.cassandra.db:type=HintedHandoffManager";
 
-    private static final int MAX_SIMULTANEOUSLY_REPLAYED_HINTS = 128;
-    private static final int LARGE_NUMBER = 65536; // 64k nodes ought to be enough for anybody.
-
-    public final HintedHandoffMetrics metrics = new HintedHandoffMetrics();
-
-    private volatile boolean hintedHandOffPaused = false;
-
-    static final int maxHintTTL = Integer.parseInt(System.getProperty("cassandra.maxHintTTL", String.valueOf(Integer.MAX_VALUE)));
-
-    private final NonBlockingHashSet<InetAddress> queuedDeliveries = new NonBlockingHashSet<>();
-
-    // To keep metrics consistent with earlier versions, where periodic tasks were run on a shared executor,
-    // we run them on this executor and so keep counts separate from those for hint delivery tasks. See CASSANDRA-9129
-    private final DebuggableScheduledThreadPoolExecutor executor =
-        new DebuggableScheduledThreadPoolExecutor(1, new NamedThreadFactory("HintedHandoffManager", Thread.MIN_PRIORITY));
-
-    // Non-scheduled executor to run the actual hint delivery tasks.
-    // Per CASSANDRA-9129, this is where the values displayed in nodetool tpstats
-    // and via the HintedHandoff mbean are obtained.
-    private final ThreadPoolExecutor hintDeliveryExecutor =
-        new JMXEnabledThreadPoolExecutor(
-            DatabaseDescriptor.getMaxHintsThread(),
-            Integer.MAX_VALUE,
-            TimeUnit.SECONDS,
-            new LinkedBlockingQueue<Runnable>(),
-            new NamedThreadFactory("HintedHandoff", Thread.MIN_PRIORITY),
-            "internal");
-
-    private final ColumnFamilyStore hintStore = Keyspace.open(SystemKeyspace.NAME).getColumnFamilyStore(SystemKeyspace.HINTS);
-
-    private static final ColumnDefinition hintColumn = SystemKeyspace.Hints.compactValueColumn();
-
-    /**
-     * Returns a mutation representing a Hint to be sent to <code>targetId</code>
-     * as soon as it becomes available again.
-     */
-    public Mutation hintFor(Mutation mutation, long now, int ttl, UUID targetId)
+    private HintedHandOffManager()
     {
-        assert ttl > 0;
-
-        InetAddress endpoint = StorageService.instance.getTokenMetadata().getEndpointForHostId(targetId);
-        // during tests we may not have a matching endpoint, but this would be unexpected in real clusters
-        if (endpoint != null)
-            metrics.incrCreatedHints(endpoint);
-        else
-            logger.warn("Unable to find matching endpoint for target {} when storing a hint", targetId);
-
-        UUID hintId = UUIDGen.getTimeUUID();
-        // serialize the hint with id and version as a composite column name
-
-        ByteBuffer key = UUIDType.instance.decompose(targetId);
-        Clustering clustering = SystemKeyspace.Hints.comparator.make(hintId, MessagingService.current_version);
-        ByteBuffer value = ByteBuffer.wrap(FBUtilities.serialize(mutation, Mutation.serializer, MessagingService.current_version));
-        Cell cell = BufferCell.expiring(hintColumn, now, ttl, FBUtilities.nowInSeconds(), value);
-
-        return new Mutation(PartitionUpdate.singleRowUpdate(SystemKeyspace.Hints, key, BTreeRow.singleCellRow(clustering, cell)));
     }
 
-    /*
-     * determine the TTL for the hint Mutation
-     * this is set at the smallest GCGraceSeconds for any of the CFs in the RM
-     * this ensures that deletes aren't "undone" by delivery of an old hint
-     */
-    public static int calculateHintTTL(Mutation mutation)
-    {
-        int ttl = maxHintTTL;
-        for (PartitionUpdate upd : mutation.getPartitionUpdates())
-            ttl = Math.min(ttl, upd.metadata().params.gcGraceSeconds);
-        return ttl;
-    }
-
-    public void start()
+    public void registerMBean()
     {
         MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
         try
@@ -177,404 +53,35 @@ public class HintedHandOffManager implements HintedHandOffManagerMBean
         {
             throw new RuntimeException(e);
         }
-        logger.debug("Created HHOM instance, registered MBean.");
-
-        Runnable runnable = new Runnable()
-        {
-            public void run()
-            {
-                scheduleAllDeliveries();
-                metrics.log();
-            }
-        };
-        executor.scheduleWithFixedDelay(runnable, 10, 10, TimeUnit.MINUTES);
     }
 
-    private static void deleteHint(ByteBuffer tokenBytes, Clustering clustering, long timestamp)
+    public void deleteHintsForEndpoint(String host)
     {
-        Cell cell = BufferCell.tombstone(hintColumn, timestamp, FBUtilities.nowInSeconds());
-        PartitionUpdate upd = PartitionUpdate.singleRowUpdate(SystemKeyspace.Hints, tokenBytes, BTreeRow.singleCellRow(clustering, cell));
-        new Mutation(upd).applyUnsafe(); // don't bother with commitlog since we're going to flush as soon as we're done with delivery
+        HintsService.instance.deleteAllHintsForEndpoint(host);
     }
 
-    public void deleteHintsForEndpoint(final String ipOrHostname)
+    public void truncateAllHints()
     {
-        try
-        {
-            InetAddress endpoint = InetAddress.getByName(ipOrHostname);
-            deleteHintsForEndpoint(endpoint);
-        }
-        catch (UnknownHostException e)
-        {
-            logger.warn("Unable to find {}, not a hostname or ipaddr of a node", ipOrHostname);
-            throw new RuntimeException(e);
-        }
-    }
-
-    public void deleteHintsForEndpoint(final InetAddress endpoint)
-    {
-        if (!StorageService.instance.getTokenMetadata().isMember(endpoint))
-            return;
-        UUID hostId = StorageService.instance.getTokenMetadata().getHostId(endpoint);
-        ByteBuffer key = ByteBuffer.wrap(UUIDGen.decompose(hostId));
-        final Mutation mutation = new Mutation(PartitionUpdate.fullPartitionDelete(SystemKeyspace.Hints, key, System.currentTimeMillis(), FBUtilities.nowInSeconds()));
-
-        // execute asynchronously to avoid blocking caller (which may be processing gossip)
-        Runnable runnable = new Runnable()
-        {
-            public void run()
-            {
-                try
-                {
-                    logger.info("Deleting any stored hints for {}", endpoint);
-                    mutation.apply();
-                    hintStore.forceBlockingFlush();
-                    compact();
-                }
-                catch (Exception e)
-                {
-                    JVMStabilityInspector.inspectThrowable(e);
-                    logger.warn("Could not delete hints for {}: {}", endpoint, e);
-                }
-            }
-        };
-        executor.submit(runnable);
-    }
-
-    //foobar
-    public void truncateAllHints() throws ExecutionException, InterruptedException
-    {
-        Runnable runnable = new Runnable()
-        {
-            public void run()
-            {
-                try
-                {
-                    logger.info("Truncating all stored hints.");
-                    Keyspace.open(SystemKeyspace.NAME).getColumnFamilyStore(SystemKeyspace.HINTS).truncateBlocking();
-                }
-                catch (Exception e)
-                {
-                    logger.warn("Could not truncate all hints.", e);
-                }
-            }
-        };
-        executor.submit(runnable).get();
-    }
-
-    @VisibleForTesting
-    protected synchronized void compact()
-    {
-        ArrayList<Descriptor> descriptors = new ArrayList<>();
-        for (SSTable sstable : hintStore.getTracker().getUncompacting())
-            descriptors.add(sstable.descriptor);
-
-        if (descriptors.isEmpty())
-            return;
-
-        try
-        {
-            CompactionManager.instance.submitUserDefined(hintStore, descriptors, (int) (System.currentTimeMillis() / 1000)).get();
-        }
-        catch (InterruptedException | ExecutionException e)
-        {
-            throw new RuntimeException(e);
-        }
-    }
-
-    private int waitForSchemaAgreement(InetAddress endpoint) throws TimeoutException
-    {
-        Gossiper gossiper = Gossiper.instance;
-        int waited = 0;
-        // first, wait for schema to be gossiped.
-        while (gossiper.getEndpointStateForEndpoint(endpoint) != null && gossiper.getEndpointStateForEndpoint(endpoint).getApplicationState(ApplicationState.SCHEMA) == null)
-        {
-            Uninterruptibles.sleepUninterruptibly(1, TimeUnit.SECONDS);
-            waited += 1000;
-            if (waited > 2 * StorageService.RING_DELAY)
-                throw new TimeoutException("Didin't receive gossiped schema from " + endpoint + " in " + 2 * StorageService.RING_DELAY + "ms");
-        }
-        if (gossiper.getEndpointStateForEndpoint(endpoint) == null)
-            throw new TimeoutException("Node " + endpoint + " vanished while waiting for agreement");
-        waited = 0;
-        // then wait for the correct schema version.
-        // usually we use DD.getDefsVersion, which checks the local schema uuid as stored in the system keyspace.
-        // here we check the one in gossip instead; this serves as a canary to warn us if we introduce a bug that
-        // causes the two to diverge (see CASSANDRA-2946)
-        while (gossiper.getEndpointStateForEndpoint(endpoint) != null && !gossiper.getEndpointStateForEndpoint(endpoint).getApplicationState(ApplicationState.SCHEMA).value.equals(
-                gossiper.getEndpointStateForEndpoint(FBUtilities.getBroadcastAddress()).getApplicationState(ApplicationState.SCHEMA).value))
-        {
-            Uninterruptibles.sleepUninterruptibly(1, TimeUnit.SECONDS);
-            waited += 1000;
-            if (waited > 2 * StorageService.RING_DELAY)
-                throw new TimeoutException("Could not reach schema agreement with " + endpoint + " in " + 2 * StorageService.RING_DELAY + "ms");
-        }
-        if (gossiper.getEndpointStateForEndpoint(endpoint) == null)
-            throw new TimeoutException("Node " + endpoint + " vanished while waiting for agreement");
-        logger.debug("schema for {} matches local schema", endpoint);
-        return waited;
-    }
-
-    private void deliverHintsToEndpoint(InetAddress endpoint)
-    {
-        if (hintStore.isEmpty())
-            return; // nothing to do, don't confuse users by logging a no-op handoff
-
-        // check if hints delivery has been paused
-        if (hintedHandOffPaused)
-        {
-            logger.debug("Hints delivery process is paused, aborting");
-            return;
-        }
-
-        logger.debug("Checking remote({}) schema before delivering hints", endpoint);
-        try
-        {
-            waitForSchemaAgreement(endpoint);
-        }
-        catch (TimeoutException e)
-        {
-            return;
-        }
-
-        if (!FailureDetector.instance.isAlive(endpoint))
-        {
-            logger.debug("Endpoint {} died before hint delivery, aborting", endpoint);
-            return;
-        }
-
-        doDeliverHintsToEndpoint(endpoint);
-
-        // Flush all the tombstones to disk
-        hintStore.forceBlockingFlush();
-    }
-
-    private boolean checkDelivered(InetAddress endpoint, List<WriteResponseHandler<Mutation>> handlers, AtomicInteger rowsReplayed)
-    {
-        for (WriteResponseHandler<Mutation> handler : handlers)
-        {
-            try
-            {
-                handler.get();
-            }
-            catch (WriteTimeoutException e)
-            {
-                logger.info("Failed replaying hints to {}; aborting ({} delivered), error : {}",
-                            endpoint, rowsReplayed, e.getMessage());
-                return false;
-            }
-        }
-        return true;
-    }
-
-    /*
-     * 1. Get the key of the endpoint we need to handoff
-     * 2. For each column, deserialize the mutation and send it to the endpoint
-     * 3. Delete the column if the write was successful
-     * 4. Force a flush
-     */
-    private void doDeliverHintsToEndpoint(InetAddress endpoint)
-    {
-        // find the hints for the node using its token.
-        UUID hostId = Gossiper.instance.getHostId(endpoint);
-        logger.info("Started hinted handoff for host: {} with IP: {}", hostId, endpoint);
-        final ByteBuffer hostIdBytes = ByteBuffer.wrap(UUIDGen.decompose(hostId));
-
-        final AtomicInteger rowsReplayed = new AtomicInteger(0);
-
-        // rate limit is in bytes per second. Uses Double.MAX_VALUE if disabled (set to 0 in cassandra.yaml).
-        // max rate is scaled by the number of nodes in the cluster (CASSANDRA-5272).
-        int throttleInKB = DatabaseDescriptor.getHintedHandoffThrottleInKB()
-                           / (StorageService.instance.getTokenMetadata().getAllEndpoints().size() - 1);
-        RateLimiter rateLimiter = RateLimiter.create(throttleInKB == 0 ? Double.MAX_VALUE : throttleInKB * 1024);
-
-        int nowInSec = FBUtilities.nowInSeconds();
-        try (OpOrder.Group op = hintStore.readOrdering.start();
-             RowIterator iter = UnfilteredRowIterators.filter(SinglePartitionReadCommand.fullPartitionRead(SystemKeyspace.Hints, nowInSec, hostIdBytes).queryMemtableAndDisk(hintStore, op), nowInSec))
-        {
-            List<WriteResponseHandler<Mutation>> responseHandlers = Lists.newArrayList();
-
-            while (iter.hasNext())
-            {
-                // check if node is still alive and we should continue delivery process
-                if (!FailureDetector.instance.isAlive(endpoint))
-                {
-                    logger.info("Endpoint {} died during hint delivery; aborting ({} delivered)", endpoint, rowsReplayed);
-                    return;
-                }
-
-                // check if hints delivery has been paused during the process
-                if (hintedHandOffPaused)
-                {
-                    logger.debug("Hints delivery process is paused, aborting");
-                    return;
-                }
-
-                // Wait regularly on the endpoint acknowledgment. If we timeout on it, the endpoint is probably dead so stop delivery
-                if (responseHandlers.size() > MAX_SIMULTANEOUSLY_REPLAYED_HINTS && !checkDelivered(endpoint, responseHandlers, rowsReplayed))
-                    return;
-
-                final Row hint = iter.next();
-                int version = Int32Type.instance.compose(hint.clustering().get(1));
-                Cell cell = hint.getCell(hintColumn);
-
-                final long timestamp = cell.timestamp();
-                DataInputPlus in = new DataInputBuffer(cell.value(), true);
-                Mutation mutation;
-                try
-                {
-                    mutation = Mutation.serializer.deserialize(in, version);
-                }
-                catch (UnknownColumnFamilyException e)
-                {
-                    logger.debug("Skipping delivery of hint for deleted table", e);
-                    deleteHint(hostIdBytes, hint.clustering(), timestamp);
-                    continue;
-                }
-                catch (IOException e)
-                {
-                    throw new AssertionError(e);
-                }
-
-                for (UUID cfId : mutation.getColumnFamilyIds())
-                {
-                    if (timestamp <= SystemKeyspace.getTruncatedAt(cfId))
-                    {
-                        logger.debug("Skipping delivery of hint for truncated table {}", cfId);
-                        mutation = mutation.without(cfId);
-                    }
-                }
-
-                if (mutation.isEmpty())
-                {
-                    deleteHint(hostIdBytes, hint.clustering(), timestamp);
-                    continue;
-                }
-
-                MessageOut<Mutation> message = mutation.createMessage();
-                rateLimiter.acquire(message.serializedSize(MessagingService.current_version));
-                Runnable callback = new Runnable()
-                {
-                    public void run()
-                    {
-                        rowsReplayed.incrementAndGet();
-                        deleteHint(hostIdBytes, hint.clustering(), timestamp);
-                    }
-                };
-                WriteResponseHandler<Mutation> responseHandler = new WriteResponseHandler<>(endpoint, WriteType.SIMPLE, callback);
-                MessagingService.instance().sendRR(message, endpoint, responseHandler, false);
-                responseHandlers.add(responseHandler);
-            }
-
-            // Wait on the last handlers
-            if (checkDelivered(endpoint, responseHandlers, rowsReplayed))
-                logger.info("Finished hinted handoff of {} rows to endpoint {}", rowsReplayed, endpoint);
-        }
-    }
-
-    /**
-     * Attempt delivery to any node for which we have hints.  Necessary since we can generate hints even for
-     * nodes which are never officially down/failed.
-     */
-    private void scheduleAllDeliveries()
-    {
-        logger.debug("Started scheduleAllDeliveries");
-
-        // Force a major compaction to get rid of the tombstones and expired hints. Do it once, before we schedule any
-        // individual replay, to avoid N - 1 redundant individual compactions (when N is the number of nodes with hints
-        // to deliver to).
-        compact();
-
-        ReadCommand cmd = new PartitionRangeReadCommand(hintStore.metadata,
-                                                        FBUtilities.nowInSeconds(),
-                                                        ColumnFilter.all(hintStore.metadata),
-                                                        RowFilter.NONE,
-                                                        DataLimits.cqlLimits(Integer.MAX_VALUE, 1),
-                                                        DataRange.allData(hintStore.metadata.partitioner));
-
-        try (ReadOrderGroup orderGroup = cmd.startOrderGroup(); UnfilteredPartitionIterator iter = cmd.executeLocally(orderGroup))
-        {
-            while (iter.hasNext())
-            {
-                try (UnfilteredRowIterator partition = iter.next())
-                {
-                    UUID hostId = UUIDGen.getUUID(partition.partitionKey().getKey());
-                    InetAddress target = StorageService.instance.getTokenMetadata().getEndpointForHostId(hostId);
-                    // token may have since been removed (in which case we have just read back a tombstone)
-                    if (target != null)
-                        scheduleHintDelivery(target, false);
-                }
-            }
-        }
-
-        logger.debug("Finished scheduleAllDeliveries");
-    }
-
-    /*
-     * This method is used to deliver hints to a particular endpoint.
-     * When we learn that some endpoint is back up we deliver the data
-     * to him via an event driven mechanism.
-    */
-    public void scheduleHintDelivery(final InetAddress to, final boolean precompact)
-    {
-        // We should not deliver hints to the same host in 2 different threads
-        if (!queuedDeliveries.add(to))
-            return;
-
-        logger.debug("Scheduling delivery of Hints to {}", to);
-
-        hintDeliveryExecutor.execute(new Runnable()
-        {
-            public void run()
-            {
-                try
-                {
-                    // If it's an individual node hint replay (triggered by Gossip or via JMX), and not the global scheduled replay
-                    // (every 10 minutes), force a major compaction to get rid of the tombstones and expired hints.
-                    if (precompact)
-                        compact();
-
-                    deliverHintsToEndpoint(to);
-                }
-                finally
-                {
-                    queuedDeliveries.remove(to);
-                }
-            }
-        });
-    }
-
-    public void scheduleHintDelivery(String to) throws UnknownHostException
-    {
-        scheduleHintDelivery(InetAddress.getByName(to), true);
-    }
-
-    public void pauseHintsDelivery(boolean b)
-    {
-        hintedHandOffPaused = b;
+        HintsService.instance.deleteAllHints();
     }
 
+    // TODO
     public List<String> listEndpointsPendingHints()
     {
-        // Extract the keys as strings to be reported.
-        List<String> result = new ArrayList<>();
+        throw new UnsupportedOperationException();
+    }
 
-        ReadCommand cmd = PartitionRangeReadCommand.allDataRead(SystemKeyspace.Hints, FBUtilities.nowInSeconds());
-        try (ReadOrderGroup orderGroup = cmd.startOrderGroup();
-             UnfilteredPartitionIterator iter = cmd.executeLocally(orderGroup))
-        {
-            while (iter.hasNext())
-            {
-                try (UnfilteredRowIterator partition = iter.next())
-                {
-                    // We don't delete by range on the hints table, so we don't have to worry about the
-                    // iterator returning only range tombstone marker
-                    if (partition.hasNext())
-                        result.add(UUIDType.instance.compose(partition.partitionKey().getKey()).toString());
-                }
-            }
-        }
+    // TODO
+    public void scheduleHintDelivery(String host)
+    {
+        throw new UnsupportedOperationException();
+    }
 
-        return result;
+    public void pauseHintsDelivery(boolean doPause)
+    {
+        if (doPause)
+            HintsService.instance.pauseDispatch();
+        else
+            HintsService.instance.resumeDispatch();
     }
 }

src/java/org/apache/cassandra/db/HintedHandOffManagerMBean.java

@@ -21,6 +21,7 @@ import java.net.UnknownHostException;
 import java.util.List;
 import java.util.concurrent.ExecutionException;
 
+@Deprecated
 public interface HintedHandOffManagerMBean
 {
     /**

src/java/org/apache/cassandra/db/Mutation.java

@@ -75,10 +75,24 @@ public class Mutation implements IMutation
 
     public Mutation copy()
     {
-        Mutation copy = new Mutation(keyspaceName, key, new HashMap<>(modifications));
+        return new Mutation(keyspaceName, key, new HashMap<>(modifications));
+    }
+
+    public Mutation without(Set<UUID> cfIds)
+    {
+        if (cfIds.isEmpty())
+            return this;
+
+        Mutation copy = copy();
+        copy.modifications.keySet().removeAll(cfIds);
         return copy;
     }
 
+    public Mutation without(UUID cfId)
+    {
+        return without(Collections.singleton(cfId));
+    }
+
     public String getKeyspaceName()
     {
         return keyspaceName;
@@ -207,6 +221,14 @@ public class Mutation implements IMutation
         return DatabaseDescriptor.getWriteRpcTimeout();
     }
 
+    public int smallestGCGS()
+    {
+        int gcgs = Integer.MAX_VALUE;
+        for (PartitionUpdate update : getPartitionUpdates())
+            gcgs = Math.min(gcgs, update.metadata().params.gcGraceSeconds);
+        return gcgs;
+    }
+
     public String toString()
     {
         return toString(false);
@@ -235,15 +257,6 @@ public class Mutation implements IMutation
         return buff.append("])").toString();
     }
 
-    public Mutation without(UUID cfId)
-    {
-        Mutation mutation = new Mutation(keyspaceName, key);
-        for (Map.Entry<UUID, PartitionUpdate> entry : modifications.entrySet())
-            if (!entry.getKey().equals(cfId))
-                mutation.add(entry.getValue());
-        return mutation;
-    }
-
     public static class MutationSerializer implements IVersionedSerializer<Mutation>
     {
         public void serialize(Mutation mutation, DataOutputPlus out, int version) throws IOException

src/java/org/apache/cassandra/db/SystemKeyspace.java

@@ -88,7 +88,6 @@ public final class SystemKeyspace
 
     public static final String NAME = "system";
 
-    public static final String HINTS = "hints";
     public static final String BATCHES = "batches";
     public static final String PAXOS = "paxos";
     public static final String BUILT_INDEXES = "IndexInfo";
@@ -103,6 +102,7 @@ public final class SystemKeyspace
     public static final String MATERIALIZED_VIEWS_BUILDS_IN_PROGRESS = "materialized_views_builds_in_progress";
     public static final String BUILT_MATERIALIZED_VIEWS = "built_materialized_views";
 
+    @Deprecated public static final String LEGACY_HINTS = "hints";
     @Deprecated public static final String LEGACY_BATCHLOG = "batchlog";
     @Deprecated public static final String LEGACY_KEYSPACES = "schema_keyspaces";
     @Deprecated public static final String LEGACY_COLUMNFAMILIES = "schema_columnfamilies";
@@ -112,19 +112,6 @@ public final class SystemKeyspace
     @Deprecated public static final String LEGACY_FUNCTIONS = "schema_functions";
     @Deprecated public static final String LEGACY_AGGREGATES = "schema_aggregates";
 
-    public static final CFMetaData Hints =
-        compile(HINTS,
-                "hints awaiting delivery",
-                "CREATE TABLE %s ("
-                + "target_id uuid,"
-                + "hint_id timeuuid,"
-                + "message_version int,"
-                + "mutation blob,"
-                + "PRIMARY KEY ((target_id), hint_id, message_version)) "
-                + "WITH COMPACT STORAGE")
-                .compaction(CompactionParams.scts(singletonMap("enabled", "false")))
-                .gcGraceSeconds(0);
-
     public static final CFMetaData Batches =
         compile(BATCHES,
                 "batches awaiting replay",
@@ -281,6 +268,20 @@ public final class SystemKeyspace
                 + "view_name text,"
                 + "PRIMARY KEY ((keyspace_name), view_name))");
 
+    @Deprecated
+    public static final CFMetaData LegacyHints =
+        compile(LEGACY_HINTS,
+                "*DEPRECATED* hints awaiting delivery",
+                "CREATE TABLE %s ("
+                + "target_id uuid,"
+                + "hint_id timeuuid,"
+                + "message_version int,"
+                + "mutation blob,"
+                + "PRIMARY KEY ((target_id), hint_id, message_version)) "
+                + "WITH COMPACT STORAGE")
+                .compaction(CompactionParams.scts(singletonMap("enabled", "false")))
+                .gcGraceSeconds(0);
+
     @Deprecated
     public static final CFMetaData LegacyBatchlog =
         compile(LEGACY_BATCHLOG,
@@ -423,7 +424,6 @@ public final class SystemKeyspace
     private static Tables tables()
     {
         return Tables.of(BuiltIndexes,
-                         Hints,
                          Batches,
                          Paxos,
                          Local,
@@ -436,6 +436,7 @@ public final class SystemKeyspace
                          AvailableRanges,
                          MaterializedViewsBuildsInProgress,
                          BuiltMaterializedViews,
+                         LegacyHints,
                          LegacyBatchlog,
                          LegacyKeyspaces,
                          LegacyColumnfamilies,

src/java/org/apache/cassandra/db/commitlog/CommitLog.java

@@ -48,6 +48,8 @@ import org.apache.cassandra.service.StorageService;
 import org.apache.cassandra.utils.JVMStabilityInspector;
 
 import static org.apache.cassandra.db.commitlog.CommitLogSegment.*;
+import static org.apache.cassandra.utils.FBUtilities.updateChecksum;
+import static org.apache.cassandra.utils.FBUtilities.updateChecksumInt;
 
 /*
  * Commit Log tracks every write operation into the system. The aim of the commit log is to be able to
@@ -61,7 +63,7 @@ public class CommitLog implements CommitLogMBean
 
     // we only permit records HALF the size of a commit log, to ensure we don't spin allocating many mostly
     // empty segments when writing large records
-    private final long MAX_MUTATION_SIZE = DatabaseDescriptor.getCommitLogSegmentSize() >> 1;
+    private final long MAX_MUTATION_SIZE = DatabaseDescriptor.getMaxMutationSize();
 
     public final CommitLogSegmentManager allocator;
     public final CommitLogArchiver archiver;
@@ -254,9 +256,9 @@ public class CommitLog implements CommitLogMBean
     {
         assert mutation != null;
 
-        long size = Mutation.serializer.serializedSize(mutation, MessagingService.current_version);
+        int size = (int) Mutation.serializer.serializedSize(mutation, MessagingService.current_version);
 
-        long totalSize = size + ENTRY_OVERHEAD_SIZE;
+        int totalSize = size + ENTRY_OVERHEAD_SIZE;
         if (totalSize > MAX_MUTATION_SIZE)
         {
             throw new IllegalArgumentException(String.format("Mutation of %s bytes is too large for the maxiumum size of %s",
@@ -269,19 +271,13 @@ public class CommitLog implements CommitLogMBean
         try (BufferedDataOutputStreamPlus dos = new DataOutputBufferFixed(buffer))
         {
             // checksummed length
-            dos.writeInt((int) size);
-
-            ByteBuffer copy = buffer.duplicate();
-            copy.position(buffer.position() - 4);
-            copy.limit(buffer.position());
-            checksum.update(copy);
+            dos.writeInt(size);
+            updateChecksumInt(checksum, size);
             buffer.putInt((int) checksum.getValue());
 
             // checksummed mutation
-            copy = buffer.duplicate();
             Mutation.serializer.serialize(mutation, dos, MessagingService.current_version);
-            copy.limit(copy.position() + (int) size);
-            checksum.update(copy);
+            updateChecksum(checksum, buffer, buffer.position() - size, size);
             buffer.putInt((int) checksum.getValue());
         }
         catch (IOException e)

src/java/org/apache/cassandra/gms/Gossiper.java

@@ -809,6 +809,20 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean
         return endpointStateMap.get(ep);
     }
 
+    public boolean valuesEqual(InetAddress ep1, InetAddress ep2, ApplicationState as)
+    {
+        EndpointState state1 = getEndpointStateForEndpoint(ep1);
+        EndpointState state2 = getEndpointStateForEndpoint(ep2);
+
+        if (state1 == null || state2 == null)
+            return false;
+
+        VersionedValue value1 = state1.getApplicationState(as);
+        VersionedValue value2 = state2.getApplicationState(as);
+
+        return !(value1 == null || value2 == null) && value1.value.equals(value2.value);
+    }
+
     // removes ALL endpoint states; should only be called after shadow gossip
     public void resetEndpointStateMap()
     {

src/java/org/apache/cassandra/hints/ChecksummedDataInput.java

@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.hints;
+
+import java.io.IOException;
+import java.util.zip.CRC32;
+
+import org.apache.cassandra.io.util.AbstractDataInput;
+
+/**
+ * An {@link AbstractDataInput} wrapper that calctulates the CRC in place.
+ *
+ * Useful for {@link org.apache.cassandra.hints.HintsReader}, for example, where we must verify the CRC, yet don't want
+ * to allocate an extra byte array just that purpose.
+ *
+ * In addition to calculating the CRC, allows to enforce a maximim known size. This is needed
+ * so that {@link org.apache.cassandra.db.Mutation.MutationSerializer} doesn't blow up the heap when deserializing a
+ * corrupted sequence by reading a huge corrupted length of bytes via
+ * via {@link org.apache.cassandra.utils.ByteBufferUtil#readWithLength(java.io.DataInput)}.
+ */
+public final class ChecksummedDataInput extends AbstractDataInput
+{
+    private final CRC32 crc;
+    private final AbstractDataInput source;
+    private int limit;
+
+    private ChecksummedDataInput(AbstractDataInput source)
+    {
+        this.source = source;
+
+        crc = new CRC32();
+        limit = Integer.MAX_VALUE;
+    }
+
+    public static ChecksummedDataInput wrap(AbstractDataInput source)
+    {
+        return new ChecksummedDataInput(source);
+    }
+
+    public void resetCrc()
+    {
+        crc.reset();
+    }
+
+    public void resetLimit()
+    {
+        limit = Integer.MAX_VALUE;
+    }
+
+    public void limit(int newLimit)
+    {
+        limit = newLimit;
+    }
+
+    public int bytesRemaining()
+    {
+        return limit;
+    }
+
+    public int getCrc()
+    {
+        return (int) crc.getValue();
+    }
+
+    public void seek(long position) throws IOException
+    {
+        source.seek(position);
+    }
+
+    public long getPosition()
+    {
+        return source.getPosition();
+    }
+
+    public long getPositionLimit()
+    {
+        return source.getPositionLimit();
+    }
+
+    public int read() throws IOException
+    {
+        int b = source.read();
+        crc.update(b);
+        limit--;
+        return b;
+    }
+
+    @Override
+    public int read(byte[] buff, int offset, int length) throws IOException
+    {
+        if (length > limit)
+            throw new IOException("Digest mismatch exception");
+
+        int copied = source.read(buff, offset, length);
+        crc.update(buff, offset, copied);
+        limit -= copied;
+        return copied;
+    }
+}

src/java/org/apache/cassandra/hints/EncodedHintMessage.java

@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.hints;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.UUID;
+
+import org.apache.cassandra.db.TypeSizes;
+import org.apache.cassandra.io.IVersionedSerializer;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.net.MessageOut;
+import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.utils.UUIDSerializer;
+
+/**
+ * A specialized version of {@link HintMessage} that takes an already encoded in a bytebuffer hint and sends it verbatim.
+ *
+ * An optimization for when dispatching a hint file of the current messaging version to a node of the same messaging version,
+ * which is the most common case. Saves on extra ByteBuffer allocations one redundant hint deserialization-serialization cycle.
+ *
+ * Never deserialized as an EncodedHintMessage - the receiving side will always deserialize the message as vanilla
+ * {@link HintMessage}.
+ */
+final class EncodedHintMessage
+{
+    private static final IVersionedSerializer<EncodedHintMessage> serializer = new Serializer();
+
+    private final UUID hostId;
+    private final ByteBuffer hint;
+    private final int version;
+
+    EncodedHintMessage(UUID hostId, ByteBuffer hint, int version)
+    {
+        this.hostId = hostId;
+        this.hint = hint;
+        this.version = version;
+    }
+
+    MessageOut<EncodedHintMessage> createMessageOut()
+    {
+        return new MessageOut<>(MessagingService.Verb.HINT, this, serializer);
+    }
+
+    private static class Serializer implements IVersionedSerializer<EncodedHintMessage>
+    {
+        public long serializedSize(EncodedHintMessage message, int version)
+        {
+            if (version != message.version)
+                throw new IllegalArgumentException("serializedSize() called with non-matching version " + version);
+
+            int size = (int) UUIDSerializer.serializer.serializedSize(message.hostId, version);
+            size += TypeSizes.sizeof(message.hint.remaining());
+            size += message.hint.remaining();
+            return size;
+        }
+
+        public void serialize(EncodedHintMessage message, DataOutputPlus out, int version) throws IOException
+        {
+            if (version != message.version)
+                throw new IllegalArgumentException("serialize() called with non-matching version " + version);
+
+            UUIDSerializer.serializer.serialize(message.hostId, out, version);
+            out.writeInt(message.hint.remaining());
+            out.write(message.hint);
+        }
+
+        public EncodedHintMessage deserialize(DataInputPlus in, int version) throws IOException
+        {
+            throw new UnsupportedOperationException();
+        }
+    }
+}

src/java/org/apache/cassandra/hints/Hint.java

@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.hints;
+
+import java.io.IOException;
+import java.util.*;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.cassandra.db.*;
+import org.apache.cassandra.io.IVersionedSerializer;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputPlus;
+
+/**
+ * Encapsulates the hinted mutation, its creation time, and the gc grace seconds param for each table involved.
+ *
+ * - Why do we need to track hint creation time?
+ * - We must exclude updates for tables that have been truncated after hint's creation, otherwise the result is data corruption.
+ *
+ * - Why do we need to track gc grace seconds?
+ * - Hints can stay in storage for a while before being applied, and without recording gc grace seconds (+ creation time),
+ *   if we apply the mutation blindly, we risk resurrecting a deleted value, a tombstone for which had been already
+ *   compacted away while the hint was in storage.
+ *
+ *   We also look at the smallest current value of the gcgs param for each affected table when applying the hint, and use
+ *   creation time + min(recorded gc gs, current gcgs + current gc grace) as the overall hint expiration time.
+ *   This allows now to safely reduce gc gs on tables without worrying that an applied old hint might resurrect any data.
+ */
+public final class Hint
+{
+    public static final Serializer serializer = new Serializer();
+
+    final Mutation mutation;
+    final long creationTime;  // time of hint creation (in milliseconds)
+    final int gcgs; // the smallest gc gs of all involved tables
+
+    private Hint(Mutation mutation, long creationTime, int gcgs)
+    {
+        this.mutation = mutation;
+        this.creationTime = creationTime;
+        this.gcgs = gcgs;
+    }
+
+    /**
+     * @param mutation the hinted mutation
+     * @param creationTime time of this hint's creation (in milliseconds since epoch)
+     */
+    public static Hint create(Mutation mutation, long creationTime)
+    {
+        return new Hint(mutation, creationTime, mutation.smallestGCGS());
+    }
+
+    /**
+     * @param mutation the hinted mutation
+     * @param creationTime time of this hint's creation (in milliseconds since epoch)
+     * @param gcgs the smallest gcgs of all tables involved at the time of hint creation (in seconds)
+     */
+    public static Hint create(Mutation mutation, long creationTime, int gcgs)
+    {
+        return new Hint(mutation, creationTime, gcgs);
+    }
+
+    /**
+     * Applies the contained mutation unless it's expired, filtering out any updates for truncated tables
+     */
+    void apply()
+    {
+        if (!isLive())
+            return;
+
+        // filter out partition update for table that have been truncated since hint's creation
+        Mutation filtered = mutation;
+        for (UUID id : mutation.getColumnFamilyIds())
+            if (creationTime <= SystemKeyspace.getTruncatedAt(id))
+                filtered = filtered.without(id);
+
+        if (!filtered.isEmpty())
+            filtered.apply();
+    }
+
+    /**
+     * @return calculates whether or not it is safe to apply the hint without risking to resurrect any deleted data
+     */
+    boolean isLive()
+    {
+        int smallestGCGS = Math.min(gcgs, mutation.smallestGCGS());
+        long expirationTime = creationTime + TimeUnit.SECONDS.toMillis(smallestGCGS);
+        return expirationTime > System.currentTimeMillis();
+    }
+
+    static final class Serializer implements IVersionedSerializer<Hint>
+    {
+        public long serializedSize(Hint hint, int version)
+        {
+            long size = TypeSizes.sizeof(hint.creationTime);
+            size += TypeSizes.sizeof(hint.gcgs);
+            size += Mutation.serializer.serializedSize(hint.mutation, version);
+            return size;
+        }
+
+        public void serialize(Hint hint, DataOutputPlus out, int version) throws IOException
+        {
+            out.writeLong(hint.creationTime);
+            out.writeInt(hint.gcgs);
+            Mutation.serializer.serialize(hint.mutation, out, version);
+        }
+
+        public Hint deserialize(DataInputPlus in, int version) throws IOException
+        {
+            long creationTime = in.readLong();
+            int gcgs = in.readInt();
+            return new Hint(Mutation.serializer.deserialize(in, version), creationTime, gcgs);
+        }
+    }
+}

src/java/org/apache/cassandra/hints/HintMessage.java

@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.cassandra.hints;
+
+import java.io.IOException;
+import java.util.Objects;
+import java.util.UUID;
+
+import javax.annotation.Nullable;
+
+import org.apache.cassandra.db.TypeSizes;
+import org.apache.cassandra.db.UnknownColumnFamilyException;
+import org.apache.cassandra.io.IVersionedSerializer;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.net.MessageOut;
+import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.utils.BytesReadTracker;
+import org.apache.cassandra.utils.UUIDSerializer;
+
+/**
+ * The message we use to dispatch and forward hints.
+ *
+ * Encodes the host id the hint is meant for and the hint itself.
+ * We use the host id to determine whether we should store or apply the hint:
+ * 1. If host id equals to the receiving node host id, then we apply the hint
+ * 2. If host id is different from the receiving node's host id, then we store the hint
+ *
+ * Scenario (1) means that we are dealing with regular hint dispatch.
+ * Scenario (2) means that we got a hint from a node that's going through decommissioning and is streaming its hints
+ * elsewhere first.
+ */
+public final class HintMessage
+{
+    public static final IVersionedSerializer<HintMessage> serializer = new Serializer();
+
+    final UUID hostId;
+
+    @Nullable // can be null if we fail do decode the hint because of an unknown table id in it
+    final Hint hint;
+
+    @Nullable // will usually be null, unless a hint deserialization fails due to an unknown table id
+    final UUID unknownTableID;
+
+    HintMessage(UUID hostId, Hint hint)
+    {
+        this.hostId = hostId;
+        this.hint = hint;
+        this.unknownTableID = null;
+    }
+
+    HintMessage(UUID hostId, UUID unknownTableID)
+    {
+        this.hostId = hostId;
+        this.hint = null;
+        this.unknownTableID = unknownTableID;
+    }
+
+    public MessageOut<HintMessage> createMessageOut()
+    {
+        return new MessageOut<>(MessagingService.Verb.HINT, this, serializer);
+    }
+
+    public static class Serializer implements IVersionedSerializer<HintMessage>
+    {
+        public long serializedSize(HintMessage message, int version)
+        {
+            int size = (int) UUIDSerializer.serializer.serializedSize(message.hostId, version);
+
+            int hintSize = (int) Hint.serializer.serializedSize(message.hint, version);
+            size += TypeSizes.sizeof(hintSize);
+            size += hintSize;
+
+            return size;
+        }
+
+        public void serialize(HintMessage message, DataOutputPlus out, int version) throws IOException
+        {
+            Objects.requireNonNull(message.hint); // we should never *send* a HintMessage with null hint
+
+            UUIDSerializer.serializer.serialize(message.hostId, out, version);
+
+            /*
+             * We are serializing the hint size so that the receiver of the message could gracefully handle
+             * deserialize failure when a table had been dropped, by simply skipping the unread bytes.
+             */
+            out.writeInt((int) Hint.serializer.serializedSize(message.hint, version));
+
+            Hint.serializer.serialize(message.hint, out, version);
+        }
+
+        /*
+         * It's not an exceptional scenario to have a hints file streamed that have partition updates for tables
+         * that don't exist anymore. We want to handle that case gracefully instead of dropping the connection for every
+         * one of them.
+         */
+        public HintMessage deserialize(DataInputPlus in, int version) throws IOException
+        {
+            UUID hostId = UUIDSerializer.serializer.deserialize(in, version);
+
+            int hintSize = in.readInt();
+            BytesReadTracker countingIn = new BytesReadTracker(in);
+            try
+            {
+                return new HintMessage(hostId, Hint.serializer.deserialize(countingIn, version));
+            }
+            catch (UnknownColumnFamilyException e)
+            {
+                in.skipBytes(hintSize - (int) countingIn.getBytesRead());
+                return new HintMessage(hostId, e.cfId);
+            }
+        }
+    }
+}

src/java/org/apache/cassandra/hints/HintResponse.java

@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.cassandra.hints;
+
+import org.apache.cassandra.io.IVersionedSerializer;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.net.MessageOut;
+import org.apache.cassandra.net.MessagingService;
+
+/**
+ * An empty successful response to a HintMessage.
+ */
+public final class HintResponse
+{
+    public static final IVersionedSerializer<HintResponse> serializer = new Serializer();
+
+    static final HintResponse instance = new HintResponse();
+    static final MessageOut<HintResponse> message =
+        new MessageOut<>(MessagingService.Verb.REQUEST_RESPONSE, instance, serializer);
+
+    private HintResponse()
+    {
+    }
+
+    private static final class Serializer implements IVersionedSerializer<HintResponse>
+    {
+        public long serializedSize(HintResponse response, int version)
+        {
+            return 0;
+        }
+
+        public void serialize(HintResponse response, DataOutputPlus out, int version)
+        {
+        }
+
+        public HintResponse deserialize(DataInputPlus in, int version)
+        {
+            return instance;
+        }
+    }
+}

src/java/org/apache/cassandra/hints/HintVerbHandler.java

@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.cassandra.hints;
+
+import java.net.InetAddress;
+import java.util.UUID;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.db.partitions.PartitionUpdate;
+import org.apache.cassandra.net.IVerbHandler;
+import org.apache.cassandra.net.MessageIn;
+import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.serializers.MarshalException;
+import org.apache.cassandra.service.StorageService;
+
+/**
+ * Verb handler used both for hint dispatch and streaming.
+ *
+ * With the non-sstable format, we cannot just stream hint sstables on node decommission. So sometimes, at decommission
+ * time, we might have to stream hints to a non-owning host (say, if the owning host B is down during decommission of host A).
+ * In that case the handler just stores the received hint in its local hint store.
+ */
+public final class HintVerbHandler implements IVerbHandler<HintMessage>
+{
+    private static final Logger logger = LoggerFactory.getLogger(HintVerbHandler.class);
+
+    public void doVerb(MessageIn<HintMessage> message, int id)
+    {
+        UUID hostId = message.payload.hostId;
+        Hint hint = message.payload.hint;
+
+        // If we see an unknown table id, it means the table, or one of the tables in the mutation, had been dropped.
+        // In that case there is nothing we can really do, or should do, other than log it go on.
+        // This will *not* happen due to a not-yet-seen table, because we don't transfer hints unless there
+        // is schema agreement between the sender and the receiver.
+        if (hint == null)
+        {
+            logger.debug("Failed to decode and apply a hint for {} - table with id {} is unknown",
+                         hostId,
+                         message.payload.unknownTableID);
+            reply(id, message.from);
+            return;
+        }
+
+        // We must perform validation before applying the hint, and there is no other place to do it other than here.
+        try
+        {
+            hint.mutation.getPartitionUpdates().forEach(PartitionUpdate::validate);
+        }
+        catch (MarshalException e)
+        {
+            logger.warn("Failed to validate a hint for {} (table id {}) - skipped", hostId);
+            reply(id, message.from);
+            return;
+        }
+
+        // Apply the hint if this node is the destination, store for future dispatch if this node isn't (must have gotten
+        // it from a decommissioned node that had streamed it before going out).
+        if (hostId.equals(StorageService.instance.getLocalHostUUID()))
+            hint.apply();
+        else
+            HintsService.instance.write(hostId, hint);
+
+        reply(id, message.from);
+    }
+
+    private static void reply(int id, InetAddress to)
+    {
+        MessagingService.instance().sendReply(HintResponse.message, id, to);
+    }
+}

src/java/org/apache/cassandra/hints/HintsBuffer.java

@@ -0,0 +1,261 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.hints;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.*;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.zip.CRC32;
+
+import org.apache.cassandra.io.util.DataOutputBufferFixed;
+import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.utils.AbstractIterator;
+import org.apache.cassandra.utils.concurrent.OpOrder;
+
+import static org.apache.cassandra.utils.FBUtilities.updateChecksum;
+import static org.apache.cassandra.utils.FBUtilities.updateChecksumInt;
+
+/**
+ * A shared buffer that temporarily holds the serialized hints before they are flushed to disk.
+ *
+ * Consists of :
+ * - a ByteBuffer holding the serialized hints (length, length checksum and total checksum included)
+ * - a pointer to the current allocation offset
+ * - an {@link OpOrder} appendOrder for {@link HintsWriteExecutor} to wait on for all writes completion
+ * - a map of (host id -> offset queue) for the hints written
+ *
+ * It's possible to write a single hint for two or more hosts at the same time, in which case the same offset will be put
+ * into two or more offset queues.
+ */
+final class HintsBuffer
+{
+    // hint entry overhead in bytes (int length, int length checksum, int body checksum)
+    static final int ENTRY_OVERHEAD_SIZE = 12;
+    static final int CLOSED = -1;
+
+    private final ByteBuffer slab; // the underlying backing ByteBuffer for all the serialized hints
+    private final AtomicInteger position; // the position in the slab that we currently allocate from
+
+    private final ConcurrentMap<UUID, Queue<Integer>> offsets;
+    private final OpOrder appendOrder;
+
+    private HintsBuffer(ByteBuffer slab)
+    {
+        this.slab = slab;
+
+        position = new AtomicInteger();
+        offsets = new ConcurrentHashMap<>();
+        appendOrder = new OpOrder();
+    }
+
+    static HintsBuffer create(int slabSize)
+    {
+        return new HintsBuffer(ByteBuffer.allocateDirect(slabSize));
+    }
+
+    boolean isClosed()
+    {
+        return position.get() == CLOSED;
+    }
+
+    int capacity()
+    {
+        return slab.capacity();
+    }
+
+    int remaining()
+    {
+        int pos = position.get();
+        return pos == CLOSED ? 0 : capacity() - pos;
+    }
+
+    HintsBuffer recycle()
+    {
+        slab.clear();
+        return new HintsBuffer(slab);
+    }
+
+    void free()
+    {
+        FileUtils.clean(slab);
+    }
+
+    /**
+     * Wait for any appends started before this method was called.
+     */
+    void waitForModifications()
+    {
+        appendOrder.awaitNewBarrier(); // issue a barrier and wait for it
+    }
+
+    Set<UUID> hostIds()
+    {
+        return offsets.keySet();
+    }
+
+    /**
+     * Coverts the queue of offsets for the selected host id into an iterator of hints encoded as ByteBuffers.
+     */
+    Iterator<ByteBuffer> consumingHintsIterator(UUID hostId)
+    {
+        final Queue<Integer> bufferOffsets = offsets.get(hostId);
+
+        if (bufferOffsets == null)
+            return Collections.emptyIterator();
+
+        return new AbstractIterator<ByteBuffer>()
+        {
+            private final ByteBuffer flyweight = slab.duplicate();
+
+            protected ByteBuffer computeNext()
+            {
+                Integer offset = bufferOffsets.poll();
+
+                if (offset == null)
+                    return endOfData();
+
+                int totalSize = slab.getInt(offset) + ENTRY_OVERHEAD_SIZE;
+
+                return (ByteBuffer) flyweight.clear().position(offset).limit(offset + totalSize);
+            }
+        };
+    }
+
+    Allocation allocate(int hintSize)
+    {
+        int totalSize = hintSize + ENTRY_OVERHEAD_SIZE;
+
+        if (totalSize > slab.capacity() / 2)
+        {
+            throw new IllegalArgumentException(String.format("Hint of %s bytes is too large - the maximum size is %s",
+                                                             hintSize,
+                                                             slab.capacity() / 2));
+        }
+
+        @SuppressWarnings("resource")
+        OpOrder.Group opGroup = appendOrder.start(); // will eventually be closed by the receiver of the allocation
+        try
+        {
+            return allocate(totalSize, opGroup);
+        }
+        catch (Throwable t)
+        {
+            opGroup.close();
+            throw t;
+        }
+    }
+
+    private Allocation allocate(int totalSize, OpOrder.Group opGroup)
+    {
+        int offset = allocateBytes(totalSize);
+        if (offset < 0)
+        {
+            opGroup.close();
+            return null;
+        }
+        return new Allocation(offset, totalSize, opGroup);
+    }
+
+    private int allocateBytes(int totalSize)
+    {
+        while (true)
+        {
+            int prev = position.get();
+            int next = prev + totalSize;
+
+            if (prev == CLOSED) // the slab has been 'closed'
+                return CLOSED;
+
+            if (next > slab.capacity())
+            {
+                position.set(CLOSED); // mark the slab as no longer allocating if we've exceeded its capacity
+                return CLOSED;
+            }
+
+            if (position.compareAndSet(prev, next))
+                return prev;
+        }
+    }
+
+    private void put(UUID hostId, int offset)
+    {
+        // we intentionally don't just return offsets.computeIfAbsent() because it's expensive compared to simple get(),
+        // and the method is on a really hot path
+        Queue<Integer> queue = offsets.get(hostId);
+        if (queue == null)
+            queue = offsets.computeIfAbsent(hostId, (id) -> new ConcurrentLinkedQueue<>());
+        queue.offer(offset);
+    }
+
+    /**
+     * A placeholder for hint serialization. Should always be used in a try-with-resources block.
+     */
+    final class Allocation implements AutoCloseable
+    {
+        private final Integer offset;
+        private final int totalSize;
+        private final OpOrder.Group opGroup;
+
+        Allocation(int offset, int totalSize, OpOrder.Group opGroup)
+        {
+            this.offset = offset;
+            this.totalSize = totalSize;
+            this.opGroup = opGroup;
+        }
+
+        void write(Iterable<UUID> hostIds, Hint hint)
+        {
+            write(hint);
+            for (UUID hostId : hostIds)
+                put(hostId, offset);
+        }
+
+        public void close()
+        {
+            opGroup.close();
+        }
+
+        private void write(Hint hint)
+        {
+            ByteBuffer buffer = (ByteBuffer) slab.duplicate().position(offset).limit(offset + totalSize);
+            DataOutputPlus dop = new DataOutputBufferFixed(buffer);
+            CRC32 crc = new CRC32();
+            int hintSize = totalSize - ENTRY_OVERHEAD_SIZE;
+            try
+            {
+                dop.writeInt(hintSize);
+                updateChecksumInt(crc, hintSize);
+                dop.writeInt((int) crc.getValue());
+
+                Hint.serializer.serialize(hint, dop, MessagingService.current_version);
+                updateChecksum(crc, buffer, buffer.position() - hintSize, hintSize);
+                dop.writeInt((int) crc.getValue());
+            }
+            catch (IOException e)
+            {
+                throw new AssertionError(); // cannot happen
+            }
+        }
+    }
+}

src/java/org/apache/cassandra/hints/HintsBufferPool.java

@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.hints;
+
+import java.util.Queue;
+import java.util.UUID;
+import java.util.concurrent.ConcurrentLinkedQueue;
+
+import org.apache.cassandra.net.MessagingService;
+
+/**
+ * A primitive pool of {@link HintsBuffer} buffers. Under normal conditions should only hold two buffers - the currently
+ * written to one, and a reserve buffer to switch to when the first one is beyond capacity.
+ */
+final class HintsBufferPool
+{
+    interface FlushCallback
+    {
+        void flush(HintsBuffer buffer, HintsBufferPool pool);
+    }
+
+    private volatile HintsBuffer currentBuffer;
+    private final Queue<HintsBuffer> reserveBuffers;
+    private final int bufferSize;
+    private final FlushCallback flushCallback;
+
+    HintsBufferPool(int bufferSize, FlushCallback flushCallback)
+    {
+        reserveBuffers = new ConcurrentLinkedQueue<>();
+
+        this.bufferSize = bufferSize;
+        this.flushCallback = flushCallback;
+    }
+
+    /**
+     * @param hostIds host ids of the hint's target nodes
+     * @param hint the hint to store
+     */
+    void write(Iterable<UUID> hostIds, Hint hint)
+    {
+        int hintSize = (int) Hint.serializer.serializedSize(hint, MessagingService.current_version);
+        try (HintsBuffer.Allocation allocation = allocate(hintSize))
+        {
+            allocation.write(hostIds, hint);
+        }
+    }
+
+    private HintsBuffer.Allocation allocate(int hintSize)
+    {
+        HintsBuffer current = currentBuffer();
+
+        while (true)
+        {
+            HintsBuffer.Allocation allocation = current.allocate(hintSize);
+            if (allocation != null)
+                return allocation;
+
+            // allocation failed due to insufficient size remaining in the buffer
+            if (switchCurrentBuffer(current))
+                flushCallback.flush(current, this);
+
+            current = currentBuffer;
+        }
+    }
+
+    boolean offer(HintsBuffer buffer)
+    {
+        if (!reserveBuffers.isEmpty())
+            return false;
+
+        reserveBuffers.offer(buffer);
+        return true;
+    }
+
+    // A wrapper to ensure a non-null currentBuffer value on the first call.
+    HintsBuffer currentBuffer()
+    {
+        if (currentBuffer == null)
+            initializeCurrentBuffer();
+
+        return currentBuffer;
+    }
+
+    private synchronized void initializeCurrentBuffer()
+    {
+        if (currentBuffer == null)
+            currentBuffer = createBuffer();
+    }
+
+    private synchronized boolean switchCurrentBuffer(HintsBuffer previous)
+    {
+        if (currentBuffer != previous)
+            return false;
+
+        HintsBuffer buffer = reserveBuffers.poll();
+        currentBuffer = buffer == null ? createBuffer() : buffer;
+
+        return true;
+    }
+
+    private HintsBuffer createBuffer()
+    {
+        return HintsBuffer.create(bufferSize);
+    }
+}

src/java/org/apache/cassandra/hints/HintsCatalog.java

@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.hints;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.util.*;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Stream;
+
+import org.apache.cassandra.io.FSReadError;
+import org.apache.cassandra.utils.CLibrary;
+import org.apache.cassandra.utils.SyncUtil;
+
+import static java.util.stream.Collectors.groupingBy;
+
+/**
+ * A simple catalog for easy host id -> {@link HintsStore} lookup and manipulation.
+ */
+final class HintsCatalog
+{
+    private final File hintsDirectory;
+    private final Map<UUID, HintsStore> stores;
+
+    private HintsCatalog(File hintsDirectory, Map<UUID, List<HintsDescriptor>> descriptors)
+    {
+        this.hintsDirectory = hintsDirectory;
+        this.stores = new ConcurrentHashMap<>();
+
+        for (Map.Entry<UUID, List<HintsDescriptor>> entry : descriptors.entrySet())
+            stores.put(entry.getKey(), HintsStore.create(entry.getKey(), hintsDirectory, entry.getValue()));
+    }
+
+    /**
+     * Loads hints stores from a given directory.
+     */
+    static HintsCatalog load(File hintsDirectory)
+    {
+        try
+        {
+            Map<UUID, List<HintsDescriptor>> stores =
+                Files.list(hintsDirectory.toPath())
+                     .filter(HintsDescriptor::isHintFileName)
+                     .map(HintsDescriptor::readFromFile)
+                     .collect(groupingBy(h -> h.hostId));
+            return new HintsCatalog(hintsDirectory, stores);
+        }
+        catch (IOException e)
+        {
+            throw new FSReadError(e, hintsDirectory);
+        }
+    }
+
+    Stream<HintsStore> stores()
+    {
+        return stores.values().stream();
+    }
+
+    void maybeLoadStores(Iterable<UUID> hostIds)
+    {
+        for (UUID hostId : hostIds)
+            get(hostId);
+    }
+
+    HintsStore get(UUID hostId)
+    {
+        // we intentionally don't just return stores.computeIfAbsent() because it's expensive compared to simple get(),
+        // and in this case would also allocate for the capturing lambda; the method is on a really hot path
+        HintsStore store = stores.get(hostId);
+        return store == null
+             ? stores.computeIfAbsent(hostId, (id) -> HintsStore.create(id, hintsDirectory, Collections.emptyList()))
+             : store;
+    }
+
+    /**
+     * Delete all hints for all host ids.
+     *
+     * Will not delete the files that are currently being dispatched, or written to.
+     */
+    void deleteAllHints()
+    {
+        stores.keySet().forEach(this::deleteAllHints);
+    }
+
+    /**
+     * Delete all hints for the specified host id.
+     *
+     * Will not delete the files that are currently being dispatched, or written to.
+     */
+    void deleteAllHints(UUID hostId)
+    {
+        HintsStore store = stores.get(hostId);
+        if (store != null)
+            store.deleteAllHints();
+    }
+
+    void exciseStore(UUID hostId)
+    {
+        deleteAllHints(hostId);
+        stores.remove(hostId);
+    }
+
+    void fsyncDirectory()
+    {
+        int fd = CLibrary.tryOpenDirectory(hintsDirectory.getAbsolutePath());
+        if (fd != -1)
+        {
+            SyncUtil.trySync(fd);
+            CLibrary.tryCloseFD(fd);
+        }
+    }
+}

src/java/org/apache/cassandra/hints/HintsDescriptor.java

@@ -0,0 +1,242 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.hints;
+
+import java.io.DataInput;
+import java.io.IOException;
+import java.io.RandomAccessFile;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Path;
+import java.util.Map;
+import java.util.UUID;
+import java.util.regex.Pattern;
+import java.util.zip.CRC32;
+
+import com.google.common.base.MoreObjects;
+import com.google.common.base.Objects;
+import com.google.common.collect.ImmutableMap;
+
+import org.apache.cassandra.db.TypeSizes;
+import org.apache.cassandra.io.FSReadError;
+import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.net.MessagingService;
+import org.json.simple.JSONValue;
+
+import static org.apache.cassandra.utils.FBUtilities.updateChecksumInt;
+
+/**
+ * Describes the host id, the version, the timestamp of creation, and an arbitrary map of JSON-encoded parameters of a
+ * hints file.
+ *
+ * Written in the beginning of each hints file.
+ */
+final class HintsDescriptor
+{
+    static final int VERSION_30 = 1;
+    static final int CURRENT_VERSION = VERSION_30;
+
+    static final Pattern pattern =
+        Pattern.compile("^[a-fA-F0-9]{8}\\-[a-fA-F0-9]{4}\\-[a-fA-F0-9]{4}\\-[a-fA-F0-9]{4}\\-[a-fA-F0-9]{12}\\-(\\d+)\\-(\\d+)\\.hints$");
+
+    final UUID hostId;
+    final int version;
+    final long timestamp;
+
+    // implemented for future compression support - see CASSANDRA-9428
+    final ImmutableMap<String, Object> parameters;
+
+    HintsDescriptor(UUID hostId, int version, long timestamp, ImmutableMap<String, Object> parameters)
+    {
+        this.hostId = hostId;
+        this.version = version;
+        this.timestamp = timestamp;
+        this.parameters = parameters;
+    }
+
+    HintsDescriptor(UUID hostId, long timestamp)
+    {
+        this(hostId, CURRENT_VERSION, timestamp, ImmutableMap.<String, Object>of());
+    }
+
+    String fileName()
+    {
+        return String.format("%s-%s-%s.hints", hostId, timestamp, version);
+    }
+
+    String checksumFileName()
+    {
+        return String.format("%s-%s-%s.crc32", hostId, timestamp, version);
+    }
+
+    int messagingVersion()
+    {
+        return messagingVersion(version);
+    }
+
+    static int messagingVersion(int hintsVersion)
+    {
+        switch (hintsVersion)
+        {
+            case VERSION_30:
+                return MessagingService.VERSION_30;
+            default:
+                throw new AssertionError();
+        }
+    }
+
+    static boolean isHintFileName(Path path)
+    {
+        return pattern.matcher(path.getFileName().toString()).matches();
+    }
+
+    static HintsDescriptor readFromFile(Path path)
+    {
+        try (RandomAccessFile raf = new RandomAccessFile(path.toFile(), "r"))
+        {
+            return deserialize(raf);
+        }
+        catch (IOException e)
+        {
+            throw new FSReadError(e, path.toFile());
+        }
+    }
+
+    @Override
+    public String toString()
+    {
+        return MoreObjects.toStringHelper(this)
+                          .add("hostId", hostId)
+                          .add("version", version)
+                          .add("timestamp", timestamp)
+                          .add("parameters", parameters)
+                          .toString();
+    }
+
+    @Override
+    public boolean equals(Object o)
+    {
+        if (this == o)
+            return true;
+
+        if (!(o instanceof HintsDescriptor))
+            return false;
+
+        HintsDescriptor hd = (HintsDescriptor) o;
+
+        return Objects.equal(hostId, hd.hostId)
+            && Objects.equal(version, hd.version)
+            && Objects.equal(timestamp, hd.timestamp)
+            && Objects.equal(parameters, hd.parameters);
+    }
+
+    @Override
+    public int hashCode()
+    {
+        return Objects.hashCode(hostId, version, timestamp, parameters);
+    }
+
+    void serialize(DataOutputPlus out) throws IOException
+    {
+        CRC32 crc = new CRC32();
+
+        out.writeInt(version);
+        updateChecksumInt(crc, version);
+
+        out.writeLong(timestamp);
+        updateChecksumLong(crc, timestamp);
+
+        out.writeLong(hostId.getMostSignificantBits());
+        updateChecksumLong(crc, hostId.getMostSignificantBits());
+        out.writeLong(hostId.getLeastSignificantBits());
+        updateChecksumLong(crc, hostId.getLeastSignificantBits());
+
+        byte[] paramsBytes = JSONValue.toJSONString(parameters).getBytes(StandardCharsets.UTF_8);
+        out.writeInt(paramsBytes.length);
+        updateChecksumInt(crc, paramsBytes.length);
+        out.writeInt((int) crc.getValue());
+
+        out.write(paramsBytes);
+        crc.update(paramsBytes, 0, paramsBytes.length);
+
+        out.writeInt((int) crc.getValue());
+    }
+
+    int serializedSize()
+    {
+        int size = TypeSizes.sizeof(version);
+        size += TypeSizes.sizeof(timestamp);
+
+        size += TypeSizes.sizeof(hostId.getMostSignificantBits());
+        size += TypeSizes.sizeof(hostId.getLeastSignificantBits());
+
+        byte[] paramsBytes = JSONValue.toJSONString(parameters).getBytes(StandardCharsets.UTF_8);
+        size += TypeSizes.sizeof(paramsBytes.length);
+        size += 4; // size checksum
+        size += paramsBytes.length;
+        size += 4; // total checksum
+
+        return size;
+    }
+
+    static HintsDescriptor deserialize(DataInput in) throws IOException
+    {
+        CRC32 crc = new CRC32();
+
+        int version = in.readInt();
+        updateChecksumInt(crc, version);
+
+        long timestamp = in.readLong();
+        updateChecksumLong(crc, timestamp);
+
+        long msb = in.readLong();
+        updateChecksumLong(crc, msb);
+        long lsb = in.readLong();
+        updateChecksumLong(crc, lsb);
+
+        UUID hostId = new UUID(msb, lsb);
+
+        int paramsLength = in.readInt();
+        updateChecksumInt(crc, paramsLength);
+        validateCRC(in.readInt(), (int) crc.getValue());
+
+        byte[] paramsBytes = new byte[paramsLength];
+        in.readFully(paramsBytes, 0, paramsLength);
+        crc.update(paramsBytes, 0, paramsLength);
+        validateCRC(in.readInt(), (int) crc.getValue());
+
+        return new HintsDescriptor(hostId, version, timestamp, decodeJSONBytes(paramsBytes));
+    }
+
+    @SuppressWarnings("unchecked")
+    private static ImmutableMap<String, Object> decodeJSONBytes(byte[] bytes)
+    {
+        return ImmutableMap.copyOf((Map<String, Object>) JSONValue.parse(new String(bytes, StandardCharsets.UTF_8)));
+    }
+
+    private static void updateChecksumLong(CRC32 crc, long value)
+    {
+        updateChecksumInt(crc, (int) (value & 0xFFFFFFFFL));
+        updateChecksumInt(crc, (int) (value >>> 32));
+    }
+
+    private static void validateCRC(int expected, int actual) throws IOException
+    {
+        if (expected != actual)
+            throw new IOException("Hints Descriptor CRC Mismatch");
+    }
+}

src/java/org/apache/cassandra/hints/HintsDispatchExecutor.java

@@ -0,0 +1,199 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.hints;
+
+import java.io.File;
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.*;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import com.google.common.util.concurrent.RateLimiter;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.concurrent.JMXEnabledThreadPoolExecutor;
+import org.apache.cassandra.concurrent.NamedThreadFactory;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.io.FSReadError;
+import org.apache.cassandra.service.StorageService;
+
+/**
+ * A multi-threaded (by default) executor for dispatching hints.
+ *
+ * Most of dispatch is triggered by {@link HintsDispatchTrigger} running every ~10 seconds.
+ */
+final class HintsDispatchExecutor
+{
+    private static final Logger logger = LoggerFactory.getLogger(HintsDispatchExecutor.class);
+
+    private final File hintsDirectory;
+    private final ExecutorService executor;
+    private final AtomicBoolean isPaused;
+    private final Map<UUID, Future> scheduledDispatches;
+
+    HintsDispatchExecutor(File hintsDirectory, int maxThreads, AtomicBoolean isPaused)
+    {
+        this.hintsDirectory = hintsDirectory;
+        this.isPaused = isPaused;
+
+        scheduledDispatches = new ConcurrentHashMap<>();
+        executor = new JMXEnabledThreadPoolExecutor(1,
+                                                    maxThreads,
+                                                    1,
+                                                    TimeUnit.MINUTES,
+                                                    new LinkedBlockingQueue<>(),
+                                                    new NamedThreadFactory("HintsDispatcher", Thread.MIN_PRIORITY),
+                                                    "internal");
+    }
+
+    /*
+     * It's safe to terminate dispatch in process and to deschedule dispatch.
+     */
+    void shutdownBlocking()
+    {
+        scheduledDispatches.clear();
+        executor.shutdownNow();
+    }
+
+    boolean isScheduled(HintsStore store)
+    {
+        return scheduledDispatches.containsKey(store.hostId);
+    }
+
+    Future dispatch(HintsStore store)
+    {
+        return dispatch(store, store.hostId);
+    }
+
+    Future dispatch(HintsStore store, UUID hostId)
+    {
+        /*
+         * It is safe to perform dispatch for the same host id concurrently in two or more threads,
+         * however there is nothing to win from it - so we don't.
+         *
+         * Additionally, having just one dispatch task per host id ensures that we'll never violate our per-destination
+         * rate limit, without having to share a ratelimiter between threads.
+         *
+         * It also simplifies reasoning about dispatch sessions.
+         */
+        return scheduledDispatches.computeIfAbsent(store.hostId, uuid -> executor.submit(new DispatchHintsTask(store, hostId)));
+    }
+
+    void completeDispatchBlockingly(HintsStore store)
+    {
+        Future future = scheduledDispatches.get(store.hostId);
+        try
+        {
+            if (future != null)
+                future.get();
+        }
+        catch (ExecutionException | InterruptedException e)
+        {
+            throw new RuntimeException(e);
+        }
+    }
+
+    private final class DispatchHintsTask implements Runnable
+    {
+        private final HintsStore store;
+        private final UUID hostId;
+        private final RateLimiter rateLimiter;
+
+        DispatchHintsTask(HintsStore store, UUID hostId)
+        {
+            this.store = store;
+            this.hostId = hostId;
+
+            // rate limit is in bytes per second. Uses Double.MAX_VALUE if disabled (set to 0 in cassandra.yaml).
+            // max rate is scaled by the number of nodes in the cluster (CASSANDRA-5272).
+            // the goal is to bound maximum hints traffic going towards a particular node from the rest of the cluster,
+            // not total outgoing hints traffic from this node - this is why the rate limiter is not shared between
+            // all the dispatch tasks (as there will be at most one dispatch task for a particular host id at a time).
+            int nodesCount = Math.max(1, StorageService.instance.getTokenMetadata().getAllEndpoints().size() - 1);
+            int throttleInKB = DatabaseDescriptor.getHintedHandoffThrottleInKB() / nodesCount;
+            this.rateLimiter = RateLimiter.create(throttleInKB == 0 ? Double.MAX_VALUE : throttleInKB * 1024);
+        }
+
+        public void run()
+        {
+            try
+            {
+                dispatch();
+            }
+            finally
+            {
+                scheduledDispatches.remove(hostId);
+            }
+        }
+
+        private void dispatch()
+        {
+            while (true)
+            {
+                if (isPaused.get())
+                    break;
+
+                HintsDescriptor descriptor = store.poll();
+                if (descriptor == null)
+                    break;
+
+                try
+                {
+                    dispatch(descriptor);
+                }
+                catch (FSReadError e)
+                {
+                    logger.error("Failed to dispatch hints file {}: file is corrupted ({})", descriptor.fileName(), e);
+                    store.cleanUp(descriptor);
+                    store.blacklist(descriptor);
+                    throw e;
+                }
+            }
+        }
+
+        private void dispatch(HintsDescriptor descriptor)
+        {
+            logger.debug("Dispatching hints file {}", descriptor.fileName());
+
+            File file = new File(hintsDirectory, descriptor.fileName());
+            Long offset = store.getDispatchOffset(descriptor).orElse(null);
+
+            try (HintsDispatcher dispatcher = HintsDispatcher.create(file, rateLimiter, hostId, isPaused))
+            {
+                if (offset != null)
+                    dispatcher.seek(offset);
+
+                if (dispatcher.dispatch())
+                {
+                    if (!file.delete())
+                        logger.error("Failed to delete hints file {}", descriptor.fileName());
+                    store.cleanUp(descriptor);
+                    logger.info("Finished hinted handoff of file {} to endpoint {}", descriptor.fileName(), hostId);
+                }
+                else
+                {
+                    store.markDispatchOffset(descriptor, dispatcher.dispatchOffset());
+                    store.offerFirst(descriptor);
+                    logger.info("Finished hinted handoff of file {} to endpoint {}, partially", descriptor.fileName(), hostId);
+                }
+            }
+        }
+    }
+}

src/java/org/apache/cassandra/hints/HintsDispatchTrigger.java

@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.hints;
+
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.cassandra.gms.ApplicationState;
+import org.apache.cassandra.gms.FailureDetector;
+import org.apache.cassandra.gms.Gossiper;
+
+import static org.apache.cassandra.utils.FBUtilities.getBroadcastAddress;
+
+/**
+ * A simple dispatch trigger that's being run every 10 seconds.
+ *
+ * Goes through all hint stores and schedules for dispatch all the hints for hosts that are:
+ * 1. Not currently scheduled for dispatch, and
+ * 2. Either have some hint files, or an active hint writer, and
+ * 3. Are live, and
+ * 4. Have matching schema versions
+ *
+ * What does triggering a hints store for dispatch mean?
+ * - If there are existing hint files, it means submitting them for dispatch;
+ * - If there is an active writer, closing it, for the next run to pick it up.
+ */
+final class HintsDispatchTrigger implements Runnable
+{
+    private final HintsCatalog catalog;
+    private final HintsWriteExecutor writeExecutor;
+    private final HintsDispatchExecutor dispatchExecutor;
+    private final AtomicBoolean isPaused;
+
+    HintsDispatchTrigger(HintsCatalog catalog,
+                         HintsWriteExecutor writeExecutor,
+                         HintsDispatchExecutor dispatchExecutor,
+                         AtomicBoolean isPaused)
+    {
+        this.catalog = catalog;
+        this.writeExecutor = writeExecutor;
+        this.dispatchExecutor = dispatchExecutor;
+        this.isPaused = isPaused;
+    }
+
+    public void run()
+    {
+        if (isPaused.get())
+            return;
+
+        catalog.stores()
+               .filter(store -> !isScheduled(store))
+               .filter(HintsStore::isLive)
+               .filter(store -> store.isWriting() || store.hasFiles())
+               .filter(store -> Gossiper.instance.valuesEqual(getBroadcastAddress(), store.address(), ApplicationState.SCHEMA))
+               .forEach(this::schedule);
+    }
+
+    private void schedule(HintsStore store)
+    {
+        if (store.hasFiles())
+            dispatchExecutor.dispatch(store);
+
+        if (store.isWriting())
+            writeExecutor.closeWriter(store);
+    }
+
+    private boolean isScheduled(HintsStore store)
+    {
+        return dispatchExecutor.isScheduled(store);
+    }
+}

src/java/org/apache/cassandra/hints/HintsDispatcher.java

@@ -0,0 +1,228 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.hints;
+
+import java.io.File;
+import java.net.InetAddress;
+import java.nio.ByteBuffer;
+import java.util.*;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.function.Function;
+
+import com.google.common.util.concurrent.RateLimiter;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.gms.FailureDetector;
+import org.apache.cassandra.net.IAsyncCallbackWithFailure;
+import org.apache.cassandra.net.MessageIn;
+import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.utils.concurrent.SimpleCondition;
+
+/**
+ * Dispatches a single hints file to a specified node in a batched manner.
+ *
+ * Uses either {@link EncodedHintMessage} - when dispatching hints into a node with the same messaging version as the hints file,
+ * or {@link HintMessage}, when conversion is required.
+ */
+final class HintsDispatcher implements AutoCloseable
+{
+    private enum Action { CONTINUE, ABORT, RETRY }
+
+    private final HintsReader reader;
+    private final UUID hostId;
+    private final InetAddress address;
+    private final int messagingVersion;
+    private final AtomicBoolean isPaused;
+
+    private long currentPageOffset;
+
+    private HintsDispatcher(HintsReader reader, UUID hostId, InetAddress address, int messagingVersion, AtomicBoolean isPaused)
+    {
+        currentPageOffset = 0L;
+
+        this.reader = reader;
+        this.hostId = hostId;
+        this.address = address;
+        this.messagingVersion = messagingVersion;
+        this.isPaused = isPaused;
+    }
+
+    static HintsDispatcher create(File file, RateLimiter rateLimiter, UUID hostId, AtomicBoolean isPaused)
+    {
+        InetAddress address = StorageService.instance.getEndpointForHostId(hostId);
+        int messagingVersion = MessagingService.instance().getVersion(address);
+        return new HintsDispatcher(HintsReader.open(file, rateLimiter), hostId, address, messagingVersion, isPaused);
+    }
+
+    public void close()
+    {
+        reader.close();
+    }
+
+    void seek(long bytes)
+    {
+        reader.seek(bytes);
+        currentPageOffset = 0L;
+    }
+
+    /**
+     * @return whether or not dispatch completed entirely and successfully
+     */
+    boolean dispatch()
+    {
+        for (HintsReader.Page page : reader)
+        {
+            currentPageOffset = page.offset;
+            if (dispatch(page) != Action.CONTINUE)
+                return false;
+        }
+
+        return true;
+    }
+
+    /**
+     * @return offset of the first non-delivered page
+     */
+    long dispatchOffset()
+    {
+        return currentPageOffset;
+    }
+
+    private boolean isHostAlive()
+    {
+        return FailureDetector.instance.isAlive(address);
+    }
+
+    private boolean isPaused()
+    {
+        return isPaused.get();
+    }
+
+    // retry in case of a timeout; stop in case of a failure, host going down, or delivery paused
+    private Action dispatch(HintsReader.Page page)
+    {
+        Action action = sendHintsAndAwait(page);
+        return action == Action.RETRY
+             ? dispatch(page)
+             : action;
+    }
+
+    private Action sendHintsAndAwait(HintsReader.Page page)
+    {
+        Collection<Callback> callbacks = new ArrayList<>();
+
+        /*
+         * If hints file messaging version matches the version of the target host, we'll use the optimised path -
+         * skipping the redundant decoding/encoding cycle of the already encoded hint.
+         *
+         * If that is not the case, we'll need to perform conversion to a newer (or an older) format, and decoding the hint
+         * is an unavoidable intermediate step.
+         */
+        Action action = reader.descriptor().messagingVersion() == messagingVersion
+                      ? sendHints(page.buffersIterator(), callbacks, this::sendEncodedHint)
+                      : sendHints(page.hintsIterator(), callbacks, this::sendHint);
+
+        if (action == Action.ABORT)
+            return action;
+
+        for (Callback cb : callbacks)
+            if (cb.await() != Callback.Outcome.SUCCESS)
+                return Action.RETRY;
+
+        return Action.CONTINUE;
+    }
+
+    /*
+     * Sending hints in compatibility mode.
+     */
+    private <T> Action sendHints(Iterator<T> hints, Collection<Callback> callbacks, Function<T, Callback> sendFunction)
+    {
+        while (hints.hasNext())
+        {
+            if (!isHostAlive() || isPaused())
+                return Action.ABORT;
+            callbacks.add(sendFunction.apply(hints.next()));
+        }
+        return Action.CONTINUE;
+    }
+
+    private Callback sendHint(Hint hint)
+    {
+        Callback callback = new Callback();
+        HintMessage message = new HintMessage(hostId, hint);
+        MessagingService.instance().sendRRWithFailure(message.createMessageOut(), address, callback);
+        return callback;
+    }
+
+    /*
+     * Sending hints in raw mode.
+     */
+
+    private Callback sendEncodedHint(ByteBuffer hint)
+    {
+        Callback callback = new Callback();
+        EncodedHintMessage message = new EncodedHintMessage(hostId, hint, messagingVersion);
+        MessagingService.instance().sendRRWithFailure(message.createMessageOut(), address, callback);
+        return callback;
+    }
+
+    private static final class Callback implements IAsyncCallbackWithFailure
+    {
+        enum Outcome { SUCCESS, TIMEOUT, FAILURE }
+
+        private final long start = System.nanoTime();
+        private final SimpleCondition condition = new SimpleCondition();
+        private volatile Outcome outcome;
+
+        Outcome await()
+        {
+            long timeout = TimeUnit.MILLISECONDS.toNanos(DatabaseDescriptor.getTimeout(MessagingService.Verb.HINT)) - (System.nanoTime() - start);
+            boolean timedOut;
+
+            try
+            {
+                timedOut = !condition.await(timeout, TimeUnit.NANOSECONDS);
+            }
+            catch (InterruptedException e)
+            {
+                throw new AssertionError(e);
+            }
+
+            return timedOut ? Outcome.TIMEOUT : outcome;
+        }
+
+        public void onFailure(InetAddress from)
+        {
+            outcome = Outcome.FAILURE;
+            condition.signalAll();
+        }
+
+        public void response(MessageIn msg)
+        {
+            outcome = Outcome.SUCCESS;
+            condition.signalAll();
+        }
+
+        public boolean isLatencyForSnitch()
+        {
+            return false;
+        }
+    }
+}

src/java/org/apache/cassandra/hints/HintsReader.java

@@ -0,0 +1,312 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.hints;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Iterator;
+
+import javax.annotation.Nullable;
+
+import com.google.common.util.concurrent.RateLimiter;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.db.UnknownColumnFamilyException;
+import org.apache.cassandra.io.FSReadError;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.io.util.RandomAccessReader;
+import org.apache.cassandra.utils.AbstractIterator;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.CLibrary;
+
+/**
+ * A paged non-compressed hints reader that provides two iterators:
+ * - a 'raw' ByteBuffer iterator that doesn't deserialize the hints, but returns the pre-encoded hints verbatim
+ * - a decoded iterator, that deserializes the underlying bytes into {@link Hint} instances.
+ *
+ * The former is an optimisation for when the messaging version of the file matches the messaging version of the destination
+ * node. Extra decoding and reencoding is a waste of effort in this scenario, so we avoid it.
+ *
+ * The latter is required for dispatch of hints to nodes that have a different messaging version, and in general is just an
+ * easy way to enable backward and future compatibilty.
+ */
+final class HintsReader implements AutoCloseable, Iterable<HintsReader.Page>
+{
+    private static final Logger logger = LoggerFactory.getLogger(HintsReader.class);
+
+    // don't read more than 512 KB of hints at a time.
+    private static final int PAGE_SIZE = 512 << 10;
+
+    private final HintsDescriptor descriptor;
+    private final File file;
+    private final RandomAccessReader reader;
+    private final ChecksummedDataInput crcInput;
+
+    // we pass the RateLimiter into HintsReader itself because it's cheaper to calculate the size before the hint is deserialized
+    @Nullable
+    private final RateLimiter rateLimiter;
+
+    private HintsReader(HintsDescriptor descriptor, File file, RandomAccessReader reader, RateLimiter rateLimiter)
+    {
+        this.descriptor = descriptor;
+        this.file = file;
+        this.reader = reader;
+        this.crcInput = ChecksummedDataInput.wrap(reader);
+        this.rateLimiter = rateLimiter;
+    }
+
+    static HintsReader open(File file, RateLimiter rateLimiter)
+    {
+        RandomAccessReader reader = RandomAccessReader.open(file);
+        try
+        {
+            HintsDescriptor descriptor = HintsDescriptor.deserialize(reader);
+            return new HintsReader(descriptor, file, reader, rateLimiter);
+        }
+        catch (IOException e)
+        {
+            reader.close();
+            throw new FSReadError(e, file);
+        }
+    }
+
+    static HintsReader open(File file)
+    {
+        return open(file, null);
+    }
+
+    public void close()
+    {
+        FileUtils.closeQuietly(reader);
+    }
+
+    public HintsDescriptor descriptor()
+    {
+        return descriptor;
+    }
+
+    void seek(long newPosition)
+    {
+        reader.seek(newPosition);
+    }
+
+    public Iterator<Page> iterator()
+    {
+        return new PagesIterator();
+    }
+
+    final class Page
+    {
+        public final long offset;
+
+        private Page(long offset)
+        {
+            this.offset = offset;
+        }
+
+        Iterator<Hint> hintsIterator()
+        {
+            return new HintsIterator(offset);
+        }
+
+        Iterator<ByteBuffer> buffersIterator()
+        {
+            return new BuffersIterator(offset);
+        }
+    }
+
+    final class PagesIterator extends AbstractIterator<Page>
+    {
+        @SuppressWarnings("resource")
+        protected Page computeNext()
+        {
+            CLibrary.trySkipCache(reader.getChannel().getFileDescriptor(), 0, reader.getFilePointer(), reader.getPath());
+
+            if (reader.length() == reader.getFilePointer())
+                return endOfData();
+
+            return new Page(reader.getFilePointer());
+        }
+    }
+
+    /**
+     * A decoding iterator that deserializes the hints as it goes.
+     */
+    final class HintsIterator extends AbstractIterator<Hint>
+    {
+        private final long offset;
+
+        HintsIterator(long offset)
+        {
+            super();
+            this.offset = offset;
+        }
+
+        protected Hint computeNext()
+        {
+            Hint hint;
+
+            do
+            {
+                long position = reader.getFilePointer();
+
+                if (reader.length() == position)
+                    return endOfData(); // reached EOF
+
+                if (position - offset >= PAGE_SIZE)
+                    return endOfData(); // read page size or more bytes
+
+                try
+                {
+                    hint = computeNextInternal();
+                }
+                catch (IOException e)
+                {
+                    throw new FSReadError(e, file);
+                }
+            }
+            while (hint == null);
+
+            return hint;
+        }
+
+        private Hint computeNextInternal() throws IOException
+        {
+            crcInput.resetCrc();
+            crcInput.resetLimit();
+
+            int size = crcInput.readInt();
+
+            // if we cannot corroborate the size via crc, then we cannot safely skip this hint
+            if (reader.readInt() != crcInput.getCrc())
+                throw new IOException("Digest mismatch exception");
+
+            return readHint(size);
+        }
+
+        private Hint readHint(int size) throws IOException
+        {
+            if (rateLimiter != null)
+                rateLimiter.acquire(size);
+            crcInput.limit(size);
+
+            Hint hint;
+            try
+            {
+                hint = Hint.serializer.deserialize(crcInput, descriptor.messagingVersion());
+            }
+            catch (UnknownColumnFamilyException e)
+            {
+                logger.warn("Failed to read a hint for {} - table with id {} is unknown in file {}",
+                            descriptor.hostId,
+                            e.cfId,
+                            descriptor.fileName());
+                reader.skipBytes(crcInput.bytesRemaining());
+
+                return null;
+            }
+
+            if (reader.readInt() == crcInput.getCrc())
+                return hint;
+
+            // log a warning and skip the corrupted entry
+            logger.warn("Failed to read a hint for {} - digest mismatch for hint at position {} in file {}",
+                        descriptor.hostId,
+                        crcInput.getPosition() - size - 4,
+                        descriptor.fileName());
+            return null;
+        }
+    }
+
+    /**
+     * A verbatim iterator that simply returns the underlying ByteBuffers.
+     */
+    final class BuffersIterator extends AbstractIterator<ByteBuffer>
+    {
+        private final long offset;
+
+        BuffersIterator(long offset)
+        {
+            super();
+            this.offset = offset;
+        }
+
+        protected ByteBuffer computeNext()
+        {
+            ByteBuffer buffer;
+
+            do
+            {
+                long position = reader.getFilePointer();
+
+                if (reader.length() == position)
+                    return endOfData(); // reached EOF
+
+                if (position - offset >= PAGE_SIZE)
+                    return endOfData(); // read page size or more bytes
+
+                try
+                {
+                    buffer = computeNextInternal();
+                }
+                catch (IOException e)
+                {
+                    throw new FSReadError(e, file);
+                }
+            }
+            while (buffer == null);
+
+            return buffer;
+        }
+
+        private ByteBuffer computeNextInternal() throws IOException
+        {
+            crcInput.resetCrc();
+            crcInput.resetLimit();
+
+            int size = crcInput.readInt();
+
+            // if we cannot corroborate the size via crc, then we cannot safely skip this hint
+            if (reader.readInt() != crcInput.getCrc())
+                throw new IOException("Digest mismatch exception");
+
+            return readBuffer(size);
+        }
+
+        private ByteBuffer readBuffer(int size) throws IOException
+        {
+            if (rateLimiter != null)
+                rateLimiter.acquire(size);
+            crcInput.limit(size);
+
+            ByteBuffer buffer = ByteBufferUtil.read(crcInput, size);
+            if (reader.readInt() == crcInput.getCrc())
+                return buffer;
+
+            // log a warning and skip the corrupted entry
+            logger.warn("Failed to read a hint for {} - digest mismatch for hint at position {} in file {}",
+                        descriptor.hostId,
+                        crcInput.getPosition() - size - 4,
+                        descriptor.fileName());
+            return null;
+        }
+    }
+}

src/java/org/apache/cassandra/hints/HintsService.java

@@ -0,0 +1,291 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.hints;
+
+import java.io.File;
+import java.lang.management.ManagementFactory;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.util.Collections;
+import java.util.UUID;
+import java.util.concurrent.*;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import javax.management.MBeanServer;
+import javax.management.ObjectName;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.concurrent.ScheduledExecutors;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.metrics.HintedHandoffMetrics;
+import org.apache.cassandra.metrics.StorageMetrics;
+import org.apache.cassandra.service.StorageService;
+
+import static com.google.common.collect.Iterables.transform;
+import static com.google.common.collect.Iterables.size;
+
+/**
+ * A singleton-ish wrapper over various hints components:
+ * - a catalog of all hints stores
+ * - a single-threaded write executor
+ * - a multi-threaded dispatch executor
+ * - the buffer pool for writing hints into
+ *
+ * The front-end for everything hints related.
+ */
+public final class HintsService implements HintsServiceMBean
+{
+    private static final Logger logger = LoggerFactory.getLogger(HintsService.class);
+
+    public static final HintsService instance = new HintsService();
+
+    private static final String MBEAN_NAME = "org.apache.cassandra.hints:type=HintsService";
+
+    private static final int MIN_BUFFER_SIZE = 32 << 20;
+
+    private final HintsCatalog catalog;
+    private final HintsWriteExecutor writeExecutor;
+    private final HintsBufferPool bufferPool;
+    private final HintsDispatchExecutor dispatchExecutor;
+    private final AtomicBoolean isDispatchPaused;
+
+    private volatile boolean isShutDown = false;
+
+    private final ScheduledFuture triggerFlushingFuture;
+    private volatile ScheduledFuture triggerDispatchFuture;
+
+    public final HintedHandoffMetrics metrics;
+
+    private HintsService()
+    {
+        File hintsDirectory = DatabaseDescriptor.getHintsDirectory();
+        int maxDeliveryThreads = DatabaseDescriptor.getMaxHintsDeliveryThreads();
+
+        catalog = HintsCatalog.load(hintsDirectory);
+        writeExecutor = new HintsWriteExecutor(catalog);
+
+        int bufferSize = Math.max(DatabaseDescriptor.getMaxMutationSize() * 2, MIN_BUFFER_SIZE);
+        bufferPool = new HintsBufferPool(bufferSize, writeExecutor::flushBuffer);
+
+        isDispatchPaused = new AtomicBoolean(true);
+        dispatchExecutor = new HintsDispatchExecutor(hintsDirectory, maxDeliveryThreads, isDispatchPaused);
+
+        // periodically empty the current content of the buffers
+        int flushPeriod = DatabaseDescriptor.getHintsFlushPeriodInMS();
+        triggerFlushingFuture = ScheduledExecutors.optionalTasks.scheduleWithFixedDelay(() -> writeExecutor.flushBufferPool(bufferPool),
+                                                                                        flushPeriod,
+                                                                                        flushPeriod,
+                                                                                        TimeUnit.MILLISECONDS);
+        metrics = new HintedHandoffMetrics();
+    }
+
+    public void registerMBean()
+    {
+        MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
+        try
+        {
+            mbs.registerMBean(this, new ObjectName(MBEAN_NAME));
+        }
+        catch (Exception e)
+        {
+            throw new RuntimeException(e);
+        }
+    }
+
+    /**
+     * Write a hint for a iterable of nodes.
+     *
+     * @param hostIds host ids of the hint's target nodes
+     * @param hint the hint to store
+     */
+    public void write(Iterable<UUID> hostIds, Hint hint)
+    {
+        if (isShutDown)
+            throw new IllegalStateException("HintsService is shut down and can't accept new hints");
+
+        // we have to make sure that the HintsStore instances get properly initialized - otherwise dispatch will not trigger
+        catalog.maybeLoadStores(hostIds);
+
+        if (hint.isLive())
+            bufferPool.write(hostIds, hint);
+
+        StorageMetrics.totalHints.inc(size(hostIds));
+    }
+
+    /**
+     * Write a hint for a single node.
+     *
+     * @param hostId host id of the hint's target node
+     * @param hint the hint to store
+     */
+    public void write(UUID hostId, Hint hint)
+    {
+        write(Collections.singleton(hostId), hint);
+    }
+
+    /**
+     * Flush the buffer pool for the selected target nodes, then fsync their writers.
+     *
+     * @param hostIds host ids of the nodes to flush and fsync hints for
+     */
+    public void flushAndFsyncBlockingly(Iterable<UUID> hostIds)
+    {
+        Iterable<HintsStore> stores = transform(hostIds, catalog::get);
+        writeExecutor.flushBufferPool(bufferPool, stores);
+        writeExecutor.fsyncWritersBlockingly(stores);
+    }
+
+    public synchronized void startDispatch()
+    {
+        if (isShutDown)
+            throw new IllegalStateException("HintsService is shut down and cannot be restarted");
+
+        isDispatchPaused.set(false);
+
+        HintsDispatchTrigger trigger = new HintsDispatchTrigger(catalog, writeExecutor, dispatchExecutor, isDispatchPaused);
+        // triggering hint dispatch is now very cheap, so we can do it more often - every 10 seconds vs. every 10 minutes,
+        // previously; this reduces mean time to delivery, and positively affects batchlog delivery latencies, too
+        triggerDispatchFuture = ScheduledExecutors.scheduledTasks.scheduleWithFixedDelay(trigger, 10, 10, TimeUnit.SECONDS);
+    }
+
+    public void pauseDispatch()
+    {
+        logger.info("Paused hints dispatch");
+        isDispatchPaused.set(true);
+    }
+
+    public void resumeDispatch()
+    {
+        logger.info("Resumed hints dispatch");
+        isDispatchPaused.set(false);
+    }
+
+    /**
+     * Gracefully and blockingly shut down the service.
+     *
+     * Will abort dispatch sessions that are currently in progress (which is okay, it's idempotent),
+     * and make sure the buffers are flushed, hints files written and fsynced.
+     */
+    public synchronized void shutdownBlocking()
+    {
+        if (isShutDown)
+            throw new IllegalStateException("HintsService has already been shut down");
+        isShutDown = true;
+
+        if (triggerDispatchFuture != null)
+            triggerDispatchFuture.cancel(false);
+        pauseDispatch();
+
+        triggerFlushingFuture.cancel(false);
+
+        writeExecutor.flushBufferPool(bufferPool);
+        writeExecutor.closeAllWriters();
+
+        dispatchExecutor.shutdownBlocking();
+        writeExecutor.shutdownBlocking();
+    }
+
+    public void decommission()
+    {
+        resumeDispatch();
+    }
+
+    /**
+     * Deletes all hints for all destinations. Doesn't make snapshots - should be used with care.
+     */
+    public void deleteAllHints()
+    {
+        catalog.deleteAllHints();
+    }
+
+    /**
+     * Deletes all hints for the provided destination. Doesn't make snapshots - should be used with care.
+     *
+     * @param address inet address of the target node - encoded as a string for easier JMX consumption
+     */
+    public void deleteAllHintsForEndpoint(String address)
+    {
+        InetAddress target;
+        try
+        {
+            target = InetAddress.getByName(address);
+        }
+        catch (UnknownHostException e)
+        {
+            throw new IllegalArgumentException(e);
+        }
+        deleteAllHintsForEndpoint(target);
+    }
+
+    /**
+     * Deletes all hints for the provided destination. Doesn't make snapshots - should be used with care.
+     *
+     * @param target inet address of the target node
+     */
+    public void deleteAllHintsForEndpoint(InetAddress target)
+    {
+        UUID hostId = StorageService.instance.getHostIdForEndpoint(target);
+        if (hostId == null)
+            throw new IllegalArgumentException("Can't delete hints for unknown address " + target);
+        catalog.deleteAllHints(hostId);
+    }
+
+    /**
+     * Cleans up hints-related state after a node with id = hostId left.
+     *
+     * Dispatcher should stop itself (isHostAlive() will start returning false for the leaving host), but we'll wait for
+     * completion anyway.
+     *
+     * We should also flush the buffer is there are any thints for the node there, and close the writer (if any),
+     * so that we don't leave any hint files lying around.
+     *
+     * Once that is done, we can simply delete all hint files and remove the host id from the catalog.
+     *
+     * The worst that can happen if we don't get everything right is a hints file (or two) remaining undeleted.
+     *
+     * @param hostId id of the node being excised
+     */
+    public void excise(UUID hostId)
+    {
+        HintsStore store = catalog.get(hostId);
+        if (store == null)
+            return;
+
+        // flush the buffer and then close the writer for the excised host id, to make sure that no new files will appear
+        // for this host id after we are done
+        Future flushFuture = writeExecutor.flushBufferPool(bufferPool, Collections.singleton(store));
+        Future closeFuture = writeExecutor.closeWriter(store);
+        try
+        {
+            flushFuture.get();
+            closeFuture.get();
+        }
+        catch (InterruptedException | ExecutionException e)
+        {
+            throw new RuntimeException(e);
+        }
+
+        // wait for the current dispatch session to end (if any), so that the currently dispatched file gets removed
+        dispatchExecutor.completeDispatchBlockingly(store);
+
+        // delete all the hints files and remove the HintsStore instance from the map in the catalog
+        catalog.exciseStore(hostId);
+    }
+}

src/java/org/apache/cassandra/hints/HintsServiceMBean.java

@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.hints;
+
+public interface HintsServiceMBean
+{
+    /**
+     * Pause dispatch of all hints. Does not affect the creation of hints.
+     */
+    void pauseDispatch();
+
+    /**
+     * Resume dispatch of all hints. Does not affect the creation of hints.
+     */
+    void resumeDispatch();
+
+    /**
+     * Irrevocably deletes all the stored hints files (with the exception of those that are being dispatched right now,
+     * or being written to).
+     */
+    void deleteAllHints();
+
+    /**
+     * Irrevocably deletes all the stored hints files for the target address (with the exception of those that are
+     * being dispatched right now, or being written to).
+     */
+    void deleteAllHintsForEndpoint(String address);
+}

src/java/org/apache/cassandra/hints/HintsStore.java

@@ -0,0 +1,210 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.hints;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.InetAddress;
+import java.util.*;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentLinkedDeque;
+import java.util.concurrent.ConcurrentLinkedQueue;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.gms.FailureDetector;
+import org.apache.cassandra.io.FSWriteError;
+import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.utils.SyncUtil;
+
+/**
+ * Encapsulates the state of a peer's hints: the queue of hints files for dispatch, and the current writer (if any).
+ *
+ * The queue for dispatch is multi-threading safe.
+ *
+ * The writer MUST only be accessed by {@link HintsWriteExecutor}.
+ */
+final class HintsStore
+{
+    private static final Logger logger = LoggerFactory.getLogger(HintsStore.class);
+
+    public final UUID hostId;
+    private final File hintsDirectory;
+
+    private final Map<HintsDescriptor, Long> dispatchOffsets;
+    private final Deque<HintsDescriptor> dispatchDequeue;
+    private final Queue<HintsDescriptor> blacklistedFiles;
+
+    // last timestamp used in a descriptor; make sure to not reuse the same timestamp for new descriptors.
+    private volatile long lastUsedTimestamp;
+    private volatile HintsWriter hintsWriter;
+
+    private HintsStore(UUID hostId, File hintsDirectory, List<HintsDescriptor> descriptors)
+    {
+        this.hostId = hostId;
+        this.hintsDirectory = hintsDirectory;
+
+        dispatchOffsets = new ConcurrentHashMap<>();
+        dispatchDequeue = new ConcurrentLinkedDeque<>(descriptors);
+        blacklistedFiles = new ConcurrentLinkedQueue<>();
+
+        //noinspection resource
+        lastUsedTimestamp = descriptors.stream().mapToLong(d -> d.timestamp).max().orElse(0L);
+    }
+
+    static HintsStore create(UUID hostId, File hintsDirectory, List<HintsDescriptor> descriptors)
+    {
+        descriptors.sort((d1, d2) -> Long.compare(d1.timestamp, d2.timestamp));
+        return new HintsStore(hostId, hintsDirectory, descriptors);
+    }
+
+    InetAddress address()
+    {
+        return StorageService.instance.getEndpointForHostId(hostId);
+    }
+
+    boolean isLive()
+    {
+        InetAddress address = address();
+        return address != null && FailureDetector.instance.isAlive(address);
+    }
+
+    HintsDescriptor poll()
+    {
+        return dispatchDequeue.poll();
+    }
+
+    void offerFirst(HintsDescriptor descriptor)
+    {
+        dispatchDequeue.offerFirst(descriptor);
+    }
+
+    void offerLast(HintsDescriptor descriptor)
+    {
+        dispatchDequeue.offerLast(descriptor);
+    }
+
+    void deleteAllHints()
+    {
+        HintsDescriptor descriptor;
+        while ((descriptor = poll()) != null)
+        {
+            cleanUp(descriptor);
+            delete(descriptor);
+        }
+
+        while ((descriptor = blacklistedFiles.poll()) != null)
+        {
+            cleanUp(descriptor);
+            delete(descriptor);
+        }
+    }
+
+    private void delete(HintsDescriptor descriptor)
+    {
+        File hintsFile = new File(hintsDirectory, descriptor.fileName());
+        if (hintsFile.delete())
+            logger.info("Deleted hint file {}", descriptor.fileName());
+        else
+            logger.error("Failed to delete hint file {}", descriptor.fileName());
+
+        //noinspection ResultOfMethodCallIgnored
+        new File(hintsDirectory, descriptor.checksumFileName()).delete();
+    }
+
+    boolean hasFiles()
+    {
+        return !dispatchDequeue.isEmpty();
+    }
+
+    Optional<Long> getDispatchOffset(HintsDescriptor descriptor)
+    {
+        return Optional.ofNullable(dispatchOffsets.get(descriptor));
+    }
+
+    void markDispatchOffset(HintsDescriptor descriptor, long mark)
+    {
+        dispatchOffsets.put(descriptor, mark);
+    }
+
+    void cleanUp(HintsDescriptor descriptor)
+    {
+        dispatchOffsets.remove(descriptor);
+    }
+
+    void blacklist(HintsDescriptor descriptor)
+    {
+        blacklistedFiles.add(descriptor);
+    }
+
+    /*
+     * Methods dealing with HintsWriter.
+     *
+     * All of these, with the exception of isWriting(), are for exclusively single-threaded use by HintsWriteExecutor.
+     */
+
+    boolean isWriting()
+    {
+        return hintsWriter != null;
+    }
+
+    HintsWriter getOrOpenWriter()
+    {
+        if (hintsWriter == null)
+            hintsWriter = openWriter();
+        return hintsWriter;
+    }
+
+    HintsWriter getWriter()
+    {
+        return hintsWriter;
+    }
+
+    private HintsWriter openWriter()
+    {
+        lastUsedTimestamp = Math.max(System.currentTimeMillis(), lastUsedTimestamp + 1);
+        HintsDescriptor descriptor = new HintsDescriptor(hostId, lastUsedTimestamp);
+
+        try
+        {
+            return HintsWriter.create(hintsDirectory, descriptor);
+        }
+        catch (IOException e)
+        {
+            throw new FSWriteError(e, descriptor.fileName());
+        }
+    }
+
+    void closeWriter()
+    {
+        if (hintsWriter != null)
+        {
+            hintsWriter.close();
+            offerLast(hintsWriter.descriptor());
+            hintsWriter = null;
+            SyncUtil.trySyncDir(hintsDirectory);
+        }
+    }
+
+    void fsyncWriter()
+    {
+        if (hintsWriter != null)
+            hintsWriter.fsync();
+    }
+}

src/java/org/apache/cassandra/hints/HintsWriteExecutor.java

@@ -0,0 +1,235 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.hints;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Iterator;
+import java.util.concurrent.*;
+
+import org.apache.cassandra.concurrent.DebuggableThreadPoolExecutor;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.io.FSWriteError;
+
+/**
+ * A single threaded executor that exclusively writes all the hints and otherwise manipulate the writers.
+ *
+ * Flushing demultiplexes the provided {@link HintsBuffer} and sequentially writes to each {@link HintsWriter},
+ * using the same shared write buffer. In the near future, when CASSANDRA-9428 (compression) is implemented,
+ * will also share a compression buffer.
+ */
+final class HintsWriteExecutor
+{
+    private static final int WRITE_BUFFER_SIZE = 256 << 10;
+
+    private final HintsCatalog catalog;
+    private final ByteBuffer writeBuffer;
+    private final ExecutorService executor;
+
+    HintsWriteExecutor(HintsCatalog catalog)
+    {
+        this.catalog = catalog;
+
+        writeBuffer = ByteBuffer.allocateDirect(WRITE_BUFFER_SIZE);
+        executor = DebuggableThreadPoolExecutor.createWithFixedPoolSize("HintsWriteExecutor", 1);
+    }
+
+    /*
+     * Should be very fast (worst case scenario - write a few 10s of megabytes to disk).
+     */
+    void shutdownBlocking()
+    {
+        executor.shutdown();
+        try
+        {
+            executor.awaitTermination(1, TimeUnit.MINUTES);
+        }
+        catch (InterruptedException e)
+        {
+            throw new AssertionError(e);
+        }
+    }
+
+    /**
+     * Flush the provided buffer, recycle it and offer it back to the pool.
+     */
+    Future flushBuffer(HintsBuffer buffer, HintsBufferPool bufferPool)
+    {
+        return executor.submit(new FlushBufferTask(buffer, bufferPool));
+    }
+
+    /**
+     * Flush the current buffer, but without clearing/recycling it.
+     */
+    Future flushBufferPool(HintsBufferPool bufferPool)
+    {
+        return executor.submit(new FlushBufferPoolTask(bufferPool));
+    }
+
+    /**
+     * Flush the current buffer just for the specified hints stores. Without clearing/recycling it.
+     */
+    Future flushBufferPool(HintsBufferPool bufferPool, Iterable<HintsStore> stores)
+    {
+        return executor.submit(new PartiallyFlushBufferPoolTask(bufferPool, stores));
+    }
+
+    void fsyncWritersBlockingly(Iterable<HintsStore> stores)
+    {
+        try
+        {
+            executor.submit(new FsyncWritersTask(stores)).get();
+        }
+        catch (InterruptedException | ExecutionException e)
+        {
+            throw new RuntimeException(e);
+        }
+    }
+
+    Future closeWriter(HintsStore store)
+    {
+        return executor.submit(store::closeWriter);
+    }
+
+    Future closeAllWriters()
+    {
+        return executor.submit(() -> catalog.stores().forEach(HintsStore::closeWriter));
+    }
+
+    private final class FlushBufferTask implements Runnable
+    {
+        private final HintsBuffer buffer;
+        private final HintsBufferPool bufferPool;
+
+        FlushBufferTask(HintsBuffer buffer, HintsBufferPool bufferPool)
+        {
+            this.buffer = buffer;
+            this.bufferPool = bufferPool;
+        }
+
+        public void run()
+        {
+            buffer.waitForModifications();
+
+            try
+            {
+                flush(buffer);
+            }
+            finally
+            {
+                HintsBuffer recycledBuffer = buffer.recycle();
+                if (!bufferPool.offer(recycledBuffer))
+                    recycledBuffer.free();
+            }
+        }
+    }
+
+    private final class FlushBufferPoolTask implements Runnable
+    {
+        private final HintsBufferPool bufferPool;
+
+        FlushBufferPoolTask(HintsBufferPool bufferPool)
+        {
+            this.bufferPool = bufferPool;
+        }
+
+        public void run()
+        {
+            HintsBuffer buffer = bufferPool.currentBuffer();
+            buffer.waitForModifications();
+            flush(buffer);
+        }
+    }
+
+    private final class PartiallyFlushBufferPoolTask implements Runnable
+    {
+        private final HintsBufferPool bufferPool;
+        private final Iterable<HintsStore> stores;
+
+        PartiallyFlushBufferPoolTask(HintsBufferPool bufferPool, Iterable<HintsStore> stores)
+        {
+            this.bufferPool = bufferPool;
+            this.stores = stores;
+        }
+
+        public void run()
+        {
+            HintsBuffer buffer = bufferPool.currentBuffer();
+            buffer.waitForModifications();
+            stores.forEach(store -> flush(buffer.consumingHintsIterator(store.hostId), store));
+        }
+    }
+
+    private final class FsyncWritersTask implements Runnable
+    {
+        private final Iterable<HintsStore> stores;
+
+        FsyncWritersTask(Iterable<HintsStore> stores)
+        {
+            this.stores = stores;
+        }
+
+        public void run()
+        {
+            stores.forEach(HintsStore::fsyncWriter);
+            catalog.fsyncDirectory();
+        }
+    }
+
+    private void flush(HintsBuffer buffer)
+    {
+        buffer.hostIds().forEach(hostId -> flush(buffer.consumingHintsIterator(hostId), catalog.get(hostId)));
+    }
+
+    private void flush(Iterator<ByteBuffer> iterator, HintsStore store)
+    {
+        while (true)
+        {
+            flushInternal(iterator, store);
+
+            if (!iterator.hasNext())
+                break;
+
+            // exceeded the size limit for an individual file, but still have more to write
+            // close the current writer and continue flushing to a new one in the next iteration
+            store.closeWriter();
+        }
+    }
+
+    private void flushInternal(Iterator<ByteBuffer> iterator, HintsStore store)
+    {
+        long maxHintsFileSize = DatabaseDescriptor.getMaxHintsFileSize();
+
+        @SuppressWarnings("resource")
+        HintsWriter writer = store.getOrOpenWriter();
+
+        try (HintsWriter.Session session = writer.newSession(writeBuffer))
+        {
+            while (iterator.hasNext())
+            {
+                session.append(iterator.next());
+                if (session.position() >= maxHintsFileSize)
+                    break;
+            }
+        }
+        catch (IOException e)
+        {
+            throw new FSWriteError(e, writer.descriptor().fileName());
+        }
+    }
+}

src/java/org/apache/cassandra/hints/HintsWriter.java

@@ -0,0 +1,272 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.hints;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+import java.nio.channels.FileChannel;
+import java.nio.file.Files;
+import java.nio.file.StandardOpenOption;
+import java.util.zip.CRC32;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.io.FSWriteError;
+import org.apache.cassandra.io.util.DataOutputBuffer;
+import org.apache.cassandra.io.util.DataOutputBufferFixed;
+import org.apache.cassandra.utils.CLibrary;
+import org.apache.cassandra.utils.SyncUtil;
+import org.apache.cassandra.utils.Throwables;
+
+import static org.apache.cassandra.utils.FBUtilities.updateChecksum;
+import static org.apache.cassandra.utils.FBUtilities.updateChecksumInt;
+import static org.apache.cassandra.utils.Throwables.perform;
+
+final class HintsWriter implements AutoCloseable
+{
+    static final int PAGE_SIZE = 4096;
+
+    private final File directory;
+    private final HintsDescriptor descriptor;
+    private final File file;
+    private final FileChannel channel;
+    private final int fd;
+    private final CRC32 globalCRC;
+
+    private volatile long lastSyncPosition = 0L;
+
+    private HintsWriter(File directory, HintsDescriptor descriptor, File file, FileChannel channel, int fd, CRC32 globalCRC)
+    {
+        this.directory = directory;
+        this.descriptor = descriptor;
+        this.file = file;
+        this.channel = channel;
+        this.fd = fd;
+        this.globalCRC = globalCRC;
+    }
+
+    static HintsWriter create(File directory, HintsDescriptor descriptor) throws IOException
+    {
+        File file = new File(directory, descriptor.fileName());
+
+        FileChannel channel = FileChannel.open(file.toPath(), StandardOpenOption.WRITE, StandardOpenOption.CREATE_NEW);
+        int fd = CLibrary.getfd(channel);
+
+        CRC32 crc = new CRC32();
+
+        try
+        {
+            // write the descriptor
+            DataOutputBuffer dob = new DataOutputBuffer();
+            descriptor.serialize(dob);
+            ByteBuffer descriptorBytes = dob.buffer();
+            updateChecksum(crc, descriptorBytes);
+            channel.write(descriptorBytes);
+        }
+        catch (Throwable e)
+        {
+            channel.close();
+            throw e;
+        }
+
+        return new HintsWriter(directory, descriptor, file, channel, fd, crc);
+    }
+
+    HintsDescriptor descriptor()
+    {
+        return descriptor;
+    }
+
+    private void writeChecksum()
+    {
+        File checksumFile = new File(directory, descriptor.checksumFileName());
+        try (OutputStream out = Files.newOutputStream(checksumFile.toPath()))
+        {
+            out.write(Integer.toHexString((int) globalCRC.getValue()).getBytes());
+        }
+        catch (IOException e)
+        {
+            throw new FSWriteError(e, checksumFile);
+        }
+    }
+
+    public void close()
+    {
+        perform(file, Throwables.FileOpType.WRITE, this::doFsync, channel::close);
+
+        writeChecksum();
+    }
+
+    public void fsync()
+    {
+        perform(file, Throwables.FileOpType.WRITE, this::doFsync);
+    }
+
+    private void doFsync() throws IOException
+    {
+        SyncUtil.force(channel, true);
+        lastSyncPosition = channel.position();
+    }
+
+    Session newSession(ByteBuffer buffer)
+    {
+        try
+        {
+            return new Session(buffer, channel.size());
+        }
+        catch (IOException e)
+        {
+            throw new FSWriteError(e, file);
+        }
+    }
+
+    /**
+     * The primary goal of the Session class is to be able to share the same buffers among potentially dozens or hundreds
+     * of hints writers, and ensure that their contents are always written to the underlying channels in the end.
+     */
+    final class Session implements AutoCloseable
+    {
+        private final ByteBuffer buffer;
+
+        private final long initialSize;
+        private long bytesWritten;
+
+        Session(ByteBuffer buffer, long initialSize)
+        {
+            buffer.clear();
+            bytesWritten = 0L;
+
+            this.buffer = buffer;
+            this.initialSize = initialSize;
+        }
+
+        long position()
+        {
+            return initialSize + bytesWritten;
+        }
+
+        /**
+         * Appends the serialized hint (with CRC included) to this session's aggregation buffer,
+         * writes to the underlying channel when the buffer is overflown.
+         *
+         * @param hint the serialized hint (with CRC included)
+         * @throws IOException
+         */
+        void append(ByteBuffer hint) throws IOException
+        {
+            bytesWritten += hint.remaining();
+
+            // if the hint fits in the aggregation buffer, then just update the aggregation buffer,
+            // otherwise write both the aggregation buffer and the new buffer to the channel
+            if (hint.remaining() <= buffer.remaining())
+            {
+                buffer.put(hint);
+                return;
+            }
+
+            buffer.flip();
+
+            // update file-global CRC checksum
+            updateChecksum(globalCRC, buffer);
+            updateChecksum(globalCRC, hint);
+
+            channel.write(new ByteBuffer[] { buffer, hint });
+            buffer.clear();
+        }
+
+        /**
+         * Serializes and appends the hint (with CRC included) to this session's aggregation buffer,
+         * writes to the underlying channel when the buffer is overflown.
+         *
+         * Used mainly by tests and {@link LegacyHintsMigrator}
+         *
+         * @param hint the unserialized hint
+         * @throws IOException
+         */
+        void append(Hint hint) throws IOException
+        {
+            int hintSize = (int) Hint.serializer.serializedSize(hint, descriptor.messagingVersion());
+            int totalSize = hintSize + HintsBuffer.ENTRY_OVERHEAD_SIZE;
+
+            if (totalSize > buffer.remaining())
+                flushBuffer();
+
+            ByteBuffer hintBuffer = totalSize <= buffer.remaining()
+                                  ? buffer
+                                  : ByteBuffer.allocate(totalSize);
+
+            CRC32 crc = new CRC32();
+            try (DataOutputBufferFixed out = new DataOutputBufferFixed(hintBuffer))
+            {
+                out.writeInt(hintSize);
+                updateChecksumInt(crc, hintSize);
+                out.writeInt((int) crc.getValue());
+
+                Hint.serializer.serialize(hint, out, descriptor.messagingVersion());
+                updateChecksum(crc, hintBuffer, hintBuffer.position() - hintSize, hintSize);
+                out.writeInt((int) crc.getValue());
+            }
+
+            if (hintBuffer == buffer)
+                bytesWritten += totalSize;
+            else
+                append((ByteBuffer) hintBuffer.flip());
+        }
+
+        /**
+         * Closes the session - flushes the aggregation buffer (if not empty), does page aligning, and potentially fsyncs.
+         * @throws IOException
+         */
+        public void close() throws IOException
+        {
+            flushBuffer();
+            maybeFsync();
+            maybeSkipCache();
+        }
+
+        private void flushBuffer() throws IOException
+        {
+            buffer.flip();
+
+            if (buffer.remaining() > 0)
+            {
+                updateChecksum(globalCRC, buffer);
+                channel.write(buffer);
+            }
+
+            buffer.clear();
+        }
+
+        private void maybeFsync()
+        {
+            if (position() >= lastSyncPosition + DatabaseDescriptor.getTrickleFsyncIntervalInKb() * 1024)
+                fsync();
+        }
+
+        private void maybeSkipCache()
+        {
+            long position = position();
+
+            // don't skip page cache for tiny files, on the assumption that if they are tiny, the target node is probably
+            // alive, and if so, the file will be closed and dispatched shortly (within a minute), and the file will be dropped.
+            if (position >= DatabaseDescriptor.getTrickleFsyncIntervalInKb() * 1024)
+                CLibrary.trySkipCache(fd, 0, position - (position % PAGE_SIZE), file.getPath());
+        }
+    }
+}

src/java/org/apache/cassandra/hints/LegacyHintsMigrator.java

@@ -0,0 +1,243 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.hints;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.*;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.cql3.QueryProcessor;
+import org.apache.cassandra.cql3.UntypedResultSet;
+import org.apache.cassandra.db.*;
+import org.apache.cassandra.db.compaction.CompactionManager;
+import org.apache.cassandra.db.marshal.UUIDType;
+import org.apache.cassandra.db.partitions.PartitionUpdate;
+import org.apache.cassandra.io.FSWriteError;
+import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.util.DataInputBuffer;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.serializers.MarshalException;
+import org.apache.cassandra.utils.FBUtilities;
+
+/**
+ * A migrator that goes through the legacy system.hints table and writes all the hints to the new hints storage format.
+ */
+@SuppressWarnings("deprecation")
+public final class LegacyHintsMigrator
+{
+    private static final Logger logger = LoggerFactory.getLogger(LegacyHintsMigrator.class);
+
+    private final File hintsDirectory;
+    private final long maxHintsFileSize;
+
+    private final ColumnFamilyStore legacyHintsTable;
+    private final int pageSize;
+
+    public LegacyHintsMigrator(File hintsDirectory, long maxHintsFileSize)
+    {
+        this.hintsDirectory = hintsDirectory;
+        this.maxHintsFileSize = maxHintsFileSize;
+
+        legacyHintsTable = Keyspace.open(SystemKeyspace.NAME).getColumnFamilyStore(SystemKeyspace.LEGACY_HINTS);
+        pageSize = calculatePageSize(legacyHintsTable);
+    }
+
+    // read fewer columns (mutations) per page if they are very large
+    private static int calculatePageSize(ColumnFamilyStore legacyHintsTable)
+    {
+        int size = 128;
+
+        int meanCellCount = legacyHintsTable.getMeanColumns();
+        double meanPartitionSize = legacyHintsTable.getMeanPartitionSize();
+
+        if (meanCellCount != 0 || meanPartitionSize != 0)
+        {
+            int avgHintSize = (int) meanPartitionSize / meanCellCount;
+            size = Math.max(2, Math.min(size, (512 << 10) / avgHintSize));
+        }
+
+        return size;
+    }
+
+    public void migrate()
+    {
+        // nothing to migrate
+        if (legacyHintsTable.isEmpty())
+            return;
+        logger.info("Migrating legacy hints to new storage");
+
+        // major-compact all of the existing sstables to get rid of the tombstones + expired hints
+        logger.info("Forcing a major compaction of {}.{} table", SystemKeyspace.NAME, SystemKeyspace.LEGACY_HINTS);
+        compactLegacyHints();
+
+        // paginate over legacy hints and write them to the new storage
+        logger.info("Migrating legacy hints to the new storage");
+        migrateLegacyHints();
+
+        // truncate the legacy hints table
+        logger.info("Truncating {}.{} table", SystemKeyspace.NAME, SystemKeyspace.LEGACY_HINTS);
+        legacyHintsTable.truncateBlocking();
+    }
+
+    private void compactLegacyHints()
+    {
+        Collection<Descriptor> descriptors = new ArrayList<>();
+        legacyHintsTable.getTracker().getUncompacting().forEach(sstable -> descriptors.add(sstable.descriptor));
+        if (!descriptors.isEmpty())
+            forceCompaction(descriptors);
+    }
+
+    private void forceCompaction(Collection<Descriptor> descriptors)
+    {
+        try
+        {
+            CompactionManager.instance.submitUserDefined(legacyHintsTable, descriptors, FBUtilities.nowInSeconds()).get();
+        }
+        catch (InterruptedException | ExecutionException e)
+        {
+            throw new RuntimeException(e);
+        }
+    }
+
+    private void migrateLegacyHints()
+    {
+        ByteBuffer buffer = ByteBuffer.allocateDirect(256 * 1024);
+        String query = String.format("SELECT DISTINCT target_id FROM %s.%s", SystemKeyspace.NAME, SystemKeyspace.LEGACY_HINTS);
+        //noinspection ConstantConditions
+        QueryProcessor.executeInternal(query).forEach(row -> migrateLegacyHints(row.getUUID("target_id"), buffer));
+        FileUtils.clean(buffer);
+    }
+
+    private void migrateLegacyHints(UUID hostId, ByteBuffer buffer)
+    {
+        String query = String.format("SELECT target_id, hint_id, message_version, mutation, ttl(mutation) AS ttl, writeTime(mutation) AS write_time " +
+                                     "FROM %s.%s " +
+                                     "WHERE target_id = ?",
+                                     SystemKeyspace.NAME,
+                                     SystemKeyspace.LEGACY_HINTS);
+
+        // read all the old hints (paged iterator), write them in the new format
+        UntypedResultSet rows = QueryProcessor.executeInternalWithPaging(query, pageSize, hostId);
+        migrateLegacyHints(hostId, rows, buffer);
+
+        // delete the whole partition in the legacy table; we would truncate the whole table afterwards, but this allows
+        // to not lose progress in case of a terminated conversion
+        deleteLegacyHintsPartition(hostId);
+    }
+
+    private void migrateLegacyHints(UUID hostId, UntypedResultSet rows, ByteBuffer buffer)
+    {
+        migrateLegacyHints(hostId, rows.iterator(), buffer);
+    }
+
+    private void migrateLegacyHints(UUID hostId, Iterator<UntypedResultSet.Row> iterator, ByteBuffer buffer)
+    {
+        do
+        {
+            migrateLegacyHintsInternal(hostId, iterator, buffer);
+            // if there are hints that didn't fit in the previous file, keep calling the method to write to a new
+            // file until we get everything written.
+        }
+        while (iterator.hasNext());
+    }
+
+    private void migrateLegacyHintsInternal(UUID hostId, Iterator<UntypedResultSet.Row> iterator, ByteBuffer buffer)
+    {
+        HintsDescriptor descriptor = new HintsDescriptor(hostId, System.currentTimeMillis());
+
+        try (HintsWriter writer = HintsWriter.create(hintsDirectory, descriptor))
+        {
+            try (HintsWriter.Session session = writer.newSession(buffer))
+            {
+                while (iterator.hasNext())
+                {
+                    Hint hint = convertLegacyHint(iterator.next());
+                    if (hint != null)
+                        session.append(hint);
+
+                    if (session.position() >= maxHintsFileSize)
+                        break;
+                }
+            }
+        }
+        catch (IOException e)
+        {
+            throw new FSWriteError(e, descriptor.fileName());
+        }
+    }
+
+    private static Hint convertLegacyHint(UntypedResultSet.Row row)
+    {
+        Mutation mutation = deserializeLegacyMutation(row);
+        if (mutation == null)
+            return null;
+
+        long creationTime = row.getLong("write_time"); // milliseconds, not micros, for the hints table
+        int expirationTime = FBUtilities.nowInSeconds() + row.getInt("ttl");
+        int originalGCGS = expirationTime - (int) TimeUnit.MILLISECONDS.toSeconds(creationTime);
+
+        int gcgs = Math.min(originalGCGS, mutation.smallestGCGS());
+
+        return Hint.create(mutation, creationTime, gcgs);
+    }
+
+    private static Mutation deserializeLegacyMutation(UntypedResultSet.Row row)
+    {
+        try
+        {
+            Mutation mutation = Mutation.serializer.deserialize(new DataInputBuffer(row.getBlob("mutation"), true),
+                                                                row.getInt("message_version"));
+            mutation.getPartitionUpdates().forEach(PartitionUpdate::validate);
+            return mutation;
+        }
+        catch (IOException e)
+        {
+            logger.error("Failed to migrate a hint for {} from legacy {}.{} table: {}",
+                         row.getUUID("target_id"),
+                         SystemKeyspace.NAME,
+                         SystemKeyspace.LEGACY_HINTS,
+                         e);
+            return null;
+        }
+        catch (MarshalException e)
+        {
+            logger.warn("Failed to validate a hint for {} (table id {}) from legacy {}.{} table - skipping: {})",
+                        row.getUUID("target_id"),
+                        SystemKeyspace.NAME,
+                        SystemKeyspace.LEGACY_HINTS,
+                        e);
+            return null;
+        }
+    }
+
+    private static void deleteLegacyHintsPartition(UUID hostId)
+    {
+        // intentionally use millis, like the rest of the legacy implementation did, just in case
+        Mutation mutation = new Mutation(PartitionUpdate.fullPartitionDelete(SystemKeyspace.LegacyHints,
+                                                                             UUIDType.instance.decompose(hostId),
+                                                                             System.currentTimeMillis(),
+                                                                             FBUtilities.nowInSeconds()));
+        mutation.applyUnsafe();
+    }
+}

src/java/org/apache/cassandra/hints/package-info.java

@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * Hints subsystem consists of several components.
+ *
+ * {@link org.apache.cassandra.hints.Hint} encodes all the required metadata and the mutation being hinted.
+ *
+ * {@link org.apache.cassandra.hints.HintsBuffer} provides a temporary buffer for writing the hints to in a concurrent manner,
+ * before we flush them to disk.
+ *
+ * {@link org.apache.cassandra.hints.HintsBufferPool} is responsible for submitting {@link org.apache.cassandra.hints.HintsBuffer}
+ * instances for flushing when they exceed their capacity, and for maitaining a reserve {@link org.apache.cassandra.hints.HintsBuffer}
+ * instance, and creating extra ones if flushing cannot keep up with arrival rate.
+ *
+ * {@link org.apache.cassandra.hints.HintsWriteExecutor} is a single-threaded executor that performs all the writing to disk.
+ *
+ * {@link org.apache.cassandra.hints.HintsDispatchExecutor} is a multi-threaded executor responsible for dispatch of
+ * the hints to their destinations.
+ *
+ * {@link org.apache.cassandra.hints.HintsStore} tracks the state of all hints files (written and being written to)
+ * for a given host id destination.
+ *
+ * {@link org.apache.cassandra.hints.HintsCatalog} maintains the mapping of host ids to {@link org.apache.cassandra.hints.HintsStore}
+ * instances, and provides some aggregate APIs.
+ *
+ * {@link org.apache.cassandra.hints.HintsService} wraps the catalog, the pool, and the two executors, acting as a front-end
+ * for hints.
+ */
+package org.apache.cassandra.hints;

src/java/org/apache/cassandra/metrics/HintedHandoffMetrics.java

@@ -21,7 +21,6 @@ import java.net.InetAddress;
 import java.util.Map.Entry;
 
 import com.codahale.metrics.Counter;
-import org.apache.cassandra.db.HintedHandOffManager;
 import org.apache.cassandra.db.SystemKeyspace;
 import org.apache.cassandra.utils.UUIDGen;
 import org.slf4j.Logger;
@@ -34,7 +33,7 @@ import com.google.common.cache.LoadingCache;
 import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics;
 
 /**
- * Metrics for {@link HintedHandOffManager}.
+ * Metrics for {@link org.apache.cassandra.hints.HintsService}.
  */
 public class HintedHandoffMetrics
 {

src/java/org/apache/cassandra/metrics/HintsServiceMetrics.java

@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.metrics;
+
+/**
+ * Metrics for {@link HintsService}.
+ */
+public final class HintsServiceMetrics
+{
+}

src/java/org/apache/cassandra/net/MessagingService.java

@@ -55,6 +55,8 @@ import org.apache.cassandra.gms.EchoMessage;
 import org.apache.cassandra.gms.GossipDigestAck;
 import org.apache.cassandra.gms.GossipDigestAck2;
 import org.apache.cassandra.gms.GossipDigestSyn;
+import org.apache.cassandra.hints.HintMessage;
+import org.apache.cassandra.hints.HintResponse;
 import org.apache.cassandra.io.IVersionedSerializer;
 import org.apache.cassandra.io.util.DataInputPlus;
 import org.apache.cassandra.io.util.DataOutputPlus;
@@ -97,7 +99,7 @@ public final class MessagingService implements MessagingServiceMBean
     public enum Verb
     {
         MUTATION,
-        @Deprecated BINARY,
+        HINT,
         READ_REPAIR,
         READ,
         REQUEST_RESPONSE, // client-initiated reads and writes
@@ -129,7 +131,6 @@ public final class MessagingService implements MessagingServiceMBean
         _TRACE, // dummy verb so we can use MS.droppedMessagesMap
         ECHO,
         REPAIR_MESSAGE,
-        // use as padding for backwards compatability where a previous version needs to validate a verb from the future.
         PAXOS_PREPARE,
         PAXOS_PROPOSE,
         PAXOS_COMMIT,
@@ -150,6 +151,7 @@ public final class MessagingService implements MessagingServiceMBean
         put(Verb.COUNTER_MUTATION, Stage.COUNTER_MUTATION);
         put(Verb.BATCHLOG_MUTATION, Stage.BATCHLOG_MUTATION);
         put(Verb.READ_REPAIR, Stage.MUTATION);
+        put(Verb.HINT, Stage.MUTATION);
         put(Verb.TRUNCATE, Stage.MUTATION);
         put(Verb.PAXOS_PREPARE, Stage.MUTATION);
         put(Verb.PAXOS_PROPOSE, Stage.MUTATION);
@@ -226,6 +228,7 @@ public final class MessagingService implements MessagingServiceMBean
         put(Verb.PAXOS_PREPARE, Commit.serializer);
         put(Verb.PAXOS_PROPOSE, Commit.serializer);
         put(Verb.PAXOS_COMMIT, Commit.serializer);
+        put(Verb.HINT, HintMessage.serializer);
     }};
 
     /**
@@ -234,6 +237,7 @@ public final class MessagingService implements MessagingServiceMBean
     public static final EnumMap<Verb, IVersionedSerializer<?>> callbackDeserializers = new EnumMap<Verb, IVersionedSerializer<?>>(Verb.class)
     {{
         put(Verb.MUTATION, WriteResponse.serializer);
+        put(Verb.HINT, HintResponse.serializer);
         put(Verb.BATCHLOG_MUTATION, WriteResponse.serializer);
         put(Verb.READ_REPAIR, WriteResponse.serializer);
         put(Verb.COUNTER_MUTATION, WriteResponse.serializer);
@@ -299,6 +303,7 @@ public final class MessagingService implements MessagingServiceMBean
                                                                    Verb.MUTATION,
                                                                    Verb.BATCHLOG_MUTATION, //FIXME: should this be droppable??
                                                                    Verb.COUNTER_MUTATION,
+                                                                   Verb.HINT,
                                                                    Verb.READ_REPAIR,
                                                                    Verb.READ,
                                                                    Verb.RANGE_SLICE,

src/java/org/apache/cassandra/service/CassandraDaemon.java

@@ -56,6 +56,7 @@ import org.apache.cassandra.db.commitlog.CommitLog;
 import org.apache.cassandra.exceptions.ConfigurationException;
 import org.apache.cassandra.exceptions.StartupException;
 import org.apache.cassandra.gms.Gossiper;
+import org.apache.cassandra.hints.LegacyHintsMigrator;
 import org.apache.cassandra.io.FSError;
 import org.apache.cassandra.io.sstable.CorruptSSTableException;
 import org.apache.cassandra.io.util.FileUtils;
@@ -276,6 +277,9 @@ public class CassandraDaemon
             throw new RuntimeException(e);
         }
 
+        // migrate any legacy (pre-3.0) hints from system.hints table into the new store
+        new LegacyHintsMigrator(DatabaseDescriptor.getHintsDirectory(), DatabaseDescriptor.getMaxHintsFileSize()).migrate();
+
         // enable auto compaction
         for (Keyspace keyspace : Keyspace.all())
         {

src/java/org/apache/cassandra/service/StartupChecks.java

@@ -179,33 +179,30 @@ public class StartupChecks
         }
     };
 
-    public static final StartupCheck checkDataDirs = new StartupCheck()
+    public static final StartupCheck checkDataDirs = () ->
     {
-        public void execute() throws StartupException
+        // check all directories(data, commitlog, saved cache) for existence and permission
+        Iterable<String> dirs = Iterables.concat(Arrays.asList(DatabaseDescriptor.getAllDataFileLocations()),
+                                                 Arrays.asList(DatabaseDescriptor.getCommitLogLocation(),
+                                                               DatabaseDescriptor.getSavedCachesLocation(),
+                                                               DatabaseDescriptor.getHintsDirectory().getAbsolutePath()));
+        for (String dataDir : dirs)
         {
-            // check all directories(data, commitlog, saved cache) for existence and permission
-            Iterable<String> dirs = Iterables.concat(Arrays.asList(DatabaseDescriptor.getAllDataFileLocations()),
-                                                     Arrays.asList(DatabaseDescriptor.getCommitLogLocation(),
-                                                                   DatabaseDescriptor.getSavedCachesLocation()));
-            for (String dataDir : dirs)
+            logger.debug("Checking directory {}", dataDir);
+            File dir = new File(dataDir);
+
+            // check that directories exist.
+            if (!dir.exists())
             {
-                logger.debug("Checking directory {}", dataDir);
-                File dir = new File(dataDir);
-
-                // check that directories exist.
-                if (!dir.exists())
-                {
-                    logger.error("Directory {} doesn't exist", dataDir);
-                    // if they don't, failing their creation, stop cassandra.
-                    if (!dir.mkdirs())
-                        throw new StartupException(3, "Has no permission to create directory "+ dataDir);
-                }
-
-                // if directories exist verify their permissions
-                if (!Directories.verifyFullPermissions(dir, dataDir))
-                    throw new StartupException(3, "Insufficient permissions on directory " + dataDir);
-
+                logger.error("Directory {} doesn't exist", dataDir);
+                // if they don't, failing their creation, stop cassandra.
+                if (!dir.mkdirs())
+                    throw new StartupException(3, "Has no permission to create directory "+ dataDir);
             }
+
+            // if directories exist verify their permissions
+            if (!Directories.verifyFullPermissions(dir, dataDir))
+                throw new StartupException(3, "Insufficient permissions on directory " + dataDir);
         }
     };
 

src/java/org/apache/cassandra/service/StorageProxy.java

@@ -34,6 +34,7 @@ import com.google.common.util.concurrent.Uninterruptibles;
 
 import org.apache.cassandra.db.view.MaterializedViewManager;
 import org.apache.cassandra.db.view.MaterializedViewUtils;
+import org.apache.cassandra.db.HintedHandOffManager;
 import org.apache.cassandra.metrics.*;
 import org.apache.commons.lang3.StringUtils;
 import org.slf4j.Logger;
@@ -57,6 +58,8 @@ import org.apache.cassandra.dht.Token;
 import org.apache.cassandra.exceptions.*;
 import org.apache.cassandra.gms.FailureDetector;
 import org.apache.cassandra.gms.Gossiper;
+import org.apache.cassandra.hints.Hint;
+import org.apache.cassandra.hints.HintsService;
 import org.apache.cassandra.io.util.DataOutputBuffer;
 import org.apache.cassandra.locator.AbstractReplicationStrategy;
 import org.apache.cassandra.locator.IEndpointSnitch;
@@ -99,7 +102,9 @@ public class StorageProxy implements StorageProxyMBean
 
     private static final double CONCURRENT_SUBREQUESTS_MARGIN = 0.10;
 
-    private StorageProxy() {}
+    private StorageProxy()
+    {
+    }
 
     static
     {
@@ -113,6 +118,9 @@ public class StorageProxy implements StorageProxyMBean
             throw new RuntimeException(e);
         }
 
+        HintsService.instance.registerMBean();
+        HintedHandOffManager.instance.registerMBean();
+
         standardWritePerformer = new WritePerformer()
         {
             public void apply(IMutation mutation,
@@ -603,32 +611,41 @@ public class StorageProxy implements StorageProxyMBean
         }
     }
 
-    /** hint all the mutations (except counters, which can't be safely retried).  This means
-      * we'll re-hint any successful ones; doesn't seem worth it to track individual success
-      * just for this unusual case.
-
-      * @param mutations the mutations that require hints
-      */
+    /**
+     * Hint all the mutations (except counters, which can't be safely retried).  This means
+     * we'll re-hint any successful ones; doesn't seem worth it to track individual success
+     * just for this unusual case.
+     *
+     * Only used for CL.ANY
+     *
+     * @param mutations the mutations that require hints
+     */
     private static void hintMutations(Collection<? extends IMutation> mutations)
     {
         for (IMutation mutation : mutations)
-        {
-            if (mutation instanceof CounterMutation)
-                continue;
+            if (!(mutation instanceof CounterMutation))
+                hintMutation((Mutation) mutation);
 
-            Token tk = mutation.key().getToken();
-            List<InetAddress> naturalEndpoints = StorageService.instance.getNaturalEndpoints(mutation.getKeyspaceName(), tk);
-            Collection<InetAddress> pendingEndpoints = StorageService.instance.getTokenMetadata().pendingEndpointsFor(tk, mutation.getKeyspaceName());
-            for (InetAddress target : Iterables.concat(naturalEndpoints, pendingEndpoints))
-            {
-                // local writes can timeout, but cannot be dropped (see LocalMutationRunnable and
-                // CASSANDRA-6510), so there is no need to hint or retry
-                if (!target.equals(FBUtilities.getBroadcastAddress()) && shouldHint(target))
-                    submitHint((Mutation) mutation, target, null);
-            }
-        }
+        Tracing.trace("Wrote hints to satisfy CL.ANY after no replicas acknowledged the write");
+    }
 
-        Tracing.trace("Wrote hint to satisfy CL.ANY after no replicas acknowledged the write");
+    private static void hintMutation(Mutation mutation)
+    {
+        Token tk = DatabaseDescriptor.getPartitioner().getToken(mutation.key().getKey());
+        List<InetAddress> naturalEndpoints = StorageService.instance.getNaturalEndpoints(mutation.getKeyspaceName(), tk);
+        Collection<InetAddress> pendingEndpoints =
+            StorageService.instance.getTokenMetadata().pendingEndpointsFor(tk, mutation.getKeyspaceName());
+
+        Iterable<InetAddress> endpoints = Iterables.concat(naturalEndpoints, pendingEndpoints);
+        ArrayList<InetAddress> endpointsToHint = new ArrayList<>(Iterables.size(endpoints));
+
+        // local writes can timeout, but cannot be dropped (see LocalMutationRunnable and CASSANDRA-6510),
+        // so there is no need to hint or retry.
+        for (InetAddress target : endpoints)
+            if (!target.equals(FBUtilities.getBroadcastAddress()) && shouldHint(target))
+                endpointsToHint.add(target);
+
+        submitHint(mutation, endpointsToHint, null);
     }
 
     /**
@@ -1071,7 +1088,7 @@ public class StorageProxy implements StorageProxyMBean
         MessageOut<Mutation> message = null;
 
         boolean insertLocal = false;
-
+        ArrayList<InetAddress> endpointsToHint = null;
 
         for (InetAddress destination : targets)
         {
@@ -1115,16 +1132,21 @@ public class StorageProxy implements StorageProxyMBean
                         messages.add(destination);
                     }
                 }
-            } else
+            }
+            else
             {
-                if (!shouldHint(destination))
-                    continue;
-
-                // Schedule a local hint
-                submitHint(mutation, destination, responseHandler);
+                if (shouldHint(destination))
+                {
+                    if (endpointsToHint == null)
+                        endpointsToHint = new ArrayList<>(Iterables.size(targets));
+                    endpointsToHint.add(destination);
+                }
             }
         }
 
+        if (endpointsToHint != null)
+            submitHint(mutation, endpointsToHint, responseHandler);
+
         if (insertLocal)
             insertLocal(stage, mutation, responseHandler);
 
@@ -1139,66 +1161,6 @@ public class StorageProxy implements StorageProxyMBean
         }
     }
 
-    private static AtomicInteger getHintsInProgressFor(InetAddress destination)
-    {
-        try
-        {
-            return hintsInProgress.load(destination);
-        }
-        catch (Exception e)
-        {
-            throw new AssertionError(e);
-        }
-    }
-
-    public static Future<Void> submitHint(final Mutation mutation,
-                                          final InetAddress target,
-                                          final AbstractWriteResponseHandler<IMutation> responseHandler)
-    {
-        // local write that time out should be handled by LocalMutationRunnable
-        assert !target.equals(FBUtilities.getBroadcastAddress()) : target;
-
-        HintRunnable runnable = new HintRunnable(target)
-        {
-            public void runMayThrow()
-            {
-                int ttl = HintedHandOffManager.calculateHintTTL(mutation);
-                if (ttl > 0)
-                {
-                    logger.debug("Adding hint for {}", target);
-                    writeHintForMutation(mutation, System.currentTimeMillis(), ttl, target);
-                    // Notify the handler only for CL == ANY
-                    if (responseHandler != null && responseHandler.consistencyLevel == ConsistencyLevel.ANY)
-                        responseHandler.response(null);
-                } else
-                {
-                    logger.debug("Skipped writing hint for {} (ttl {})", target, ttl);
-                }
-            }
-        };
-
-        return submitHint(runnable);
-    }
-
-    private static Future<Void> submitHint(HintRunnable runnable)
-    {
-        StorageMetrics.totalHintsInProgress.inc();
-        getHintsInProgressFor(runnable.target).incrementAndGet();
-        return (Future<Void>) StageManager.getStage(Stage.MUTATION).submit(runnable);
-    }
-
-    /**
-     * @param now current time in milliseconds - relevant for hint replay handling of truncated CFs
-     */
-    public static void writeHintForMutation(Mutation mutation, long now, int ttl, InetAddress target)
-    {
-        assert ttl > 0;
-        UUID hostId = StorageService.instance.getTokenMetadata().getHostId(target);
-        assert hostId != null : "Missing host ID for " + target.getHostAddress();
-        HintedHandOffManager.instance.hintFor(mutation, now, ttl, hostId).apply();
-        StorageMetrics.totalHints.inc();
-    }
-
     private static void sendMessagesToNonlocalDC(MessageOut<? extends IMutation> message,
                                                  Collection<InetAddress> targets,
                                                  AbstractWriteResponseHandler<IMutation> handler)
@@ -2209,7 +2171,7 @@ public class StorageProxy implements StorageProxyMBean
     {
         if (!DatabaseDescriptor.hintedHandoffEnabled())
         {
-            HintedHandOffManager.instance.metrics.incrPastWindow(ep);
+            HintsService.instance.metrics.incrPastWindow(ep);
             return false;
         }
 
@@ -2220,7 +2182,7 @@ public class StorageProxy implements StorageProxyMBean
             if (disabledDCs.contains(dc))
             {
                 Tracing.trace("Not hinting {} since its data center {} has been disabled {}", ep, dc, disabledDCs);
-                HintedHandOffManager.instance.metrics.incrPastWindow(ep);
+                HintsService.instance.metrics.incrPastWindow(ep);
                 return false;
             }
         }
@@ -2228,7 +2190,7 @@ public class StorageProxy implements StorageProxyMBean
         boolean hintWindowExpired = Gossiper.instance.getEndpointDowntime(ep) > DatabaseDescriptor.getMaxHintWindow();
         if (hintWindowExpired)
         {
-            HintedHandOffManager.instance.metrics.incrPastWindow(ep);
+            HintsService.instance.metrics.incrPastWindow(ep);
             Tracing.trace("Not hinting {} which has been down {} ms", ep, Gossiper.instance.getEndpointDowntime(ep));
         }
         return !hintWindowExpired;
@@ -2363,7 +2325,7 @@ public class StorageProxy implements StorageProxyMBean
             if (System.currentTimeMillis() > constructionTime + DatabaseDescriptor.getTimeout(MessagingService.Verb.MUTATION))
             {
                 MessagingService.instance().incrementDroppedMessages(MessagingService.Verb.MUTATION);
-                HintRunnable runnable = new HintRunnable(FBUtilities.getBroadcastAddress())
+                HintRunnable runnable = new HintRunnable(Collections.singleton(FBUtilities.getBroadcastAddress()))
                 {
                     protected void runMayThrow() throws Exception
                     {
@@ -2393,11 +2355,11 @@ public class StorageProxy implements StorageProxyMBean
      */
     private abstract static class HintRunnable implements Runnable
     {
-        public final InetAddress target;
+        public final Collection<InetAddress> targets;
 
-        protected HintRunnable(InetAddress target)
+        protected HintRunnable(Collection<InetAddress> targets)
         {
-            this.target = target;
+            this.targets = targets;
         }
 
         public void run()
@@ -2412,8 +2374,9 @@ public class StorageProxy implements StorageProxyMBean
             }
             finally
             {
-                StorageMetrics.totalHintsInProgress.dec();
-                getHintsInProgressFor(target).decrementAndGet();
+                StorageMetrics.totalHintsInProgress.dec(targets.size());
+                for (InetAddress target : targets)
+                    getHintsInProgressFor(target).decrementAndGet();
             }
         }
 
@@ -2446,6 +2409,52 @@ public class StorageProxy implements StorageProxyMBean
             logger.warn("Some hints were not written before shutdown.  This is not supposed to happen.  You should (a) run repair, and (b) file a bug report");
     }
 
+    private static AtomicInteger getHintsInProgressFor(InetAddress destination)
+    {
+        try
+        {
+            return hintsInProgress.load(destination);
+        }
+        catch (Exception e)
+        {
+            throw new AssertionError(e);
+        }
+    }
+
+    public static Future<Void> submitHint(Mutation mutation, InetAddress target, AbstractWriteResponseHandler<IMutation> responseHandler)
+    {
+        return submitHint(mutation, Collections.singleton(target), responseHandler);
+    }
+
+    public static Future<Void> submitHint(Mutation mutation,
+                                          Collection<InetAddress> targets,
+                                          AbstractWriteResponseHandler<IMutation> responseHandler)
+    {
+        HintRunnable runnable = new HintRunnable(targets)
+        {
+            public void runMayThrow()
+            {
+                logger.debug("Adding hints for {}", targets);
+                HintsService.instance.write(Iterables.transform(targets, StorageService.instance::getHostIdForEndpoint),
+                                            Hint.create(mutation, System.currentTimeMillis()));
+                targets.forEach(HintsService.instance.metrics::incrCreatedHints);
+                // Notify the handler only for CL == ANY
+                if (responseHandler != null && responseHandler.consistencyLevel == ConsistencyLevel.ANY)
+                    responseHandler.response(null);
+            }
+        };
+
+        return submitHint(runnable);
+    }
+
+    private static Future<Void> submitHint(HintRunnable runnable)
+    {
+        StorageMetrics.totalHintsInProgress.inc(runnable.targets.size());
+        for (InetAddress target : runnable.targets)
+            getHintsInProgressFor(target).incrementAndGet();
+        return (Future<Void>) StageManager.getStage(Stage.MUTATION).submit(runnable);
+    }
+
     public Long getRpcTimeout() { return DatabaseDescriptor.getRpcTimeout(); }
     public void setRpcTimeout(Long timeoutInMillis) { DatabaseDescriptor.setRpcTimeout(timeoutInMillis); }
 
@@ -2478,11 +2487,11 @@ public class StorageProxy implements StorageProxyMBean
     public long getReadRepairAttempted() {
         return ReadRepairMetrics.attempted.getCount();
     }
-    
+
     public long getReadRepairRepairedBlocking() {
         return ReadRepairMetrics.repairedBlocking.getCount();
     }
-    
+
     public long getReadRepairRepairedBackground() {
         return ReadRepairMetrics.repairedBackground.getCount();
     }

src/java/org/apache/cassandra/service/StorageService.java

@@ -28,22 +28,8 @@ import java.lang.management.ManagementFactory;
 import java.net.InetAddress;
 import java.net.UnknownHostException;
 import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.LinkedHashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
 import java.util.Map.Entry;
-import java.util.Set;
-import java.util.SortedMap;
-import java.util.TreeMap;
-import java.util.UUID;
 import java.util.concurrent.CopyOnWriteArrayList;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorService;
@@ -54,6 +40,7 @@ import java.util.concurrent.TimeoutException;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 
+import javax.annotation.Nullable;
 import javax.management.JMX;
 import javax.management.MBeanServer;
 import javax.management.NotificationBroadcasterSupport;
@@ -95,6 +82,8 @@ import org.apache.cassandra.gms.IEndpointStateChangeSubscriber;
 import org.apache.cassandra.gms.IFailureDetector;
 import org.apache.cassandra.gms.TokenSerializer;
 import org.apache.cassandra.gms.VersionedValue;
+import org.apache.cassandra.hints.HintVerbHandler;
+import org.apache.cassandra.hints.HintsService;
 import org.apache.cassandra.io.sstable.SSTableLoader;
 import org.apache.cassandra.io.util.FileUtils;
 import org.apache.cassandra.locator.AbstractReplicationStrategy;
@@ -303,6 +292,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
         MessagingService.instance().registerVerbHandlers(MessagingService.Verb.PAXOS_PREPARE, new PrepareVerbHandler());
         MessagingService.instance().registerVerbHandlers(MessagingService.Verb.PAXOS_PROPOSE, new ProposeVerbHandler());
         MessagingService.instance().registerVerbHandlers(MessagingService.Verb.PAXOS_COMMIT, new CommitVerbHandler());
+        MessagingService.instance().registerVerbHandlers(MessagingService.Verb.HINT, new HintVerbHandler());
 
         // see BootStrapper for a summary of how the bootstrap verbs interact
         MessagingService.instance().registerVerbHandlers(MessagingService.Verb.REPLICATION_FINISHED, new ReplicationFinishedVerbHandler());
@@ -585,7 +575,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
         logger.info("Cassandra version: {}", FBUtilities.getReleaseVersionString());
         logger.info("Thrift API version: {}", cassandraConstants.VERSION);
         logger.info("CQL supported versions: {} (default: {})",
-                    StringUtils.join(ClientState.getCQLSupportedVersion(), ","), ClientState.DEFAULT_CQL_VERSION);
+                StringUtils.join(ClientState.getCQLSupportedVersion(), ","), ClientState.DEFAULT_CQL_VERSION);
 
         initialized = true;
 
@@ -649,6 +639,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
                 MessagingService.instance().shutdown();
                 materializedViewMutationStage.shutdown();
                 batchlogMutationStage.shutdown();
+                HintsService.instance.pauseDispatch();
                 counterMutationStage.shutdown();
                 mutationStage.shutdown();
                 materializedViewMutationStage.awaitTermination(3600, TimeUnit.SECONDS);
@@ -681,6 +672,8 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
                 if (FBUtilities.isWindows())
                     WindowsTimer.endTimerPeriod(DatabaseDescriptor.getWindowsTimerInterval());
 
+                HintsService.instance.shutdownBlocking();
+
                 // wait for miscellaneous tasks like sstable and commitlog segment deletion
                 ScheduledExecutors.nonPeriodicTasks.shutdown();
                 if (!ScheduledExecutors.nonPeriodicTasks.awaitTermination(1, TimeUnit.MINUTES))
@@ -792,7 +785,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
                 MessagingService.instance().listen(FBUtilities.getLocalAddress());
             LoadBroadcaster.instance.startBroadcasting();
 
-            HintedHandOffManager.instance.start();
+            HintsService.instance.startDispatch();
             BatchlogManager.instance.start();
         }
     }
@@ -1564,6 +1557,11 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
         return getTokenMetadata().getHostId(FBUtilities.getBroadcastAddress()).toString();
     }
 
+    public UUID getLocalHostUUID()
+    {
+        return getTokenMetadata().getHostId(FBUtilities.getBroadcastAddress());
+    }
+
     public Map<String, String> getHostIdMap()
     {
         Map<String, String> mapOut = new HashMap<>();
@@ -2119,11 +2117,13 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
     private void excise(Collection<Token> tokens, InetAddress endpoint)
     {
         logger.info("Removing tokens {} for {}", tokens, endpoint);
-        HintedHandOffManager.instance.deleteHintsForEndpoint(endpoint);
+
+        if (tokenMetadata.isMember(endpoint))
+            HintsService.instance.excise(tokenMetadata.getHostId(endpoint));
+
         removeEndpoint(endpoint);
         tokenMetadata.removeEndpoint(endpoint);
         tokenMetadata.removeBootstrapTokens(tokens);
-
         notifyLeft(endpoint);
         PendingRangeCalculatorService.instance.update();
     }
@@ -2337,10 +2337,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
         MigrationManager.instance.scheduleSchemaPull(endpoint, state);
 
         if (tokenMetadata.isMember(endpoint))
-        {
-            HintedHandOffManager.instance.scheduleHintDelivery(endpoint, true);
             notifyUp(endpoint);
-        }
     }
 
     public void onRemove(InetAddress endpoint)
@@ -2380,9 +2377,10 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
         return map;
     }
 
+    // TODO
     public final void deliverHints(String host) throws UnknownHostException
     {
-        HintedHandOffManager.instance.scheduleHintDelivery(host);
+        throw new UnsupportedOperationException();
     }
 
     public Collection<Token> getLocalTokens()
@@ -2392,6 +2390,18 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
         return tokens;
     }
 
+    @Nullable
+    public InetAddress getEndpointForHostId(UUID hostId)
+    {
+        return tokenMetadata.getEndpointForHostId(hostId);
+    }
+
+    @Nullable
+    public UUID getHostIdForEndpoint(InetAddress address)
+    {
+        return tokenMetadata.getHostId(address);
+    }
+
     /* These methods belong to the MBean interface */
 
     public List<String> getTokens()
@@ -3418,7 +3428,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
     private Future<StreamState> streamHints()
     {
         // StreamPlan will not fail if there are zero files to transfer, so flush anyway (need to get any in-memory hints, as well)
-        ColumnFamilyStore hintsCF = Keyspace.open(SystemKeyspace.NAME).getColumnFamilyStore(SystemKeyspace.HINTS);
+        ColumnFamilyStore hintsCF = Keyspace.open(SystemKeyspace.NAME).getColumnFamilyStore(SystemKeyspace.LEGACY_HINTS);
         FBUtilities.waitOnFuture(hintsCF.forceFlush());
 
         // gather all live nodes in the cluster that aren't also leaving
@@ -3451,7 +3461,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
                                                           preferred,
                                                           SystemKeyspace.NAME,
                                                           ranges,
-                                                          SystemKeyspace.HINTS)
+                                                          SystemKeyspace.LEGACY_HINTS)
                                           .execute();
         }
     }
@@ -3835,7 +3845,11 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
     public synchronized void drain() throws IOException, InterruptedException, ExecutionException
     {
         inShutdownHook = true;
-        
+
+        BatchlogManager.shutdown();
+
+        HintsService.instance.pauseDispatch();
+
         ExecutorService counterMutationStage = StageManager.getStage(Stage.COUNTER_MUTATION);
         ExecutorService batchlogMutationStage = StageManager.getStage(Stage.BATCHLOG_MUTATION);
         ExecutorService materializedViewMutationStage = StageManager.getStage(Stage.MATERIALIZED_VIEW_MUTATION);
@@ -3899,7 +3913,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
         }
         FBUtilities.waitOnFutures(flushes);
 
-        BatchlogManager.shutdown();
+        HintsService.instance.shutdownBlocking();
 
         // whilst we've flushed all the CFs, which will have recycled all completed segments, we want to ensure
         // there are no segments to replay, so we force the recycling of any remaining (should be at most one)

src/java/org/apache/cassandra/tools/NodeProbe.java

@@ -57,7 +57,6 @@ import org.apache.cassandra.concurrent.Stage;
 import org.apache.cassandra.db.BatchlogManager;
 import org.apache.cassandra.db.BatchlogManagerMBean;
 import org.apache.cassandra.db.ColumnFamilyStoreMBean;
-import org.apache.cassandra.db.HintedHandOffManager;
 import org.apache.cassandra.db.HintedHandOffManagerMBean;
 import org.apache.cassandra.db.compaction.CompactionManager;
 import org.apache.cassandra.db.compaction.CompactionManagerMBean;
@@ -65,6 +64,7 @@ import org.apache.cassandra.gms.FailureDetector;
 import org.apache.cassandra.gms.FailureDetectorMBean;
 import org.apache.cassandra.gms.Gossiper;
 import org.apache.cassandra.gms.GossiperMBean;
+import org.apache.cassandra.db.HintedHandOffManager;
 import org.apache.cassandra.locator.EndpointSnitchInfoMBean;
 import org.apache.cassandra.metrics.CassandraMetricsRegistry;
 import org.apache.cassandra.metrics.TableMetrics.Sampler;

src/java/org/apache/cassandra/utils/FBUtilities.java

@@ -26,10 +26,10 @@ import java.security.MessageDigest;
 import java.security.NoSuchAlgorithmException;
 import java.util.*;
 import java.util.concurrent.*;
+import java.util.zip.CRC32;
 import java.util.zip.Checksum;
 
 import com.google.common.base.Joiner;
-import org.apache.cassandra.utils.AbstractIterator;
 import org.apache.commons.lang3.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -332,7 +332,7 @@ public class FBUtilities
 
     public static int nowInSeconds()
     {
-        return (int)(System.currentTimeMillis() / 1000);
+        return (int) (System.currentTimeMillis() / 1000);
     }
 
     public static void waitOnFutures(Iterable<Future<?>> futures)
@@ -595,6 +595,34 @@ public class FBUtilities
         checksum.update((v >>> 0) & 0xFF);
     }
 
+    /**
+      * Updates checksum with the provided ByteBuffer at the given offset + length.
+      * Resets position and limit back to their original values on return.
+      * This method is *NOT* thread-safe.
+      */
+    public static void updateChecksum(CRC32 checksum, ByteBuffer buffer, int offset, int length)
+    {
+        int position = buffer.position();
+        int limit = buffer.limit();
+
+        buffer.position(offset).limit(offset + length);
+        checksum.update(buffer);
+
+        buffer.position(position).limit(limit);
+    }
+
+    /**
+     * Updates checksum with the provided ByteBuffer.
+     * Resets position back to its original values on return.
+     * This method is *NOT* thread-safe.
+     */
+    public static void updateChecksum(CRC32 checksum, ByteBuffer buffer)
+    {
+        int position = buffer.position();
+        checksum.update(buffer);
+        buffer.position(position);
+    }
+
     private static final ThreadLocal<byte[]> threadLocalScratchBuffer = new ThreadLocal<byte[]>()
     {
         @Override

src/java/org/apache/cassandra/utils/Throwables.java

@@ -18,10 +18,25 @@
 */
 package org.apache.cassandra.utils;
 
-public class Throwables
-{
+import java.io.File;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Iterator;
+import java.util.stream.Stream;
 
-    public static Throwable merge(Throwable existingFail, Throwable newFail)
+import org.apache.cassandra.io.FSReadError;
+import org.apache.cassandra.io.FSWriteError;
+
+public final class Throwables
+{
+    public enum FileOpType { READ, WRITE }
+
+    public interface DiscreteAction<E extends Exception>
+    {
+        void perform() throws E;
+    }
+
+    public static <T extends Throwable> T merge(T existingFail, T newFail)
     {
         if (existingFail == null)
             return newFail;
@@ -31,7 +46,74 @@ public class Throwables
 
     public static void maybeFail(Throwable fail)
     {
-        if (fail != null)
-            com.google.common.base.Throwables.propagate(fail);
+        if (failIfCanCast(fail, null))
+            throw new RuntimeException(fail);
+    }
+
+    public static <T extends Throwable> void maybeFail(Throwable fail, Class<T> checked) throws T
+    {
+        if (failIfCanCast(fail, checked))
+            throw new RuntimeException(fail);
+    }
+
+    public static <T extends Throwable> boolean failIfCanCast(Throwable fail, Class<T> checked) throws T
+    {
+        if (fail == null)
+            return false;
+
+        if (fail instanceof Error)
+            throw (Error) fail;
+
+        if (fail instanceof RuntimeException)
+            throw (RuntimeException) fail;
+
+        if (checked != null && checked.isInstance(fail))
+            throw checked.cast(fail);
+
+        return true;
+    }
+
+    @SafeVarargs
+    public static <E extends Exception> void perform(DiscreteAction<? extends E> ... actions) throws E
+    {
+        perform(Arrays.stream(actions));
+    }
+
+    @SuppressWarnings("unchecked")
+    public static <E extends Exception> void perform(Stream<DiscreteAction<? extends E>> actions) throws E
+    {
+        Throwable fail = null;
+        Iterator<DiscreteAction<? extends E>> iter = actions.iterator();
+        while (iter.hasNext())
+        {
+            DiscreteAction<? extends E> action = iter.next();
+            try
+            {
+                action.perform();
+            }
+            catch (Throwable t)
+            {
+                fail = merge(fail, t);
+            }
+        }
+
+        if (failIfCanCast(fail, null))
+            throw (E) fail;
+    }
+
+    @SafeVarargs
+    public static void perform(File against, FileOpType opType, DiscreteAction<? extends IOException> ... actions)
+    {
+        perform(Arrays.stream(actions).map((action) -> () ->
+        {
+            try
+            {
+                action.perform();
+            }
+            catch (IOException e)
+            {
+                throw (opType == FileOpType.WRITE) ? new FSWriteError(e, against) : new FSReadError(e, against);
+            }
+        }));
     }
 }

test/conf/cassandra.yaml

@@ -8,6 +8,7 @@ commitlog_sync: batch
 commitlog_sync_batch_window_in_ms: 1.0
 commitlog_segment_size_in_mb: 5
 commitlog_directory: build/test/cassandra/commitlog
+hints_directory: build/test/cassandra/hints
 partitioner: org.apache.cassandra.dht.ByteOrderedPartitioner
 listen_address: 127.0.0.1
 storage_port: 7010

test/long/org/apache/cassandra/hints/HintsWriteThenReadTest.java

@@ -0,0 +1,191 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.hints;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.ByteBuffer;
+import java.nio.file.Files;
+import java.util.Iterator;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.zip.CRC32;
+
+import com.google.common.collect.Iterables;
+import org.junit.Test;
+
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.config.Schema;
+import org.apache.cassandra.db.Mutation;
+import org.apache.cassandra.db.RowUpdateBuilder;
+import org.apache.cassandra.db.rows.Cell;
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.utils.FBUtilities;
+
+import static junit.framework.Assert.assertEquals;
+import static junit.framework.Assert.assertNotNull;
+import static junit.framework.Assert.assertTrue;
+
+import static org.apache.cassandra.Util.dk;
+import static org.apache.cassandra.utils.ByteBufferUtil.bytes;
+
+public class HintsWriteThenReadTest
+{
+    private static final String KEYSPACE = "hints_write_then_read_test";
+    private static final String TABLE = "table";
+
+    private static final int HINTS_COUNT = 10_000_000;
+
+    @Test
+    public void testWriteReadCycle() throws IOException
+    {
+        SchemaLoader.prepareServer();
+        SchemaLoader.createKeyspace(KEYSPACE, KeyspaceParams.simple(1), SchemaLoader.standardCFMD(KEYSPACE, TABLE));
+
+        HintsDescriptor descriptor = new HintsDescriptor(UUID.randomUUID(), System.currentTimeMillis());
+
+        File directory = Files.createTempDirectory(null).toFile();
+        try
+        {
+            testWriteReadCycle(directory, descriptor);
+        }
+        finally
+        {
+            directory.deleteOnExit();
+        }
+    }
+
+    private void testWriteReadCycle(File directory, HintsDescriptor descriptor) throws IOException
+    {
+        // write HINTS_COUNT hints to a file
+        writeHints(directory, descriptor);
+
+        // calculate the checksum of the file, then compare to the .crc32 checksum file content
+        verifyChecksum(directory, descriptor);
+
+        // iterate over the written hints, make sure they are all present
+        verifyHints(directory, descriptor);
+    }
+
+    private void writeHints(File directory, HintsDescriptor descriptor) throws IOException
+    {
+        try (HintsWriter writer = HintsWriter.create(directory, descriptor))
+        {
+            write(writer, descriptor.timestamp);
+        }
+    }
+
+    private static void verifyChecksum(File directory, HintsDescriptor descriptor) throws IOException
+    {
+        File hintsFile = new File(directory, descriptor.fileName());
+        File checksumFile = new File(directory, descriptor.checksumFileName());
+
+        assertTrue(checksumFile.exists());
+
+        String actualChecksum = Integer.toHexString(calculateChecksum(hintsFile));
+        String expectedChecksum = Files.readAllLines(checksumFile.toPath()).iterator().next();
+
+        assertEquals(expectedChecksum, actualChecksum);
+    }
+
+    private void verifyHints(File directory, HintsDescriptor descriptor)
+    {
+        long baseTimestamp = descriptor.timestamp;
+        int index = 0;
+
+        try (HintsReader reader = HintsReader.open(new File(directory, descriptor.fileName())))
+        {
+            for (HintsReader.Page page : reader)
+            {
+                Iterator<Hint> hints = page.hintsIterator();
+                while (hints.hasNext())
+                {
+                    Hint hint = hints.next();
+
+                    long timestamp = baseTimestamp + index;
+                    Mutation mutation = hint.mutation;
+
+                    assertEquals(timestamp, hint.creationTime);
+                    assertEquals(dk(bytes(index)), mutation.key());
+
+                    Row row = mutation.getPartitionUpdates().iterator().next().iterator().next();
+                    assertEquals(1, Iterables.size(row.cells()));
+                    assertEquals(bytes(index), row.clustering().get(0));
+                    Cell cell = row.cells().iterator().next();
+                    assertNotNull(cell);
+                    assertEquals(bytes(index), cell.value());
+                    assertEquals(timestamp * 1000, cell.timestamp());
+
+                    index++;
+                }
+            }
+        }
+
+        assertEquals(index, HINTS_COUNT);
+    }
+
+    private void write(HintsWriter writer, long timestamp) throws IOException
+    {
+        ByteBuffer buffer = ByteBuffer.allocateDirect(256 * 1024);
+        try (HintsWriter.Session session = writer.newSession(buffer))
+        {
+            write(session, timestamp);
+        }
+        FileUtils.clean(buffer);
+    }
+
+    private void write(HintsWriter.Session session, long timestamp) throws IOException
+    {
+        for (int i = 0; i < HINTS_COUNT; i++)
+            session.append(createHint(i, timestamp));
+    }
+
+    private static Hint createHint(int idx, long baseTimestamp)
+    {
+        long timestamp = baseTimestamp + idx;
+        return Hint.create(createMutation(idx, TimeUnit.MILLISECONDS.toMicros(timestamp)), timestamp);
+    }
+
+    private static Mutation createMutation(int index, long timestamp)
+    {
+        CFMetaData table = Schema.instance.getCFMetaData(KEYSPACE, TABLE);
+        return new RowUpdateBuilder(table, timestamp, bytes(index))
+               .clustering(bytes(index))
+               .add("val", bytes(index))
+               .build();
+    }
+
+    private static int calculateChecksum(File file) throws IOException
+    {
+        CRC32 crc = new CRC32();
+        byte[] buffer = new byte[FBUtilities.MAX_UNSIGNED_SHORT];
+
+        try (InputStream in = Files.newInputStream(file.toPath()))
+        {
+            int bytesRead;
+            while((bytesRead = in.read(buffer)) != -1)
+                crc.update(buffer, 0, bytesRead);
+        }
+
+        return (int) crc.getValue();
+    }
+}

test/unit/org/apache/cassandra/OffsetAwareConfigurationLoader.java

@@ -54,6 +54,7 @@ public class OffsetAwareConfigurationLoader extends YamlConfigurationLoader
 
         config.commitlog_directory += File.pathSeparator + offset;
         config.saved_caches_directory += File.pathSeparator + offset;
+        config.hints_directory += File.pathSeparator + offset;
         for (int i = 0; i < config.data_file_directories.length; i++)
             config.data_file_directories[i] += File.pathSeparator + offset;
 

test/unit/org/apache/cassandra/db/HintedHandOffTest.java

@@ -1,150 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-package org.apache.cassandra.db;
-
-import java.net.InetAddress;
-import java.util.Map;
-import java.util.UUID;
-
-import com.google.common.collect.Iterators;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import org.apache.cassandra.SchemaLoader;
-import org.apache.cassandra.cql3.UntypedResultSet;
-import org.apache.cassandra.db.marshal.Int32Type;
-import org.apache.cassandra.db.marshal.UUIDType;
-import org.apache.cassandra.exceptions.ConfigurationException;
-import org.apache.cassandra.schema.KeyspaceParams;
-import org.apache.cassandra.utils.ByteBufferUtil;
-import org.apache.cassandra.utils.FBUtilities;
-
-import static org.apache.cassandra.cql3.QueryProcessor.executeInternal;
-import static org.junit.Assert.assertEquals;
-
-public class HintedHandOffTest
-{
-
-    public static final String KEYSPACE4 = "HintedHandOffTest4";
-    public static final String STANDARD1_CF = "Standard1";
-
-    @BeforeClass
-    public static void defineSchema() throws ConfigurationException
-    {
-        SchemaLoader.prepareServer();
-        SchemaLoader.createKeyspace(KEYSPACE4,
-                                    KeyspaceParams.simple(1),
-                                    SchemaLoader.standardCFMD(KEYSPACE4, STANDARD1_CF));
-    }
-
-    @Before
-    public void clearHints()
-    {
-        Keyspace systemKeyspace = Keyspace.open("system");
-        ColumnFamilyStore hintStore = systemKeyspace.getColumnFamilyStore(SystemKeyspace.HINTS);
-        hintStore.clearUnsafe();
-    }
-
-    // Test compaction of hints column family. It shouldn't remove all columns on compaction.
-    @Test
-    public void testCompactionOfHintsCF() throws Exception
-    {
-        // prepare hints column family
-        Keyspace systemKeyspace = Keyspace.open("system");
-        ColumnFamilyStore hintStore = systemKeyspace.getColumnFamilyStore(SystemKeyspace.HINTS);
-        hintStore.clearUnsafe();
-        hintStore.metadata.gcGraceSeconds(36000); // 10 hours
-        hintStore.disableAutoCompaction();
-
-        // insert 1 hint
-        Mutation rm = mutation();
-        HintedHandOffManager.instance.hintFor(rm,
-                                              System.currentTimeMillis(),
-                                              HintedHandOffManager.calculateHintTTL(rm),
-                                              UUID.randomUUID())
-                                     .applyUnsafe();
-
-        // flush data to disk
-        hintStore.forceBlockingFlush();
-        assertEquals(1, hintStore.getLiveSSTables().size());
-
-        // submit compaction
-        HintedHandOffManager.instance.compact();
-
-        // single row should not be removed because of gc_grace_seconds
-        // is 10 hours and there are no any tombstones in sstable
-        assertEquals(1, hintStore.getLiveSSTables().size());
-    }
-
-    @Test
-    public void testHintsMetrics() throws Exception
-    {
-        for (int i = 0; i < 99; i++)
-            HintedHandOffManager.instance.metrics.incrPastWindow(InetAddress.getLocalHost());
-        HintedHandOffManager.instance.metrics.log();
-
-        UntypedResultSet rows = executeInternal("SELECT hints_dropped FROM system." + SystemKeyspace.PEER_EVENTS);
-        Map<UUID, Integer> returned = rows.one().getMap("hints_dropped", UUIDType.instance, Int32Type.instance);
-        assertEquals(Iterators.getLast(returned.values().iterator()).intValue(), 99);
-    }
-
-    @Test(timeout = 5000)
-    public void testTruncateHints() throws Exception
-    {
-        // insert 1 hint
-        Mutation rm = mutation();
-        HintedHandOffManager.instance.hintFor(rm,
-                                              System.currentTimeMillis(),
-                                              HintedHandOffManager.calculateHintTTL(rm),
-                                              UUID.randomUUID())
-                                     .applyUnsafe();
-
-        assert getNoOfHints() == 1;
-
-        HintedHandOffManager.instance.truncateAllHints();
-
-        while(getNoOfHints() > 0)
-        {
-            Thread.sleep(100);
-        }
-
-        assert getNoOfHints() == 0;
-    }
-
-    private Mutation mutation()
-    {
-        // get a random mutation to write a hint for
-        return new RowUpdateBuilder(Keyspace.open(KEYSPACE4).getColumnFamilyStore(STANDARD1_CF).metadata,
-                                    FBUtilities.timestampMicros(),
-                                    ByteBufferUtil.bytes(1))
-               .clustering("cluster_col0")
-               .add("val", "value0")
-               .build();
-    }
-
-    private int getNoOfHints()
-    {
-        String req = "SELECT * FROM system.%s";
-        UntypedResultSet resultSet = executeInternal(String.format(req, SystemKeyspace.HINTS));
-        return resultSet.size();
-    }
-}

test/unit/org/apache/cassandra/db/commitlog/CommitLogTest.java

@@ -250,7 +250,7 @@ public class CommitLogTest
         // Adding new mutation on another CF, large enough (including CL entry overhead) that a new segment is created
         Mutation rm2 = new RowUpdateBuilder(cfs2.metadata, 0, "k")
                        .clustering("bytes")
-                       .add("val", ByteBuffer.allocate((DatabaseDescriptor.getCommitLogSegmentSize()/2) - 200))
+                       .add("val", ByteBuffer.allocate(DatabaseDescriptor.getMaxMutationSize() - 200))
                        .build();
         CommitLog.instance.add(rm2);
         // also forces a new segment, since each entry-with-overhead is just under half the CL size
@@ -280,7 +280,7 @@ public class CommitLogTest
                       .clustering(colName)
                       .add("val", ByteBuffer.allocate(allocSize)).build();
 
-        int max = (DatabaseDescriptor.getCommitLogSegmentSize() / 2);
+        int max = DatabaseDescriptor.getMaxMutationSize();
         max -= CommitLogSegment.ENTRY_OVERHEAD_SIZE; // log entry overhead
 
         // Note that the size of the value if vint encoded. So we first compute the ovehead of the mutation without the value and it's size

test/unit/org/apache/cassandra/hints/ChecksummedDataInputTest.java

@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.hints;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.zip.CRC32;
+
+import org.junit.Test;
+
+import org.apache.cassandra.hints.ChecksummedDataInput;
+import org.apache.cassandra.io.util.AbstractDataInput;
+import org.apache.cassandra.io.util.DataOutputBuffer;
+import org.apache.cassandra.utils.FBUtilities;
+
+import static junit.framework.Assert.assertEquals;
+import static junit.framework.Assert.assertTrue;
+
+public class ChecksummedDataInputTest
+{
+    @Test
+    public void testThatItWorks() throws IOException
+    {
+        // fill a bytebuffer with some input
+        DataOutputBuffer out = new DataOutputBuffer();
+        out.write(127);
+        out.write(new byte[]{ 0, 1, 2, 3, 4, 5, 6 });
+        out.writeBoolean(false);
+        out.writeByte(10);
+        out.writeChar('t');
+        out.writeDouble(3.3);
+        out.writeFloat(2.2f);
+        out.writeInt(42);
+        out.writeLong(Long.MAX_VALUE);
+        out.writeShort(Short.MIN_VALUE);
+        out.writeUTF("utf");
+        ByteBuffer buffer = out.buffer();
+
+        // calculate resulting CRC
+        CRC32 crc = new CRC32();
+        FBUtilities.updateChecksum(crc, buffer);
+        int expectedCRC = (int) crc.getValue();
+
+        ChecksummedDataInput crcInput = ChecksummedDataInput.wrap(new DummyByteBufferDataInput(buffer.duplicate()));
+        crcInput.limit(buffer.remaining());
+
+        // assert that we read all the right values back
+        assertEquals(127, crcInput.read());
+        byte[] bytes = new byte[7];
+        crcInput.readFully(bytes);
+        assertTrue(Arrays.equals(new byte[]{ 0, 1, 2, 3, 4, 5, 6 }, bytes));
+        assertEquals(false, crcInput.readBoolean());
+        assertEquals(10, crcInput.readByte());
+        assertEquals('t', crcInput.readChar());
+        assertEquals(3.3, crcInput.readDouble());
+        assertEquals(2.2f, crcInput.readFloat());
+        assertEquals(42, crcInput.readInt());
+        assertEquals(Long.MAX_VALUE, crcInput.readLong());
+        assertEquals(Short.MIN_VALUE, crcInput.readShort());
+        assertEquals("utf", crcInput.readUTF());
+
+        // assert that the crc matches, and that we've read exactly as many bytes as expected
+        assertEquals(0, crcInput.bytesRemaining());
+        assertEquals(expectedCRC, crcInput.getCrc());
+    }
+
+    private static final class DummyByteBufferDataInput extends AbstractDataInput
+    {
+        private final ByteBuffer buffer;
+
+        DummyByteBufferDataInput(ByteBuffer buffer)
+        {
+            this.buffer = buffer;
+        }
+
+        public void seek(long position)
+        {
+            throw new UnsupportedOperationException();
+        }
+
+        public long getPosition()
+        {
+            throw new UnsupportedOperationException();
+        }
+
+        public long getPositionLimit()
+        {
+            throw new UnsupportedOperationException();
+        }
+
+        public int read()
+        {
+            return buffer.get() & 0xFF;
+        }
+    }
+}

test/unit/org/apache/cassandra/hints/HintMessageTest.java

@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.hints;
+
+import java.io.IOException;
+import java.util.UUID;
+
+import org.junit.Test;
+
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.config.Schema;
+import org.apache.cassandra.db.Mutation;
+import org.apache.cassandra.db.RowUpdateBuilder;
+import org.apache.cassandra.io.util.DataInputBuffer;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputBuffer;
+import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.utils.FBUtilities;
+
+import static junit.framework.Assert.assertEquals;
+
+import static org.apache.cassandra.hints.HintsTestUtil.assertHintsEqual;
+import static org.apache.cassandra.utils.ByteBufferUtil.bytes;
+
+public class HintMessageTest
+{
+    private static final String KEYSPACE = "hint_message_test";
+    private static final String TABLE = "table";
+
+    @Test
+    public void testSerializer() throws IOException
+    {
+        SchemaLoader.prepareServer();
+        SchemaLoader.createKeyspace(KEYSPACE, KeyspaceParams.simple(1), SchemaLoader.standardCFMD(KEYSPACE, TABLE));
+
+        UUID hostId = UUID.randomUUID();
+        long now = FBUtilities.timestampMicros();
+
+        CFMetaData table = Schema.instance.getCFMetaData(KEYSPACE, TABLE);
+        Mutation mutation =
+            new RowUpdateBuilder(table, now, bytes("key"))
+                .clustering("column")
+                .add("val", "val" + 1234)
+                .build();
+        Hint hint = Hint.create(mutation, now / 1000);
+        HintMessage message = new HintMessage(hostId, hint);
+
+        // serialize
+        int serializedSize = (int) HintMessage.serializer.serializedSize(message, MessagingService.current_version);
+        DataOutputBuffer dob = new DataOutputBuffer();
+        HintMessage.serializer.serialize(message, dob, MessagingService.current_version);
+        assertEquals(serializedSize, dob.getLength());
+
+        // deserialize
+        DataInputPlus di = new DataInputBuffer(dob.buffer(), true);
+        HintMessage deserializedMessage = HintMessage.serializer.deserialize(di, MessagingService.current_version);
+
+        // compare before/after
+        assertEquals(hostId, deserializedMessage.hostId);
+        assertHintsEqual(message.hint, deserializedMessage.hint);
+    }
+}

test/unit/org/apache/cassandra/hints/HintTest.java

@@ -0,0 +1,231 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.hints;
+
+import java.io.IOException;
+
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.Util;
+import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.config.Schema;
+import org.apache.cassandra.db.*;
+import org.apache.cassandra.db.partitions.FilteredPartition;
+import org.apache.cassandra.db.partitions.PartitionIterator;
+import org.apache.cassandra.db.partitions.PartitionUpdate;
+import org.apache.cassandra.io.util.DataInputBuffer;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputBuffer;
+import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.schema.TableParams;
+import org.apache.cassandra.utils.FBUtilities;
+
+import static junit.framework.Assert.*;
+
+import static org.apache.cassandra.Util.dk;
+import static org.apache.cassandra.hints.HintsTestUtil.assertHintsEqual;
+import static org.apache.cassandra.hints.HintsTestUtil.assertPartitionsEqual;
+import static org.apache.cassandra.utils.ByteBufferUtil.bytes;
+
+public class HintTest
+{
+    private static final String KEYSPACE = "hint_test";
+    private static final String TABLE0 = "table_0";
+    private static final String TABLE1 = "table_1";
+    private static final String TABLE2 = "table_2";
+
+    @BeforeClass
+    public static void defineSchema()
+    {
+        SchemaLoader.prepareServer();
+        SchemaLoader.createKeyspace(KEYSPACE,
+                                    KeyspaceParams.simple(1),
+                                    SchemaLoader.standardCFMD(KEYSPACE, TABLE0),
+                                    SchemaLoader.standardCFMD(KEYSPACE, TABLE1),
+                                    SchemaLoader.standardCFMD(KEYSPACE, TABLE2));
+    }
+
+    @Before
+    public void resetGcGraceSeconds()
+    {
+        for (CFMetaData table : Schema.instance.getTables(KEYSPACE))
+            table.gcGraceSeconds(TableParams.DEFAULT_GC_GRACE_SECONDS);
+    }
+
+    @Test
+    public void testSerializer() throws IOException
+    {
+        long now = FBUtilities.timestampMicros();
+        Mutation mutation = createMutation("testSerializer", now);
+        Hint hint = Hint.create(mutation, now / 1000);
+
+        // serialize
+        int serializedSize = (int) Hint.serializer.serializedSize(hint, MessagingService.current_version);
+        DataOutputBuffer dob = new DataOutputBuffer();
+        Hint.serializer.serialize(hint, dob, MessagingService.current_version);
+        assertEquals(serializedSize, dob.getLength());
+
+        // deserialize
+        DataInputPlus di = new DataInputBuffer(dob.buffer(), true);
+        Hint deserializedHint = Hint.serializer.deserialize(di, MessagingService.current_version);
+
+        // compare before/after
+        assertHintsEqual(hint, deserializedHint);
+    }
+
+    @Test
+    public void testApply()
+    {
+        long now = FBUtilities.timestampMicros();
+        String key = "testApply";
+        Mutation mutation = createMutation(key, now);
+        Hint hint = Hint.create(mutation, now / 1000);
+
+        // sanity check that there is no data inside yet
+        assertNoPartitions(key, TABLE0);
+        assertNoPartitions(key, TABLE1);
+        assertNoPartitions(key, TABLE2);
+
+        hint.apply();
+
+        // assert that we can read the inserted partitions
+        for (PartitionUpdate partition : mutation.getPartitionUpdates())
+            assertPartitionsEqual(partition, readPartition(key, partition.metadata().cfName));
+    }
+
+    @Test
+    public void testApplyWithTruncation()
+    {
+        long now = FBUtilities.timestampMicros();
+        String key = "testApplyWithTruncation";
+        Mutation mutation = createMutation(key, now);
+
+        // sanity check that there is no data inside yet
+        assertNoPartitions(key, TABLE0);
+        assertNoPartitions(key, TABLE1);
+        assertNoPartitions(key, TABLE2);
+
+        // truncate TABLE1
+        Keyspace.open(KEYSPACE).getColumnFamilyStore(TABLE1).truncateBlocking();
+
+        // create and apply a hint with creation time in the past (one second before the truncation)
+        Hint.create(mutation, now / 1000 - 1).apply();
+
+        // TABLE1 update should have been skipped and not applied, as expired
+        assertNoPartitions(key, TABLE1);
+
+        // TABLE0 and TABLE2 updates should have been applied successfully
+        assertPartitionsEqual(mutation.getPartitionUpdate(Schema.instance.getId(KEYSPACE, TABLE0)), readPartition(key, TABLE0));
+        assertPartitionsEqual(mutation.getPartitionUpdate(Schema.instance.getId(KEYSPACE, TABLE2)), readPartition(key, TABLE2));
+    }
+
+    @Test
+    public void testApplyWithRegularExpiration()
+    {
+        long now = FBUtilities.timestampMicros();
+        String key = "testApplyWithRegularExpiration";
+        Mutation mutation = createMutation(key, now);
+
+        // sanity check that there is no data inside yet
+        assertNoPartitions(key, TABLE0);
+        assertNoPartitions(key, TABLE1);
+        assertNoPartitions(key, TABLE2);
+
+        // lower the GC GS on TABLE0 to 0 BEFORE the hint is created
+        Schema.instance.getCFMetaData(KEYSPACE, TABLE0).gcGraceSeconds(0);
+
+        Hint.create(mutation, now / 1000).apply();
+
+        // all updates should have been skipped and not applied, as expired
+        assertNoPartitions(key, TABLE0);
+        assertNoPartitions(key, TABLE1);
+        assertNoPartitions(key, TABLE2);
+    }
+
+    @Test
+    public void testApplyWithGCGSReducedLater()
+    {
+        long now = FBUtilities.timestampMicros();
+        String key = "testApplyWithGCGSReducedLater";
+        Mutation mutation = createMutation(key, now);
+        Hint hint = Hint.create(mutation, now / 1000);
+
+        // sanity check that there is no data inside yet
+        assertNoPartitions(key, TABLE0);
+        assertNoPartitions(key, TABLE1);
+        assertNoPartitions(key, TABLE2);
+
+        // lower the GC GS on TABLE0 AFTER the hint is already created
+        Schema.instance.getCFMetaData(KEYSPACE, TABLE0).gcGraceSeconds(0);
+
+        hint.apply();
+
+        // all updates should have been skipped and not applied, as expired
+        assertNoPartitions(key, TABLE0);
+        assertNoPartitions(key, TABLE1);
+        assertNoPartitions(key, TABLE2);
+    }
+
+    private static Mutation createMutation(String key, long now)
+    {
+        Mutation mutation = new Mutation(KEYSPACE, dk(key));
+
+        new RowUpdateBuilder(Schema.instance.getCFMetaData(KEYSPACE, TABLE0), now, mutation)
+            .clustering("column0")
+            .add("val", "value0")
+            .build();
+
+        new RowUpdateBuilder(Schema.instance.getCFMetaData(KEYSPACE, TABLE1), now + 1, mutation)
+            .clustering("column1")
+            .add("val", "value1")
+            .build();
+
+        new RowUpdateBuilder(Schema.instance.getCFMetaData(KEYSPACE, TABLE2), now + 2, mutation)
+            .clustering("column2")
+            .add("val", "value2")
+            .build();
+
+        return mutation;
+    }
+
+    private static SinglePartitionReadCommand cmd(String key, String table)
+    {
+        CFMetaData meta = Schema.instance.getCFMetaData(KEYSPACE, table);
+        return SinglePartitionReadCommand.fullPartitionRead(meta, FBUtilities.nowInSeconds(), bytes(key));
+    }
+
+    private static FilteredPartition readPartition(String key, String table)
+    {
+        return Util.getOnlyPartition(cmd(key, table));
+    }
+
+    private static void assertNoPartitions(String key, String table)
+    {
+        ReadCommand cmd = cmd(key, table);
+
+        try (ReadOrderGroup orderGroup = cmd.startOrderGroup();
+             PartitionIterator iterator = cmd.executeInternal(orderGroup))
+        {
+            assertFalse(iterator.hasNext());
+        }
+    }
+}

test/unit/org/apache/cassandra/hints/HintsBufferTest.java

@@ -0,0 +1,236 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.hints;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.*;
+import java.util.concurrent.TimeUnit;
+import java.util.zip.CRC32;
+
+import com.google.common.collect.Iterables;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.config.Schema;
+import org.apache.cassandra.db.Mutation;
+import org.apache.cassandra.db.RowUpdateBuilder;
+import org.apache.cassandra.db.rows.Cell;
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.io.util.DataInputBuffer;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.schema.KeyspaceParams;
+
+import static junit.framework.Assert.*;
+
+import static org.apache.cassandra.utils.ByteBufferUtil.bytes;
+import static org.apache.cassandra.utils.FBUtilities.updateChecksum;
+
+public class HintsBufferTest
+{
+    private static final String KEYSPACE = "hints_buffer_test";
+    private static final String TABLE = "table";
+
+    private static final int HINTS_COUNT = 300_000;
+    private static final int HINT_THREADS_COUNT = 10;
+    private static final int HOST_ID_COUNT = 10;
+
+    @BeforeClass
+    public static void defineSchema()
+    {
+        SchemaLoader.prepareServer();
+        SchemaLoader.createKeyspace(KEYSPACE, KeyspaceParams.simple(1), SchemaLoader.standardCFMD(KEYSPACE, TABLE));
+    }
+
+    @Test
+    @SuppressWarnings("resource")
+    public void testOverlyLargeAllocation()
+    {
+        // create a small, 128 bytes buffer
+        HintsBuffer buffer = HintsBuffer.create(128);
+
+        // try allocating an entry of 65 bytes (53 bytes hint + 12 bytes of overhead)
+        try
+        {
+            buffer.allocate(65 - HintsBuffer.ENTRY_OVERHEAD_SIZE);
+            fail("Allocation of the buffer should have failed but hasn't");
+        }
+        catch (IllegalArgumentException e)
+        {
+            assertEquals(String.format("Hint of %s bytes is too large - the maximum size is 64", 65 - HintsBuffer.ENTRY_OVERHEAD_SIZE),
+                         e.getMessage());
+        }
+
+        // assert that a 1-byte smaller allocation fits properly
+        try (HintsBuffer.Allocation allocation = buffer.allocate(64 - HintsBuffer.ENTRY_OVERHEAD_SIZE))
+        {
+            assertNotNull(allocation);
+        }
+    }
+
+    @Test
+    public void testWrite() throws IOException, InterruptedException
+    {
+        // generate 10 random host ids to choose from
+        UUID[] hostIds = new UUID[HOST_ID_COUNT];
+        for (int i = 0; i < hostIds.length; i++)
+            hostIds[i] = UUID.randomUUID();
+
+        // map each index to one random UUID from the previously created UUID array
+        Random random = new Random(System.currentTimeMillis());
+        UUID[] load = new UUID[HINTS_COUNT];
+        for (int i = 0; i < load.length; i++)
+            load[i] = hostIds[random.nextInt(HOST_ID_COUNT)];
+
+        // calculate the size of a single hint (they will all have an equal size in this test)
+        int hintSize = (int) Hint.serializer.serializedSize(createHint(0, System.currentTimeMillis()), MessagingService.current_version);
+        int entrySize = hintSize + HintsBuffer.ENTRY_OVERHEAD_SIZE;
+
+        // allocate a slab to fit *precisely* HINTS_COUNT hints
+        int slabSize = entrySize * HINTS_COUNT;
+        HintsBuffer buffer = HintsBuffer.create(slabSize);
+
+        // use a fixed timestamp base for all mutation timestamps
+        long baseTimestamp = System.currentTimeMillis();
+
+        // create HINT_THREADS_COUNT, start them, and wait for them to finish
+        List<Thread> threads = new ArrayList<>(HINT_THREADS_COUNT);
+        for (int i = 0; i < HINT_THREADS_COUNT; i ++)
+            threads.add(new Thread(new Writer(buffer, load, hintSize, i, baseTimestamp)));
+        threads.forEach(java.lang.Thread::start);
+        for (Thread thread : threads)
+            thread.join();
+
+        // sanity check that we are full
+        assertEquals(slabSize, buffer.capacity());
+        assertEquals(0, buffer.remaining());
+
+        // try to allocate more bytes, ensure that the allocation fails
+        assertNull(buffer.allocate(1));
+
+        // a failed allocation should automatically close the oporder
+        buffer.waitForModifications();
+
+        // a failed allocation should also automatically make the buffer as closed
+        assertTrue(buffer.isClosed());
+
+        // assert that host id set in the buffer equals to hostIds
+        assertEquals(HOST_ID_COUNT, buffer.hostIds().size());
+        assertEquals(new HashSet<>(Arrays.asList(hostIds)), buffer.hostIds());
+
+        // iterate over *every written hint*, validate its content
+        for (UUID hostId : hostIds)
+        {
+            Iterator<ByteBuffer> iter = buffer.consumingHintsIterator(hostId);
+            while (iter.hasNext())
+            {
+                int idx = validateEntry(hostId, iter.next(), baseTimestamp, load);
+                load[idx] = null; // nullify each visited entry
+            }
+        }
+
+        // assert that all the entries in load array have been visited and nullified
+        for (UUID hostId : load)
+            assertNull(hostId);
+
+        // free the buffer
+        buffer.free();
+    }
+
+    private static int validateEntry(UUID hostId, ByteBuffer buffer, long baseTimestamp, UUID[] load) throws IOException
+    {
+        CRC32 crc = new CRC32();
+        DataInputPlus di = new DataInputBuffer(buffer, true);
+
+        // read and validate size
+        int hintSize = di.readInt();
+        assertEquals(hintSize + HintsBuffer.ENTRY_OVERHEAD_SIZE, buffer.remaining());
+
+        // read and validate size crc
+        updateChecksum(crc, buffer, buffer.position(), 4);
+        assertEquals((int) crc.getValue(), di.readInt());
+
+        // read the hint and update/validate overall crc
+        Hint hint = Hint.serializer.deserialize(di, MessagingService.current_version);
+        updateChecksum(crc, buffer, buffer.position() + 8, hintSize);
+        assertEquals((int) crc.getValue(), di.readInt());
+
+        // further validate hint correctness
+        int idx = (int) (hint.creationTime - baseTimestamp);
+        assertEquals(hostId, load[idx]);
+
+        Row row = hint.mutation.getPartitionUpdates().iterator().next().iterator().next();
+        assertEquals(1, Iterables.size(row.cells()));
+
+        assertEquals(bytes(idx), row.clustering().get(0));
+        Cell cell = row.cells().iterator().next();
+        assertEquals(TimeUnit.MILLISECONDS.toMicros(baseTimestamp + idx), cell.timestamp());
+        assertEquals(bytes(idx), cell.value());
+
+        return idx;
+    }
+
+    private static Hint createHint(int idx, long baseTimestamp)
+    {
+        long timestamp = baseTimestamp + idx;
+        return Hint.create(createMutation(idx, TimeUnit.MILLISECONDS.toMicros(timestamp)), timestamp);
+    }
+
+    private static Mutation createMutation(int index, long timestamp)
+    {
+        CFMetaData table = Schema.instance.getCFMetaData(KEYSPACE, TABLE);
+        return new RowUpdateBuilder(table, timestamp, bytes(index))
+                   .clustering(bytes(index))
+                   .add("val", bytes(index))
+                   .build();
+    }
+
+    static class Writer implements Runnable
+    {
+        final HintsBuffer buffer;
+        final UUID[] load;
+        final int hintSize;
+        final int index;
+        final long baseTimestamp;
+
+        Writer(HintsBuffer buffer, UUID[] load, int hintSize, int index, long baseTimestamp)
+        {
+            this.buffer = buffer;
+            this.load = load;
+            this.hintSize = hintSize;
+            this.index = index;
+            this.baseTimestamp = baseTimestamp;
+        }
+
+        public void run()
+        {
+            int hintsPerThread = HINTS_COUNT / HINT_THREADS_COUNT;
+            for (int i = index * hintsPerThread; i < (index + 1) * hintsPerThread; i++)
+            {
+                try (HintsBuffer.Allocation allocation = buffer.allocate(hintSize))
+                {
+                    Hint hint = createHint(i, baseTimestamp);
+                    allocation.write(Collections.singleton(load[i]), hint);
+                }
+            }
+        }
+    }
+}

test/unit/org/apache/cassandra/hints/HintsCatalogTest.java

@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.hints;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.util.*;
+
+import org.junit.Test;
+
+import static junit.framework.Assert.*;
+
+public class HintsCatalogTest
+{
+    @Test
+    public void loadCompletenessAndOrderTest() throws IOException
+    {
+        File directory = Files.createTempDirectory(null).toFile();
+        try
+        {
+            loadCompletenessAndOrderTest(directory);
+        }
+        finally
+        {
+            directory.deleteOnExit();
+        }
+    }
+
+    public static void loadCompletenessAndOrderTest(File directory) throws IOException
+    {
+        UUID hostId1 = UUID.randomUUID();
+        UUID hostId2 = UUID.randomUUID();
+
+        long timestamp1 = System.currentTimeMillis();
+        long timestamp2 = System.currentTimeMillis() + 1;
+        long timestamp3 = System.currentTimeMillis() + 2;
+        long timestamp4 = System.currentTimeMillis() + 3;
+
+        HintsDescriptor descriptor1 = new HintsDescriptor(hostId1, timestamp1);
+        HintsDescriptor descriptor2 = new HintsDescriptor(hostId2, timestamp3);
+        HintsDescriptor descriptor3 = new HintsDescriptor(hostId2, timestamp2);
+        HintsDescriptor descriptor4 = new HintsDescriptor(hostId1, timestamp4);
+
+        writeDescriptor(directory, descriptor1);
+        writeDescriptor(directory, descriptor2);
+        writeDescriptor(directory, descriptor3);
+        writeDescriptor(directory, descriptor4);
+
+        HintsCatalog catalog = HintsCatalog.load(directory);
+        assertEquals(2, catalog.stores().count());
+
+        HintsStore store1 = catalog.get(hostId1);
+        assertNotNull(store1);
+        assertEquals(descriptor1, store1.poll());
+        assertEquals(descriptor4, store1.poll());
+        assertNull(store1.poll());
+
+        HintsStore store2 = catalog.get(hostId2);
+        assertNotNull(store2);
+        assertEquals(descriptor3, store2.poll());
+        assertEquals(descriptor2, store2.poll());
+        assertNull(store2.poll());
+    }
+
+    @SuppressWarnings("EmptyTryBlock")
+    private static void writeDescriptor(File directory, HintsDescriptor descriptor) throws IOException
+    {
+        try (HintsWriter ignored = HintsWriter.create(directory, descriptor))
+        {
+        }
+    }
+}

test/unit/org/apache/cassandra/hints/HintsDescriptorTest.java

@@ -0,0 +1,153 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.hints;
+
+import java.io.DataInput;
+import java.io.File;
+import java.io.IOException;
+import java.util.UUID;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.common.io.ByteStreams;
+import com.google.common.io.Files;
+import org.junit.Test;
+
+import org.apache.cassandra.io.compress.LZ4Compressor;
+import org.apache.cassandra.io.util.DataOutputBuffer;
+
+import static junit.framework.Assert.assertEquals;
+import static junit.framework.Assert.assertNotSame;
+import static junit.framework.Assert.fail;
+
+public class HintsDescriptorTest
+{
+    @Test
+    public void testSerializerNormal() throws IOException
+    {
+        UUID hostId = UUID.randomUUID();
+        int version = HintsDescriptor.CURRENT_VERSION;
+        long timestamp = System.currentTimeMillis();
+        ImmutableMap<String, Object> parameters =
+                ImmutableMap.of("compression", (Object) ImmutableMap.of("class_name", LZ4Compressor.class.getName()));
+        HintsDescriptor descriptor = new HintsDescriptor(hostId, version, timestamp, parameters);
+
+        testSerializeDeserializeLoop(descriptor);
+    }
+
+    @Test
+    public void testSerializerWithEmptyParameters() throws IOException
+    {
+        UUID hostId = UUID.randomUUID();
+        int version = HintsDescriptor.CURRENT_VERSION;
+        long timestamp = System.currentTimeMillis();
+        ImmutableMap<String, Object> parameters = ImmutableMap.of();
+        HintsDescriptor descriptor = new HintsDescriptor(hostId, version, timestamp, parameters);
+
+        testSerializeDeserializeLoop(descriptor);
+    }
+
+    @Test
+    public void testCorruptedDeserialize() throws IOException
+    {
+        UUID hostId = UUID.randomUUID();
+        int version = HintsDescriptor.CURRENT_VERSION;
+        long timestamp = System.currentTimeMillis();
+        ImmutableMap<String, Object> parameters = ImmutableMap.of();
+        HintsDescriptor descriptor = new HintsDescriptor(hostId, version, timestamp, parameters);
+
+        byte[] bytes = serializeDescriptor(descriptor);
+
+        // mess up the parameters size
+        bytes[28] = (byte) 0xFF;
+        bytes[29] = (byte) 0xFF;
+        bytes[30] = (byte) 0xFF;
+        bytes[31] = (byte) 0x7F;
+
+        // attempt to deserialize
+        try
+        {
+            deserializeDescriptor(bytes);
+            fail("Deserializing the descriptor should but didn't");
+        }
+        catch (IOException e)
+        {
+            assertEquals("Hints Descriptor CRC Mismatch", e.getMessage());
+        }
+    }
+
+    @Test
+    @SuppressWarnings("EmptyTryBlock")
+    public void testReadFromFile() throws IOException
+    {
+        UUID hostId = UUID.randomUUID();
+        int version = HintsDescriptor.CURRENT_VERSION;
+        long timestamp = System.currentTimeMillis();
+        ImmutableMap<String, Object> parameters = ImmutableMap.of();
+        HintsDescriptor expected = new HintsDescriptor(hostId, version, timestamp, parameters);
+
+        File directory = Files.createTempDir();
+        try
+        {
+            try (HintsWriter ignored = HintsWriter.create(directory, expected))
+            {
+            }
+            HintsDescriptor actual = HintsDescriptor.readFromFile(new File(directory, expected.fileName()).toPath());
+            assertEquals(expected, actual);
+        }
+        finally
+        {
+            directory.deleteOnExit();
+        }
+    }
+
+    private static void testSerializeDeserializeLoop(HintsDescriptor descriptor) throws IOException
+    {
+        // serialize to a byte array
+        byte[] bytes = serializeDescriptor(descriptor);
+        // make sure the sizes match
+        assertEquals(bytes.length, descriptor.serializedSize());
+        // deserialize back
+        HintsDescriptor deserializedDescriptor = deserializeDescriptor(bytes);
+        // compare equality
+        assertDescriptorsEqual(descriptor, deserializedDescriptor);
+    }
+
+    private static byte[] serializeDescriptor(HintsDescriptor descriptor) throws IOException
+    {
+        DataOutputBuffer dob = new DataOutputBuffer();
+        descriptor.serialize(dob);
+        return dob.toByteArray();
+    }
+
+    private static HintsDescriptor deserializeDescriptor(byte[] bytes) throws IOException
+    {
+        DataInput in = ByteStreams.newDataInput(bytes);
+        return HintsDescriptor.deserialize(in);
+    }
+
+    private static void assertDescriptorsEqual(HintsDescriptor expected, HintsDescriptor actual)
+    {
+        assertNotSame(expected, actual);
+        assertEquals(expected, actual);
+        assertEquals(expected.hashCode(), actual.hashCode());
+        assertEquals(expected.hostId, actual.hostId);
+        assertEquals(expected.version, actual.version);
+        assertEquals(expected.timestamp, actual.timestamp);
+        assertEquals(expected.parameters, actual.parameters);
+    }
+}

test/unit/org/apache/cassandra/hints/HintsTestUtil.java

@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.hints;
+
+import java.util.UUID;
+
+import com.google.common.collect.Iterators;
+
+import org.apache.cassandra.db.Mutation;
+import org.apache.cassandra.db.partitions.AbstractBTreePartition;
+import org.apache.cassandra.db.partitions.PartitionUpdate;
+
+import static junit.framework.Assert.assertEquals;
+import static junit.framework.Assert.assertTrue;
+
+final class HintsTestUtil
+{
+    static void assertMutationsEqual(Mutation expected, Mutation actual)
+    {
+        assertEquals(expected.key(), actual.key());
+        assertEquals(expected.getPartitionUpdates().size(), actual.getPartitionUpdates().size());
+
+        for (UUID id : expected.getColumnFamilyIds())
+            assertPartitionsEqual(expected.getPartitionUpdate(id), actual.getPartitionUpdate(id));
+    }
+
+    static void assertPartitionsEqual(AbstractBTreePartition expected, AbstractBTreePartition actual)
+    {
+        assertEquals(expected.partitionKey(), actual.partitionKey());
+        assertEquals(expected.deletionInfo(), actual.deletionInfo());
+        assertEquals(expected.columns(), actual.columns());
+        assertTrue(Iterators.elementsEqual(expected.iterator(), actual.iterator()));
+    }
+
+    static void assertHintsEqual(Hint expected, Hint actual)
+    {
+        assertEquals(expected.mutation.getKeyspaceName(), actual.mutation.getKeyspaceName());
+        assertEquals(expected.mutation.key(), actual.mutation.key());
+        assertEquals(expected.mutation.getColumnFamilyIds(), actual.mutation.getColumnFamilyIds());
+        for (PartitionUpdate partitionUpdate : expected.mutation.getPartitionUpdates())
+            assertPartitionsEqual(partitionUpdate, actual.mutation.getPartitionUpdate(partitionUpdate.metadata().cfId));
+        assertEquals(expected.creationTime, actual.creationTime);
+        assertEquals(expected.gcgs, actual.gcgs);
+    }
+}

test/unit/org/apache/cassandra/hints/LegacyHintsMigratorTest.java

@@ -0,0 +1,195 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.hints;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.file.Files;
+import java.util.*;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.config.Schema;
+import org.apache.cassandra.db.*;
+import org.apache.cassandra.db.marshal.UUIDType;
+import org.apache.cassandra.db.partitions.PartitionUpdate;
+import org.apache.cassandra.db.rows.BTreeRow;
+import org.apache.cassandra.db.rows.BufferCell;
+import org.apache.cassandra.db.rows.Cell;
+import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.UUIDGen;
+
+import static junit.framework.Assert.assertEquals;
+import static junit.framework.Assert.assertNotNull;
+import static junit.framework.Assert.assertTrue;
+
+import static org.apache.cassandra.hints.HintsTestUtil.assertMutationsEqual;
+import static org.apache.cassandra.utils.ByteBufferUtil.bytes;
+
+// TODO: test split into several files
+@SuppressWarnings("deprecation")
+public class LegacyHintsMigratorTest
+{
+    private static final String KEYSPACE = "legacy_hints_migrator_test";
+    private static final String TABLE = "table";
+
+    @BeforeClass
+    public static void defineSchema()
+    {
+        SchemaLoader.prepareServer();
+        SchemaLoader.createKeyspace(KEYSPACE, KeyspaceParams.simple(1), SchemaLoader.standardCFMD(KEYSPACE, TABLE));
+    }
+
+    @Test
+    public void testNothingToMigrate() throws IOException
+    {
+        File directory = Files.createTempDirectory(null).toFile();
+        try
+        {
+            testNothingToMigrate(directory);
+        }
+        finally
+        {
+            directory.deleteOnExit();
+        }
+    }
+
+    private static void testNothingToMigrate(File directory)
+    {
+        // truncate system.hints to enseure nothing inside
+        Keyspace.open(SystemKeyspace.NAME).getColumnFamilyStore(SystemKeyspace.LEGACY_HINTS).truncateBlocking();
+        new LegacyHintsMigrator(directory, 128 * 1024 * 1024).migrate();
+        HintsCatalog catalog = HintsCatalog.load(directory);
+        assertEquals(0, catalog.stores().count());
+    }
+
+    @Test
+    public void testMigrationIsComplete() throws IOException
+    {
+        File directory = Files.createTempDirectory(null).toFile();
+        try
+        {
+            testMigrationIsComplete(directory);
+        }
+        finally
+        {
+            directory.deleteOnExit();
+        }
+    }
+
+    private static void testMigrationIsComplete(File directory)
+    {
+        long timestamp = System.currentTimeMillis();
+
+        // write 100 mutations for each of the 10 generated endpoints
+        Map<UUID, Queue<Mutation>> mutations = new HashMap<>();
+        for (int i = 0; i < 10; i++)
+        {
+            UUID hostId = UUID.randomUUID();
+            Queue<Mutation> queue = new LinkedList<>();
+            mutations.put(hostId, queue);
+
+            for (int j = 0; j < 100; j++)
+            {
+                Mutation mutation = createMutation(j, timestamp + j);
+                queue.offer(mutation);
+                Mutation legacyHint = createLegacyHint(mutation, timestamp, hostId);
+                legacyHint.applyUnsafe();
+            }
+        }
+
+        // run the migration
+        new LegacyHintsMigrator(directory, 128 * 1024 * 1024).migrate();
+
+        // validate that the hints table is truncated now
+        assertTrue(Keyspace.open(SystemKeyspace.NAME).getColumnFamilyStore(SystemKeyspace.LEGACY_HINTS).isEmpty());
+
+        HintsCatalog catalog = HintsCatalog.load(directory);
+
+        // assert that we've correctly loaded 10 hints stores
+        assertEquals(10, catalog.stores().count());
+
+        // for each of the 10 stores, make sure the mutations have been migrated correctly
+        for (Map.Entry<UUID, Queue<Mutation>> entry : mutations.entrySet())
+        {
+            HintsStore store = catalog.get(entry.getKey());
+            assertNotNull(store);
+
+            HintsDescriptor descriptor = store.poll();
+            assertNotNull(descriptor);
+
+            // read all the hints
+            Queue<Hint> actualHints = new LinkedList<>();
+            try (HintsReader reader = HintsReader.open(new File(directory, descriptor.fileName())))
+            {
+                for (HintsReader.Page page : reader)
+                    page.hintsIterator().forEachRemaining(actualHints::offer);
+            }
+
+            // assert the size matches
+            assertEquals(100, actualHints.size());
+
+            // compare expected hints to actual hints
+            for (int i = 0; i < 100; i++)
+            {
+                Hint hint = actualHints.poll();
+                Mutation mutation = entry.getValue().poll();
+                int ttl = mutation.smallestGCGS();
+
+                assertEquals(timestamp, hint.creationTime);
+                assertEquals(ttl, hint.gcgs);
+                assertMutationsEqual(mutation, hint.mutation);
+            }
+        }
+    }
+
+    // legacy hint mutation creation code, copied more or less verbatim from the previous implementation
+    private static Mutation createLegacyHint(Mutation mutation, long now, UUID targetId)
+    {
+        int version = MessagingService.VERSION_21;
+        int ttl = mutation.smallestGCGS();
+        UUID hintId = UUIDGen.getTimeUUID();
+
+        ByteBuffer key = UUIDType.instance.decompose(targetId);
+        Clustering clustering = SystemKeyspace.LegacyHints.comparator.make(hintId, version);
+        ByteBuffer value = ByteBuffer.wrap(FBUtilities.serialize(mutation, Mutation.serializer, version));
+        Cell cell = BufferCell.expiring(SystemKeyspace.LegacyHints.compactValueColumn(),
+                                        now,
+                                        ttl,
+                                        FBUtilities.nowInSeconds(),
+                                        value);
+        return new Mutation(PartitionUpdate.singleRowUpdate(SystemKeyspace.LegacyHints,
+                                                            key,
+                                                            BTreeRow.singleCellRow(clustering, cell)));
+    }
+
+    private static Mutation createMutation(int index, long timestamp)
+    {
+        CFMetaData table = Schema.instance.getCFMetaData(KEYSPACE, TABLE);
+        return new RowUpdateBuilder(table, timestamp, bytes(index))
+               .clustering(bytes(index))
+               .add("val", bytes(index))
+               .build();
+    }
+}

test/unit/org/apache/cassandra/metrics/HintedHandOffMetricsTest.java

@@ -0,0 +1,56 @@
+/*
+ * 
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ * 
+ */
+package org.apache.cassandra.metrics;
+
+import java.net.InetAddress;
+import java.util.Map;
+import java.util.UUID;
+
+import org.junit.Test;
+
+import com.google.common.collect.Iterators;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.cql3.UntypedResultSet;
+import org.apache.cassandra.db.SystemKeyspace;
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.db.marshal.UUIDType;
+import org.apache.cassandra.hints.HintsService;
+
+import static org.junit.Assert.assertEquals;
+import static org.apache.cassandra.cql3.QueryProcessor.executeInternal;
+
+public class HintedHandOffMetricsTest
+{
+    @Test
+    public void testHintsMetrics() throws Exception
+    {
+        DatabaseDescriptor.getHintsDirectory().mkdirs();
+
+        for (int i = 0; i < 99; i++)
+            HintsService.instance.metrics.incrPastWindow(InetAddress.getLocalHost());
+        HintsService.instance.metrics.log();
+
+        UntypedResultSet rows = executeInternal("SELECT hints_dropped FROM system." + SystemKeyspace.PEER_EVENTS);
+        Map<UUID, Integer> returned = rows.one().getMap("hints_dropped", UUIDType.instance, Int32Type.instance);
+        assertEquals(Iterators.getLast(returned.values().iterator()).intValue(), 99);
+    }
+}

test/unit/org/apache/cassandra/service/StorageProxyTest.java

@@ -25,9 +25,9 @@ import org.junit.BeforeClass;
 import org.junit.Test;
 
 import org.apache.cassandra.db.PartitionPosition;
+import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.dht.*;
 import org.apache.cassandra.locator.TokenMetadata;
-import org.apache.cassandra.utils.ByteBufferUtil;
 
 import static org.apache.cassandra.Util.rp;
 import static org.apache.cassandra.Util.token;
@@ -78,6 +78,7 @@ public class StorageProxyTest
     @BeforeClass
     public static void beforeClass() throws Throwable
     {
+        DatabaseDescriptor.getHintsDirectory().mkdir();
         TokenMetadata tmd = StorageService.instance.getTokenMetadata();
         tmd.updateNormalToken(token("1"), InetAddress.getByName("127.0.0.1"));
         tmd.updateNormalToken(token("6"), InetAddress.getByName("127.0.0.6"));