diff --git a/CHANGES.txt b/CHANGES.txt index 54a6a07737..0d172352f8 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ 3.0.0-beta1 + * Rewrite hinted handoff (CASSANDRA-6230) * Fix query on static compact tables (CASSANDRA-10093) * Fix race during construction of commit log (CASSANDRA-10049) * Add option to only purge repaired tombstones (CASSANDRA-6434) diff --git a/NEWS.txt b/NEWS.txt index 365ed31538..61d3180732 100644 --- a/NEWS.txt +++ b/NEWS.txt @@ -23,6 +23,9 @@ New features for non-primary key queries, and perform much better for indexing high cardinality columns. See http://www.datastax.com/dev/blog/new-in-cassandra-3-0-materialized-views + - Hinted handoff has been completely rewritten. Hints are now stored in flat + files, with less overhead for storage and more efficient dispatch. + See CASSANDRA-6230 for full details. - Option to not purge unrepaired tombstones. To avoid users having data resurrected if repair has not been run within gc_grace_seconds, an option has been added to only allow tombstones from repaired sstables to be purged. To enable, set the @@ -30,10 +33,11 @@ New features you do not run repair for a long time, you will keep all tombstones around which can cause other problems. - Upgrading --------- - - 3.0 requires Java 8u20 or later. + - Max mutation size is now configurable via max_mutation_size_in_kb setting in + cassandra.yaml; the default is half the size commitlog_segment_size_in_mb * 1024. + - 3.0 requires Java 8u40 or later. - The default JVM GC has been changed to G1GC. - The default JVM flag -XX:+PerfDisableSharedMem will cause the following tools JVM to stop working: jps, jstack, jinfo, jmc, jcmd as well as 3rd party tools like Jolokia. @@ -91,6 +95,7 @@ Upgrading set/getCompactionParameters or set/getCompactionParametersJson instead. - SizeTieredCompactionStrategy parameter cold_reads_to_omit has been removed. + 2.2 === diff --git a/conf/cassandra.yaml b/conf/cassandra.yaml index 67c37bc598..58a343af65 100644 --- a/conf/cassandra.yaml +++ b/conf/cassandra.yaml @@ -53,17 +53,30 @@ hinted_handoff_enabled: true # generated. After it has been dead this long, new hints for it will not be # created until it has been seen alive and gone down again. max_hint_window_in_ms: 10800000 # 3 hours + # Maximum throttle in KBs per second, per delivery thread. This will be # reduced proportionally to the number of nodes in the cluster. (If there # are two nodes in the cluster, each delivery thread will use the maximum # rate; if there are three, each will throttle to half of the maximum, # since we expect two nodes to be delivering hints simultaneously.) hinted_handoff_throttle_in_kb: 1024 + # Number of threads with which to deliver hints; # Consider increasing this number when you have multi-dc deployments, since # cross-dc handoff tends to be slower max_hints_delivery_threads: 2 +# Directory where Cassandra should store hints. +# If not set, the default directory is $CASSANDRA_HOME/data/hints. +# hints_directory: /var/lib/cassandra/hints + +# How often hints should be flushed from the internal buffers to disk. +# Will *not* trigger fsync. +hints_flush_period_in_ms: 10000 + +# Maximum size for a single hints file, in megabytes. +max_hints_file_size_in_mb: 128 + # Maximum throttle in KBs per second, total. This will be # reduced proportionally to the number of nodes in the cluster. batchlog_replay_throttle_in_kb: 1024 diff --git a/src/java/org/apache/cassandra/config/Config.java b/src/java/org/apache/cassandra/config/Config.java index e93d09089d..762935dc74 100644 --- a/src/java/org/apache/cassandra/config/Config.java +++ b/src/java/org/apache/cassandra/config/Config.java @@ -56,6 +56,7 @@ public class Config public volatile boolean hinted_handoff_enabled = true; public Set hinted_handoff_disabled_datacenters = Sets.newConcurrentHashSet(); public volatile Integer max_hint_window_in_ms = 3 * 3600 * 1000; // three hours + public String hints_directory; public ParameterizedClass seed_provider; public DiskAccessMode disk_access_mode = DiskAccessMode.auto; @@ -169,7 +170,9 @@ public class Config public int commitlog_segment_size_in_mb = 32; public ParameterizedClass commitlog_compression; public int commitlog_max_compression_buffers_in_pool = 3; - + + public Integer max_mutation_size_in_kb; + @Deprecated public int commitlog_periodic_queue_size = -1; @@ -195,7 +198,9 @@ public class Config public int hinted_handoff_throttle_in_kb = 1024; public int batchlog_replay_throttle_in_kb = 1024; - public int max_hints_delivery_threads = 1; + public int max_hints_delivery_threads = 2; + public int hints_flush_period_in_ms = 10000; + public int max_hints_file_size_in_mb = 128; public int sstable_preemptive_open_interval_in_mb = 50; public volatile boolean incremental_backups = false; diff --git a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java index 926678525b..b3bc4d2b2c 100644 --- a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java +++ b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java @@ -482,6 +482,15 @@ public class DatabaseDescriptor throw new ConfigurationException("commitlog_directory is missing and -Dcassandra.storagedir is not set", false); conf.commitlog_directory += File.separator + "commitlog"; } + + if (conf.hints_directory == null) + { + conf.hints_directory = System.getProperty("cassandra.storagedir", null); + if (conf.hints_directory == null) + throw new ConfigurationException("hints_directory is missing and -Dcassandra.storagedir is not set", false); + conf.hints_directory += File.separator + "hints"; + } + if (conf.saved_caches_directory == null) { conf.saved_caches_directory = System.getProperty("cassandra.storagedir", null); @@ -502,12 +511,18 @@ public class DatabaseDescriptor { if (datadir.equals(conf.commitlog_directory)) throw new ConfigurationException("commitlog_directory must not be the same as any data_file_directories", false); + if (datadir.equals(conf.hints_directory)) + throw new ConfigurationException("hints_directory must not be the same as any data_file_directories", false); if (datadir.equals(conf.saved_caches_directory)) throw new ConfigurationException("saved_caches_directory must not be the same as any data_file_directories", false); } if (conf.commitlog_directory.equals(conf.saved_caches_directory)) throw new ConfigurationException("saved_caches_directory must not be the same as the commitlog_directory", false); + if (conf.commitlog_directory.equals(conf.hints_directory)) + throw new ConfigurationException("hints_directory must not be the same as the commitlog_directory", false); + if (conf.hints_directory.equals(conf.saved_caches_directory)) + throw new ConfigurationException("saved_caches_directory must not be the same as the hints_directory", false); if (conf.memtable_flush_writers == null) conf.memtable_flush_writers = Math.min(8, Math.max(2, Math.min(FBUtilities.getAvailableProcessors(), conf.data_file_directories.length))); @@ -613,6 +628,11 @@ public class DatabaseDescriptor if (conf.user_defined_function_fail_timeout < conf.user_defined_function_warn_timeout) throw new ConfigurationException("user_defined_function_warn_timeout must less than user_defined_function_fail_timeout", false); + + if (conf.max_mutation_size_in_kb == null) + conf.max_mutation_size_in_kb = conf.commitlog_segment_size_in_mb * 1024 / 2; + else if (conf.commitlog_segment_size_in_mb * 1024 < 2 * conf.max_mutation_size_in_kb) + throw new ConfigurationException("commitlog_segment_size_in_mb must be at least twice the size of max_mutation_size_in_kb / 1024", false); } private static IEndpointSnitch createEndpointSnitch(String snitchClassName) throws ConfigurationException @@ -708,18 +728,18 @@ public class DatabaseDescriptor throw new ConfigurationException("At least one DataFileDirectory must be specified", false); for (String dataFileDirectory : conf.data_file_directories) - { FileUtils.createDirectory(dataFileDirectory); - } if (conf.commitlog_directory == null) throw new ConfigurationException("commitlog_directory must be specified", false); - FileUtils.createDirectory(conf.commitlog_directory); + if (conf.hints_directory == null) + throw new ConfigurationException("hints_directory must be specified", false); + FileUtils.createDirectory(conf.hints_directory); + if (conf.saved_caches_directory == null) throw new ConfigurationException("saved_caches_directory must be specified", false); - FileUtils.createDirectory(conf.saved_caches_directory); } catch (ConfigurationException e) @@ -992,6 +1012,7 @@ public class DatabaseDescriptor case PAXOS_PREPARE: case PAXOS_PROPOSE: case BATCHLOG_MUTATION: + case HINT: return getWriteRpcTimeout(); case COUNTER_MUTATION: return getCounterWriteRpcTimeout(); @@ -1119,6 +1140,11 @@ public class DatabaseDescriptor return conf.commitlog_max_compression_buffers_in_pool; } + public static int getMaxMutationSize() + { + return conf.max_mutation_size_in_kb * 1024; + } + public static int getTombstoneWarnThreshold() { return conf.tombstone_warn_threshold; @@ -1415,6 +1441,11 @@ public class DatabaseDescriptor return conf.max_hint_window_in_ms; } + public static File getHintsDirectory() + { + return new File(conf.hints_directory); + } + public static File getSerializedCachePath(String ksName, String cfName, UUID cfId, @@ -1484,11 +1515,21 @@ public class DatabaseDescriptor conf.hinted_handoff_throttle_in_kb = throttleInKB; } - public static int getMaxHintsThread() + public static int getMaxHintsDeliveryThreads() { return conf.max_hints_delivery_threads; } + public static int getHintsFlushPeriodInMS() + { + return conf.hints_flush_period_in_ms; + } + + public static long getMaxHintsFileSize() + { + return conf.max_hints_file_size_in_mb * 1024 * 1024; + } + public static boolean isIncrementalBackupsEnabled() { return conf.incremental_backups; diff --git a/src/java/org/apache/cassandra/db/BatchlogManager.java b/src/java/org/apache/cassandra/db/BatchlogManager.java index 8ea4318419..de859252d1 100644 --- a/src/java/org/apache/cassandra/db/BatchlogManager.java +++ b/src/java/org/apache/cassandra/db/BatchlogManager.java @@ -41,18 +41,20 @@ import org.apache.cassandra.dht.Token; import org.apache.cassandra.exceptions.WriteFailureException; import org.apache.cassandra.exceptions.WriteTimeoutException; import org.apache.cassandra.gms.FailureDetector; +import org.apache.cassandra.hints.Hint; +import org.apache.cassandra.hints.HintsService; import org.apache.cassandra.io.util.DataInputBuffer; import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputBuffer; import org.apache.cassandra.net.MessageIn; import org.apache.cassandra.net.MessageOut; import org.apache.cassandra.net.MessagingService; -import org.apache.cassandra.service.StorageProxy; import org.apache.cassandra.service.StorageService; import org.apache.cassandra.service.WriteResponseHandler; import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.UUIDGen; +import static com.google.common.collect.Iterables.transform; import static org.apache.cassandra.cql3.QueryProcessor.executeInternal; import static org.apache.cassandra.cql3.QueryProcessor.executeInternalWithPaging; @@ -210,6 +212,9 @@ public class BatchlogManager implements BatchlogManagerMBean int positionInPage = 0; ArrayList unfinishedBatches = new ArrayList<>(pageSize); + Set hintedNodes = new HashSet<>(); + Set replayedBatches = new HashSet<>(); + // Sending out batches for replay without waiting for them, so that one stuck batch doesn't affect others for (UntypedResultSet.Row row : batches) { @@ -218,7 +223,7 @@ public class BatchlogManager implements BatchlogManagerMBean Batch batch = new Batch(id, row.getBytes("data"), version); try { - if (batch.replay(rateLimiter) > 0) + if (batch.replay(rateLimiter, hintedNodes) > 0) { unfinishedBatches.add(batch); } @@ -238,21 +243,29 @@ public class BatchlogManager implements BatchlogManagerMBean { // We have reached the end of a batch. To avoid keeping more than a page of mutations in memory, // finish processing the page before requesting the next row. - finishAndClearBatches(unfinishedBatches); + finishAndClearBatches(unfinishedBatches, hintedNodes, replayedBatches); positionInPage = 0; } } - finishAndClearBatches(unfinishedBatches); + + finishAndClearBatches(unfinishedBatches, hintedNodes, replayedBatches); + + // to preserve batch guarantees, we must ensure that hints (if any) have made it to disk, before deleting the batches + HintsService.instance.flushAndFsyncBlockingly(transform(hintedNodes, StorageService.instance::getHostIdForEndpoint)); + + // once all generated hints are fsynced, actually delete the batches + replayedBatches.forEach(BatchlogManager::deleteBatch); } - private void finishAndClearBatches(ArrayList batches) + private void finishAndClearBatches(ArrayList batches, Set hintedNodes, Set replayedBatches) { // schedule hints for timed out deliveries for (Batch batch : batches) { - batch.finish(); - deleteBatch(batch.id); + batch.finish(hintedNodes); + replayedBatches.add(batch.id); } + totalBatchesReplayed += batches.size(); batches.clear(); } @@ -279,7 +292,7 @@ public class BatchlogManager implements BatchlogManagerMBean this.version = version; } - public int replay(RateLimiter rateLimiter) throws IOException + public int replay(RateLimiter rateLimiter, Set hintedNodes) throws IOException { logger.debug("Replaying batch {}", id); @@ -288,18 +301,18 @@ public class BatchlogManager implements BatchlogManagerMBean if (mutations.isEmpty()) return 0; - int ttl = calculateHintTTL(mutations); - if (ttl <= 0) + int gcgs = gcgs(mutations); + if (TimeUnit.MILLISECONDS.toSeconds(writtenAt) + gcgs <= FBUtilities.nowInSeconds()) return 0; - replayHandlers = sendReplays(mutations, writtenAt, ttl); + replayHandlers = sendReplays(mutations, writtenAt, hintedNodes); rateLimiter.acquire(data.remaining()); // acquire afterwards, to not mess up ttl calculation. return replayHandlers.size(); } - public void finish() + public void finish(Set hintedNodes) { for (int i = 0; i < replayHandlers.size(); i++) { @@ -313,7 +326,7 @@ public class BatchlogManager implements BatchlogManagerMBean logger.debug("Failed replaying a batched mutation to a node, will write a hint"); logger.debug("Failure was : {}", e.getMessage()); // writing hints for the rest to hints, starting from i - writeHintsForUndeliveredEndpoints(i); + writeHintsForUndeliveredEndpoints(i, hintedNodes); return; } } @@ -341,7 +354,7 @@ public class BatchlogManager implements BatchlogManagerMBean return mutations; } - private void writeHintsForUndeliveredEndpoints(int startFrom) + private void writeHintsForUndeliveredEndpoints(int startFrom, Set hintedNodes) { try { @@ -353,12 +366,15 @@ public class BatchlogManager implements BatchlogManagerMBean for (int i = startFrom; i < replayHandlers.size(); i++) { Mutation undeliveredMutation = replayingMutations.get(i); - int ttl = calculateHintTTL(replayingMutations); + int gcgs = gcgs(replayingMutations); ReplayWriteResponseHandler handler = replayHandlers.get(i); - if (ttl > 0 && handler != null) - for (InetAddress endpoint : handler.undelivered) - StorageProxy.writeHintForMutation(undeliveredMutation, writtenAt, ttl, endpoint); + if (TimeUnit.MILLISECONDS.toSeconds(writtenAt) + gcgs > FBUtilities.nowInSeconds() && handler != null) + { + hintedNodes.addAll(handler.undelivered); + HintsService.instance.write(transform(handler.undelivered, StorageService.instance::getHostIdForEndpoint), + Hint.create(undeliveredMutation, writtenAt)); + } } } catch (IOException e) @@ -367,12 +383,14 @@ public class BatchlogManager implements BatchlogManagerMBean } } - private static List> sendReplays(List mutations, long writtenAt, int ttl) + private static List> sendReplays(List mutations, + long writtenAt, + Set hintedNodes) { List> handlers = new ArrayList<>(mutations.size()); for (Mutation mutation : mutations) { - ReplayWriteResponseHandler handler = sendSingleReplayMutation(mutation, writtenAt, ttl); + ReplayWriteResponseHandler handler = sendSingleReplayMutation(mutation, writtenAt, hintedNodes); if (handler != null) handlers.add(handler); } @@ -385,7 +403,9 @@ public class BatchlogManager implements BatchlogManagerMBean * * @return direct delivery handler to wait on or null, if no live nodes found */ - private static ReplayWriteResponseHandler sendSingleReplayMutation(final Mutation mutation, long writtenAt, int ttl) + private static ReplayWriteResponseHandler sendSingleReplayMutation(final Mutation mutation, + long writtenAt, + Set hintedNodes) { Set liveEndpoints = new HashSet<>(); String ks = mutation.getKeyspaceName(); @@ -395,11 +415,19 @@ public class BatchlogManager implements BatchlogManagerMBean StorageService.instance.getTokenMetadata().pendingEndpointsFor(tk, ks))) { if (endpoint.equals(FBUtilities.getBroadcastAddress())) + { mutation.apply(); + } else if (FailureDetector.instance.isAlive(endpoint)) + { liveEndpoints.add(endpoint); // will try delivering directly instead of writing a hint. + } else - StorageProxy.writeHintForMutation(mutation, writtenAt, ttl, endpoint); + { + hintedNodes.add(endpoint); + HintsService.instance.write(StorageService.instance.getHostIdForEndpoint(endpoint), + Hint.create(mutation, writtenAt)); + } } if (liveEndpoints.isEmpty()) @@ -412,16 +440,12 @@ public class BatchlogManager implements BatchlogManagerMBean return handler; } - /* - * Calculate ttl for the mutations' hints (and reduce ttl by the time the mutations spent in the batchlog). - * This ensures that deletes aren't "undone" by an old batch replay. - */ - private int calculateHintTTL(Collection mutations) + private static int gcgs(Collection mutations) { - int unadjustedTTL = Integer.MAX_VALUE; + int gcgs = Integer.MAX_VALUE; for (Mutation mutation : mutations) - unadjustedTTL = Math.min(unadjustedTTL, HintedHandOffManager.calculateHintTTL(mutation)); - return unadjustedTTL - (int) TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis() - writtenAt); + gcgs = Math.min(gcgs, mutation.smallestGCGS()); + return gcgs; } /** diff --git a/src/java/org/apache/cassandra/db/HintedHandOffManager.java b/src/java/org/apache/cassandra/db/HintedHandOffManager.java index 8bea2e80b3..3279acf88c 100644 --- a/src/java/org/apache/cassandra/db/HintedHandOffManager.java +++ b/src/java/org/apache/cassandra/db/HintedHandOffManager.java @@ -17,156 +17,32 @@ */ package org.apache.cassandra.db; -import java.io.IOException; import java.lang.management.ManagementFactory; -import java.net.InetAddress; -import java.net.UnknownHostException; -import java.nio.ByteBuffer; -import java.util.*; -import java.util.concurrent.*; -import java.util.concurrent.atomic.AtomicInteger; +import java.util.List; import javax.management.MBeanServer; import javax.management.ObjectName; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.Lists; -import com.google.common.util.concurrent.RateLimiter; -import com.google.common.util.concurrent.Uninterruptibles; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import org.apache.cassandra.concurrent.DebuggableScheduledThreadPoolExecutor; -import org.apache.cassandra.concurrent.JMXEnabledThreadPoolExecutor; -import org.apache.cassandra.concurrent.NamedThreadFactory; -import org.apache.cassandra.config.ColumnDefinition; -import org.apache.cassandra.config.DatabaseDescriptor; -import org.apache.cassandra.db.compaction.CompactionManager; - -import org.apache.cassandra.db.rows.*; -import org.apache.cassandra.db.partitions.*; -import org.apache.cassandra.db.filter.*; -import org.apache.cassandra.db.marshal.Int32Type; -import org.apache.cassandra.db.marshal.UUIDType; -import org.apache.cassandra.exceptions.WriteTimeoutException; -import org.apache.cassandra.gms.ApplicationState; -import org.apache.cassandra.gms.FailureDetector; -import org.apache.cassandra.gms.Gossiper; -import org.apache.cassandra.io.sstable.Descriptor; -import org.apache.cassandra.io.sstable.SSTable; -import org.apache.cassandra.io.util.DataInputBuffer; -import org.apache.cassandra.io.util.DataInputPlus; -import org.apache.cassandra.metrics.HintedHandoffMetrics; -import org.apache.cassandra.net.MessageOut; -import org.apache.cassandra.net.MessagingService; -import org.apache.cassandra.service.StorageProxy; -import org.apache.cassandra.service.StorageService; -import org.apache.cassandra.service.WriteResponseHandler; -import org.apache.cassandra.utils.*; -import org.apache.cassandra.utils.concurrent.OpOrder; -import org.cliffc.high_scale_lib.NonBlockingHashSet; +import org.apache.cassandra.hints.HintsService; /** - * The hint schema looks like this: + * A proxy class that implement the deprecated legacy HintedHandoffManagerMBean interface. * - * CREATE TABLE hints ( - * target_id uuid, - * hint_id timeuuid, - * message_version int, - * mutation blob, - * PRIMARY KEY (target_id, hint_id, message_version) - * ) WITH COMPACT STORAGE; - * - * Thus, for each node in the cluster we treat its uuid as the partition key; each hint is a logical row - * (physical composite column) containing the mutation to replay and associated metadata. - * - * When FailureDetector signals that a node that was down is back up, we page through - * the hinted mutations and send them over one at a time, waiting for - * hinted_handoff_throttle_delay in between each. - * - * deliverHints is also exposed to JMX so it can be run manually if FD ever misses - * its cue somehow. + * TODO: remove in 4.0. */ - -public class HintedHandOffManager implements HintedHandOffManagerMBean +@SuppressWarnings("deprecation") +@Deprecated +public final class HintedHandOffManager implements HintedHandOffManagerMBean { - public static final String MBEAN_NAME = "org.apache.cassandra.db:type=HintedHandoffManager"; public static final HintedHandOffManager instance = new HintedHandOffManager(); - private static final Logger logger = LoggerFactory.getLogger(HintedHandOffManager.class); + public static final String MBEAN_NAME = "org.apache.cassandra.db:type=HintedHandoffManager"; - private static final int MAX_SIMULTANEOUSLY_REPLAYED_HINTS = 128; - private static final int LARGE_NUMBER = 65536; // 64k nodes ought to be enough for anybody. - - public final HintedHandoffMetrics metrics = new HintedHandoffMetrics(); - - private volatile boolean hintedHandOffPaused = false; - - static final int maxHintTTL = Integer.parseInt(System.getProperty("cassandra.maxHintTTL", String.valueOf(Integer.MAX_VALUE))); - - private final NonBlockingHashSet queuedDeliveries = new NonBlockingHashSet<>(); - - // To keep metrics consistent with earlier versions, where periodic tasks were run on a shared executor, - // we run them on this executor and so keep counts separate from those for hint delivery tasks. See CASSANDRA-9129 - private final DebuggableScheduledThreadPoolExecutor executor = - new DebuggableScheduledThreadPoolExecutor(1, new NamedThreadFactory("HintedHandoffManager", Thread.MIN_PRIORITY)); - - // Non-scheduled executor to run the actual hint delivery tasks. - // Per CASSANDRA-9129, this is where the values displayed in nodetool tpstats - // and via the HintedHandoff mbean are obtained. - private final ThreadPoolExecutor hintDeliveryExecutor = - new JMXEnabledThreadPoolExecutor( - DatabaseDescriptor.getMaxHintsThread(), - Integer.MAX_VALUE, - TimeUnit.SECONDS, - new LinkedBlockingQueue(), - new NamedThreadFactory("HintedHandoff", Thread.MIN_PRIORITY), - "internal"); - - private final ColumnFamilyStore hintStore = Keyspace.open(SystemKeyspace.NAME).getColumnFamilyStore(SystemKeyspace.HINTS); - - private static final ColumnDefinition hintColumn = SystemKeyspace.Hints.compactValueColumn(); - - /** - * Returns a mutation representing a Hint to be sent to targetId - * as soon as it becomes available again. - */ - public Mutation hintFor(Mutation mutation, long now, int ttl, UUID targetId) + private HintedHandOffManager() { - assert ttl > 0; - - InetAddress endpoint = StorageService.instance.getTokenMetadata().getEndpointForHostId(targetId); - // during tests we may not have a matching endpoint, but this would be unexpected in real clusters - if (endpoint != null) - metrics.incrCreatedHints(endpoint); - else - logger.warn("Unable to find matching endpoint for target {} when storing a hint", targetId); - - UUID hintId = UUIDGen.getTimeUUID(); - // serialize the hint with id and version as a composite column name - - ByteBuffer key = UUIDType.instance.decompose(targetId); - Clustering clustering = SystemKeyspace.Hints.comparator.make(hintId, MessagingService.current_version); - ByteBuffer value = ByteBuffer.wrap(FBUtilities.serialize(mutation, Mutation.serializer, MessagingService.current_version)); - Cell cell = BufferCell.expiring(hintColumn, now, ttl, FBUtilities.nowInSeconds(), value); - - return new Mutation(PartitionUpdate.singleRowUpdate(SystemKeyspace.Hints, key, BTreeRow.singleCellRow(clustering, cell))); } - /* - * determine the TTL for the hint Mutation - * this is set at the smallest GCGraceSeconds for any of the CFs in the RM - * this ensures that deletes aren't "undone" by delivery of an old hint - */ - public static int calculateHintTTL(Mutation mutation) - { - int ttl = maxHintTTL; - for (PartitionUpdate upd : mutation.getPartitionUpdates()) - ttl = Math.min(ttl, upd.metadata().params.gcGraceSeconds); - return ttl; - } - - public void start() + public void registerMBean() { MBeanServer mbs = ManagementFactory.getPlatformMBeanServer(); try @@ -177,404 +53,35 @@ public class HintedHandOffManager implements HintedHandOffManagerMBean { throw new RuntimeException(e); } - logger.debug("Created HHOM instance, registered MBean."); - - Runnable runnable = new Runnable() - { - public void run() - { - scheduleAllDeliveries(); - metrics.log(); - } - }; - executor.scheduleWithFixedDelay(runnable, 10, 10, TimeUnit.MINUTES); } - private static void deleteHint(ByteBuffer tokenBytes, Clustering clustering, long timestamp) + public void deleteHintsForEndpoint(String host) { - Cell cell = BufferCell.tombstone(hintColumn, timestamp, FBUtilities.nowInSeconds()); - PartitionUpdate upd = PartitionUpdate.singleRowUpdate(SystemKeyspace.Hints, tokenBytes, BTreeRow.singleCellRow(clustering, cell)); - new Mutation(upd).applyUnsafe(); // don't bother with commitlog since we're going to flush as soon as we're done with delivery + HintsService.instance.deleteAllHintsForEndpoint(host); } - public void deleteHintsForEndpoint(final String ipOrHostname) + public void truncateAllHints() { - try - { - InetAddress endpoint = InetAddress.getByName(ipOrHostname); - deleteHintsForEndpoint(endpoint); - } - catch (UnknownHostException e) - { - logger.warn("Unable to find {}, not a hostname or ipaddr of a node", ipOrHostname); - throw new RuntimeException(e); - } - } - - public void deleteHintsForEndpoint(final InetAddress endpoint) - { - if (!StorageService.instance.getTokenMetadata().isMember(endpoint)) - return; - UUID hostId = StorageService.instance.getTokenMetadata().getHostId(endpoint); - ByteBuffer key = ByteBuffer.wrap(UUIDGen.decompose(hostId)); - final Mutation mutation = new Mutation(PartitionUpdate.fullPartitionDelete(SystemKeyspace.Hints, key, System.currentTimeMillis(), FBUtilities.nowInSeconds())); - - // execute asynchronously to avoid blocking caller (which may be processing gossip) - Runnable runnable = new Runnable() - { - public void run() - { - try - { - logger.info("Deleting any stored hints for {}", endpoint); - mutation.apply(); - hintStore.forceBlockingFlush(); - compact(); - } - catch (Exception e) - { - JVMStabilityInspector.inspectThrowable(e); - logger.warn("Could not delete hints for {}: {}", endpoint, e); - } - } - }; - executor.submit(runnable); - } - - //foobar - public void truncateAllHints() throws ExecutionException, InterruptedException - { - Runnable runnable = new Runnable() - { - public void run() - { - try - { - logger.info("Truncating all stored hints."); - Keyspace.open(SystemKeyspace.NAME).getColumnFamilyStore(SystemKeyspace.HINTS).truncateBlocking(); - } - catch (Exception e) - { - logger.warn("Could not truncate all hints.", e); - } - } - }; - executor.submit(runnable).get(); - } - - @VisibleForTesting - protected synchronized void compact() - { - ArrayList descriptors = new ArrayList<>(); - for (SSTable sstable : hintStore.getTracker().getUncompacting()) - descriptors.add(sstable.descriptor); - - if (descriptors.isEmpty()) - return; - - try - { - CompactionManager.instance.submitUserDefined(hintStore, descriptors, (int) (System.currentTimeMillis() / 1000)).get(); - } - catch (InterruptedException | ExecutionException e) - { - throw new RuntimeException(e); - } - } - - private int waitForSchemaAgreement(InetAddress endpoint) throws TimeoutException - { - Gossiper gossiper = Gossiper.instance; - int waited = 0; - // first, wait for schema to be gossiped. - while (gossiper.getEndpointStateForEndpoint(endpoint) != null && gossiper.getEndpointStateForEndpoint(endpoint).getApplicationState(ApplicationState.SCHEMA) == null) - { - Uninterruptibles.sleepUninterruptibly(1, TimeUnit.SECONDS); - waited += 1000; - if (waited > 2 * StorageService.RING_DELAY) - throw new TimeoutException("Didin't receive gossiped schema from " + endpoint + " in " + 2 * StorageService.RING_DELAY + "ms"); - } - if (gossiper.getEndpointStateForEndpoint(endpoint) == null) - throw new TimeoutException("Node " + endpoint + " vanished while waiting for agreement"); - waited = 0; - // then wait for the correct schema version. - // usually we use DD.getDefsVersion, which checks the local schema uuid as stored in the system keyspace. - // here we check the one in gossip instead; this serves as a canary to warn us if we introduce a bug that - // causes the two to diverge (see CASSANDRA-2946) - while (gossiper.getEndpointStateForEndpoint(endpoint) != null && !gossiper.getEndpointStateForEndpoint(endpoint).getApplicationState(ApplicationState.SCHEMA).value.equals( - gossiper.getEndpointStateForEndpoint(FBUtilities.getBroadcastAddress()).getApplicationState(ApplicationState.SCHEMA).value)) - { - Uninterruptibles.sleepUninterruptibly(1, TimeUnit.SECONDS); - waited += 1000; - if (waited > 2 * StorageService.RING_DELAY) - throw new TimeoutException("Could not reach schema agreement with " + endpoint + " in " + 2 * StorageService.RING_DELAY + "ms"); - } - if (gossiper.getEndpointStateForEndpoint(endpoint) == null) - throw new TimeoutException("Node " + endpoint + " vanished while waiting for agreement"); - logger.debug("schema for {} matches local schema", endpoint); - return waited; - } - - private void deliverHintsToEndpoint(InetAddress endpoint) - { - if (hintStore.isEmpty()) - return; // nothing to do, don't confuse users by logging a no-op handoff - - // check if hints delivery has been paused - if (hintedHandOffPaused) - { - logger.debug("Hints delivery process is paused, aborting"); - return; - } - - logger.debug("Checking remote({}) schema before delivering hints", endpoint); - try - { - waitForSchemaAgreement(endpoint); - } - catch (TimeoutException e) - { - return; - } - - if (!FailureDetector.instance.isAlive(endpoint)) - { - logger.debug("Endpoint {} died before hint delivery, aborting", endpoint); - return; - } - - doDeliverHintsToEndpoint(endpoint); - - // Flush all the tombstones to disk - hintStore.forceBlockingFlush(); - } - - private boolean checkDelivered(InetAddress endpoint, List> handlers, AtomicInteger rowsReplayed) - { - for (WriteResponseHandler handler : handlers) - { - try - { - handler.get(); - } - catch (WriteTimeoutException e) - { - logger.info("Failed replaying hints to {}; aborting ({} delivered), error : {}", - endpoint, rowsReplayed, e.getMessage()); - return false; - } - } - return true; - } - - /* - * 1. Get the key of the endpoint we need to handoff - * 2. For each column, deserialize the mutation and send it to the endpoint - * 3. Delete the column if the write was successful - * 4. Force a flush - */ - private void doDeliverHintsToEndpoint(InetAddress endpoint) - { - // find the hints for the node using its token. - UUID hostId = Gossiper.instance.getHostId(endpoint); - logger.info("Started hinted handoff for host: {} with IP: {}", hostId, endpoint); - final ByteBuffer hostIdBytes = ByteBuffer.wrap(UUIDGen.decompose(hostId)); - - final AtomicInteger rowsReplayed = new AtomicInteger(0); - - // rate limit is in bytes per second. Uses Double.MAX_VALUE if disabled (set to 0 in cassandra.yaml). - // max rate is scaled by the number of nodes in the cluster (CASSANDRA-5272). - int throttleInKB = DatabaseDescriptor.getHintedHandoffThrottleInKB() - / (StorageService.instance.getTokenMetadata().getAllEndpoints().size() - 1); - RateLimiter rateLimiter = RateLimiter.create(throttleInKB == 0 ? Double.MAX_VALUE : throttleInKB * 1024); - - int nowInSec = FBUtilities.nowInSeconds(); - try (OpOrder.Group op = hintStore.readOrdering.start(); - RowIterator iter = UnfilteredRowIterators.filter(SinglePartitionReadCommand.fullPartitionRead(SystemKeyspace.Hints, nowInSec, hostIdBytes).queryMemtableAndDisk(hintStore, op), nowInSec)) - { - List> responseHandlers = Lists.newArrayList(); - - while (iter.hasNext()) - { - // check if node is still alive and we should continue delivery process - if (!FailureDetector.instance.isAlive(endpoint)) - { - logger.info("Endpoint {} died during hint delivery; aborting ({} delivered)", endpoint, rowsReplayed); - return; - } - - // check if hints delivery has been paused during the process - if (hintedHandOffPaused) - { - logger.debug("Hints delivery process is paused, aborting"); - return; - } - - // Wait regularly on the endpoint acknowledgment. If we timeout on it, the endpoint is probably dead so stop delivery - if (responseHandlers.size() > MAX_SIMULTANEOUSLY_REPLAYED_HINTS && !checkDelivered(endpoint, responseHandlers, rowsReplayed)) - return; - - final Row hint = iter.next(); - int version = Int32Type.instance.compose(hint.clustering().get(1)); - Cell cell = hint.getCell(hintColumn); - - final long timestamp = cell.timestamp(); - DataInputPlus in = new DataInputBuffer(cell.value(), true); - Mutation mutation; - try - { - mutation = Mutation.serializer.deserialize(in, version); - } - catch (UnknownColumnFamilyException e) - { - logger.debug("Skipping delivery of hint for deleted table", e); - deleteHint(hostIdBytes, hint.clustering(), timestamp); - continue; - } - catch (IOException e) - { - throw new AssertionError(e); - } - - for (UUID cfId : mutation.getColumnFamilyIds()) - { - if (timestamp <= SystemKeyspace.getTruncatedAt(cfId)) - { - logger.debug("Skipping delivery of hint for truncated table {}", cfId); - mutation = mutation.without(cfId); - } - } - - if (mutation.isEmpty()) - { - deleteHint(hostIdBytes, hint.clustering(), timestamp); - continue; - } - - MessageOut message = mutation.createMessage(); - rateLimiter.acquire(message.serializedSize(MessagingService.current_version)); - Runnable callback = new Runnable() - { - public void run() - { - rowsReplayed.incrementAndGet(); - deleteHint(hostIdBytes, hint.clustering(), timestamp); - } - }; - WriteResponseHandler responseHandler = new WriteResponseHandler<>(endpoint, WriteType.SIMPLE, callback); - MessagingService.instance().sendRR(message, endpoint, responseHandler, false); - responseHandlers.add(responseHandler); - } - - // Wait on the last handlers - if (checkDelivered(endpoint, responseHandlers, rowsReplayed)) - logger.info("Finished hinted handoff of {} rows to endpoint {}", rowsReplayed, endpoint); - } - } - - /** - * Attempt delivery to any node for which we have hints. Necessary since we can generate hints even for - * nodes which are never officially down/failed. - */ - private void scheduleAllDeliveries() - { - logger.debug("Started scheduleAllDeliveries"); - - // Force a major compaction to get rid of the tombstones and expired hints. Do it once, before we schedule any - // individual replay, to avoid N - 1 redundant individual compactions (when N is the number of nodes with hints - // to deliver to). - compact(); - - ReadCommand cmd = new PartitionRangeReadCommand(hintStore.metadata, - FBUtilities.nowInSeconds(), - ColumnFilter.all(hintStore.metadata), - RowFilter.NONE, - DataLimits.cqlLimits(Integer.MAX_VALUE, 1), - DataRange.allData(hintStore.metadata.partitioner)); - - try (ReadOrderGroup orderGroup = cmd.startOrderGroup(); UnfilteredPartitionIterator iter = cmd.executeLocally(orderGroup)) - { - while (iter.hasNext()) - { - try (UnfilteredRowIterator partition = iter.next()) - { - UUID hostId = UUIDGen.getUUID(partition.partitionKey().getKey()); - InetAddress target = StorageService.instance.getTokenMetadata().getEndpointForHostId(hostId); - // token may have since been removed (in which case we have just read back a tombstone) - if (target != null) - scheduleHintDelivery(target, false); - } - } - } - - logger.debug("Finished scheduleAllDeliveries"); - } - - /* - * This method is used to deliver hints to a particular endpoint. - * When we learn that some endpoint is back up we deliver the data - * to him via an event driven mechanism. - */ - public void scheduleHintDelivery(final InetAddress to, final boolean precompact) - { - // We should not deliver hints to the same host in 2 different threads - if (!queuedDeliveries.add(to)) - return; - - logger.debug("Scheduling delivery of Hints to {}", to); - - hintDeliveryExecutor.execute(new Runnable() - { - public void run() - { - try - { - // If it's an individual node hint replay (triggered by Gossip or via JMX), and not the global scheduled replay - // (every 10 minutes), force a major compaction to get rid of the tombstones and expired hints. - if (precompact) - compact(); - - deliverHintsToEndpoint(to); - } - finally - { - queuedDeliveries.remove(to); - } - } - }); - } - - public void scheduleHintDelivery(String to) throws UnknownHostException - { - scheduleHintDelivery(InetAddress.getByName(to), true); - } - - public void pauseHintsDelivery(boolean b) - { - hintedHandOffPaused = b; + HintsService.instance.deleteAllHints(); } + // TODO public List listEndpointsPendingHints() { - // Extract the keys as strings to be reported. - List result = new ArrayList<>(); + throw new UnsupportedOperationException(); + } - ReadCommand cmd = PartitionRangeReadCommand.allDataRead(SystemKeyspace.Hints, FBUtilities.nowInSeconds()); - try (ReadOrderGroup orderGroup = cmd.startOrderGroup(); - UnfilteredPartitionIterator iter = cmd.executeLocally(orderGroup)) - { - while (iter.hasNext()) - { - try (UnfilteredRowIterator partition = iter.next()) - { - // We don't delete by range on the hints table, so we don't have to worry about the - // iterator returning only range tombstone marker - if (partition.hasNext()) - result.add(UUIDType.instance.compose(partition.partitionKey().getKey()).toString()); - } - } - } + // TODO + public void scheduleHintDelivery(String host) + { + throw new UnsupportedOperationException(); + } - return result; + public void pauseHintsDelivery(boolean doPause) + { + if (doPause) + HintsService.instance.pauseDispatch(); + else + HintsService.instance.resumeDispatch(); } } diff --git a/src/java/org/apache/cassandra/db/HintedHandOffManagerMBean.java b/src/java/org/apache/cassandra/db/HintedHandOffManagerMBean.java index bbb2a14c80..9ba425e2c0 100644 --- a/src/java/org/apache/cassandra/db/HintedHandOffManagerMBean.java +++ b/src/java/org/apache/cassandra/db/HintedHandOffManagerMBean.java @@ -21,6 +21,7 @@ import java.net.UnknownHostException; import java.util.List; import java.util.concurrent.ExecutionException; +@Deprecated public interface HintedHandOffManagerMBean { /** diff --git a/src/java/org/apache/cassandra/db/Mutation.java b/src/java/org/apache/cassandra/db/Mutation.java index 709c78fb67..6e78b0e477 100644 --- a/src/java/org/apache/cassandra/db/Mutation.java +++ b/src/java/org/apache/cassandra/db/Mutation.java @@ -75,10 +75,24 @@ public class Mutation implements IMutation public Mutation copy() { - Mutation copy = new Mutation(keyspaceName, key, new HashMap<>(modifications)); + return new Mutation(keyspaceName, key, new HashMap<>(modifications)); + } + + public Mutation without(Set cfIds) + { + if (cfIds.isEmpty()) + return this; + + Mutation copy = copy(); + copy.modifications.keySet().removeAll(cfIds); return copy; } + public Mutation without(UUID cfId) + { + return without(Collections.singleton(cfId)); + } + public String getKeyspaceName() { return keyspaceName; @@ -207,6 +221,14 @@ public class Mutation implements IMutation return DatabaseDescriptor.getWriteRpcTimeout(); } + public int smallestGCGS() + { + int gcgs = Integer.MAX_VALUE; + for (PartitionUpdate update : getPartitionUpdates()) + gcgs = Math.min(gcgs, update.metadata().params.gcGraceSeconds); + return gcgs; + } + public String toString() { return toString(false); @@ -235,15 +257,6 @@ public class Mutation implements IMutation return buff.append("])").toString(); } - public Mutation without(UUID cfId) - { - Mutation mutation = new Mutation(keyspaceName, key); - for (Map.Entry entry : modifications.entrySet()) - if (!entry.getKey().equals(cfId)) - mutation.add(entry.getValue()); - return mutation; - } - public static class MutationSerializer implements IVersionedSerializer { public void serialize(Mutation mutation, DataOutputPlus out, int version) throws IOException diff --git a/src/java/org/apache/cassandra/db/SystemKeyspace.java b/src/java/org/apache/cassandra/db/SystemKeyspace.java index d24b18be5b..38cfed6f71 100644 --- a/src/java/org/apache/cassandra/db/SystemKeyspace.java +++ b/src/java/org/apache/cassandra/db/SystemKeyspace.java @@ -88,7 +88,6 @@ public final class SystemKeyspace public static final String NAME = "system"; - public static final String HINTS = "hints"; public static final String BATCHES = "batches"; public static final String PAXOS = "paxos"; public static final String BUILT_INDEXES = "IndexInfo"; @@ -103,6 +102,7 @@ public final class SystemKeyspace public static final String MATERIALIZED_VIEWS_BUILDS_IN_PROGRESS = "materialized_views_builds_in_progress"; public static final String BUILT_MATERIALIZED_VIEWS = "built_materialized_views"; + @Deprecated public static final String LEGACY_HINTS = "hints"; @Deprecated public static final String LEGACY_BATCHLOG = "batchlog"; @Deprecated public static final String LEGACY_KEYSPACES = "schema_keyspaces"; @Deprecated public static final String LEGACY_COLUMNFAMILIES = "schema_columnfamilies"; @@ -112,19 +112,6 @@ public final class SystemKeyspace @Deprecated public static final String LEGACY_FUNCTIONS = "schema_functions"; @Deprecated public static final String LEGACY_AGGREGATES = "schema_aggregates"; - public static final CFMetaData Hints = - compile(HINTS, - "hints awaiting delivery", - "CREATE TABLE %s (" - + "target_id uuid," - + "hint_id timeuuid," - + "message_version int," - + "mutation blob," - + "PRIMARY KEY ((target_id), hint_id, message_version)) " - + "WITH COMPACT STORAGE") - .compaction(CompactionParams.scts(singletonMap("enabled", "false"))) - .gcGraceSeconds(0); - public static final CFMetaData Batches = compile(BATCHES, "batches awaiting replay", @@ -281,6 +268,20 @@ public final class SystemKeyspace + "view_name text," + "PRIMARY KEY ((keyspace_name), view_name))"); + @Deprecated + public static final CFMetaData LegacyHints = + compile(LEGACY_HINTS, + "*DEPRECATED* hints awaiting delivery", + "CREATE TABLE %s (" + + "target_id uuid," + + "hint_id timeuuid," + + "message_version int," + + "mutation blob," + + "PRIMARY KEY ((target_id), hint_id, message_version)) " + + "WITH COMPACT STORAGE") + .compaction(CompactionParams.scts(singletonMap("enabled", "false"))) + .gcGraceSeconds(0); + @Deprecated public static final CFMetaData LegacyBatchlog = compile(LEGACY_BATCHLOG, @@ -423,7 +424,6 @@ public final class SystemKeyspace private static Tables tables() { return Tables.of(BuiltIndexes, - Hints, Batches, Paxos, Local, @@ -436,6 +436,7 @@ public final class SystemKeyspace AvailableRanges, MaterializedViewsBuildsInProgress, BuiltMaterializedViews, + LegacyHints, LegacyBatchlog, LegacyKeyspaces, LegacyColumnfamilies, diff --git a/src/java/org/apache/cassandra/db/commitlog/CommitLog.java b/src/java/org/apache/cassandra/db/commitlog/CommitLog.java index ff27225f7d..37fcbe211c 100644 --- a/src/java/org/apache/cassandra/db/commitlog/CommitLog.java +++ b/src/java/org/apache/cassandra/db/commitlog/CommitLog.java @@ -48,6 +48,8 @@ import org.apache.cassandra.service.StorageService; import org.apache.cassandra.utils.JVMStabilityInspector; import static org.apache.cassandra.db.commitlog.CommitLogSegment.*; +import static org.apache.cassandra.utils.FBUtilities.updateChecksum; +import static org.apache.cassandra.utils.FBUtilities.updateChecksumInt; /* * Commit Log tracks every write operation into the system. The aim of the commit log is to be able to @@ -61,7 +63,7 @@ public class CommitLog implements CommitLogMBean // we only permit records HALF the size of a commit log, to ensure we don't spin allocating many mostly // empty segments when writing large records - private final long MAX_MUTATION_SIZE = DatabaseDescriptor.getCommitLogSegmentSize() >> 1; + private final long MAX_MUTATION_SIZE = DatabaseDescriptor.getMaxMutationSize(); public final CommitLogSegmentManager allocator; public final CommitLogArchiver archiver; @@ -254,9 +256,9 @@ public class CommitLog implements CommitLogMBean { assert mutation != null; - long size = Mutation.serializer.serializedSize(mutation, MessagingService.current_version); + int size = (int) Mutation.serializer.serializedSize(mutation, MessagingService.current_version); - long totalSize = size + ENTRY_OVERHEAD_SIZE; + int totalSize = size + ENTRY_OVERHEAD_SIZE; if (totalSize > MAX_MUTATION_SIZE) { throw new IllegalArgumentException(String.format("Mutation of %s bytes is too large for the maxiumum size of %s", @@ -269,19 +271,13 @@ public class CommitLog implements CommitLogMBean try (BufferedDataOutputStreamPlus dos = new DataOutputBufferFixed(buffer)) { // checksummed length - dos.writeInt((int) size); - - ByteBuffer copy = buffer.duplicate(); - copy.position(buffer.position() - 4); - copy.limit(buffer.position()); - checksum.update(copy); + dos.writeInt(size); + updateChecksumInt(checksum, size); buffer.putInt((int) checksum.getValue()); // checksummed mutation - copy = buffer.duplicate(); Mutation.serializer.serialize(mutation, dos, MessagingService.current_version); - copy.limit(copy.position() + (int) size); - checksum.update(copy); + updateChecksum(checksum, buffer, buffer.position() - size, size); buffer.putInt((int) checksum.getValue()); } catch (IOException e) diff --git a/src/java/org/apache/cassandra/gms/Gossiper.java b/src/java/org/apache/cassandra/gms/Gossiper.java index 5fa402afea..7aa604e7b7 100644 --- a/src/java/org/apache/cassandra/gms/Gossiper.java +++ b/src/java/org/apache/cassandra/gms/Gossiper.java @@ -809,6 +809,20 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean return endpointStateMap.get(ep); } + public boolean valuesEqual(InetAddress ep1, InetAddress ep2, ApplicationState as) + { + EndpointState state1 = getEndpointStateForEndpoint(ep1); + EndpointState state2 = getEndpointStateForEndpoint(ep2); + + if (state1 == null || state2 == null) + return false; + + VersionedValue value1 = state1.getApplicationState(as); + VersionedValue value2 = state2.getApplicationState(as); + + return !(value1 == null || value2 == null) && value1.value.equals(value2.value); + } + // removes ALL endpoint states; should only be called after shadow gossip public void resetEndpointStateMap() { diff --git a/src/java/org/apache/cassandra/hints/ChecksummedDataInput.java b/src/java/org/apache/cassandra/hints/ChecksummedDataInput.java new file mode 100644 index 0000000000..fa727bc0ed --- /dev/null +++ b/src/java/org/apache/cassandra/hints/ChecksummedDataInput.java @@ -0,0 +1,114 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.hints; + +import java.io.IOException; +import java.util.zip.CRC32; + +import org.apache.cassandra.io.util.AbstractDataInput; + +/** + * An {@link AbstractDataInput} wrapper that calctulates the CRC in place. + * + * Useful for {@link org.apache.cassandra.hints.HintsReader}, for example, where we must verify the CRC, yet don't want + * to allocate an extra byte array just that purpose. + * + * In addition to calculating the CRC, allows to enforce a maximim known size. This is needed + * so that {@link org.apache.cassandra.db.Mutation.MutationSerializer} doesn't blow up the heap when deserializing a + * corrupted sequence by reading a huge corrupted length of bytes via + * via {@link org.apache.cassandra.utils.ByteBufferUtil#readWithLength(java.io.DataInput)}. + */ +public final class ChecksummedDataInput extends AbstractDataInput +{ + private final CRC32 crc; + private final AbstractDataInput source; + private int limit; + + private ChecksummedDataInput(AbstractDataInput source) + { + this.source = source; + + crc = new CRC32(); + limit = Integer.MAX_VALUE; + } + + public static ChecksummedDataInput wrap(AbstractDataInput source) + { + return new ChecksummedDataInput(source); + } + + public void resetCrc() + { + crc.reset(); + } + + public void resetLimit() + { + limit = Integer.MAX_VALUE; + } + + public void limit(int newLimit) + { + limit = newLimit; + } + + public int bytesRemaining() + { + return limit; + } + + public int getCrc() + { + return (int) crc.getValue(); + } + + public void seek(long position) throws IOException + { + source.seek(position); + } + + public long getPosition() + { + return source.getPosition(); + } + + public long getPositionLimit() + { + return source.getPositionLimit(); + } + + public int read() throws IOException + { + int b = source.read(); + crc.update(b); + limit--; + return b; + } + + @Override + public int read(byte[] buff, int offset, int length) throws IOException + { + if (length > limit) + throw new IOException("Digest mismatch exception"); + + int copied = source.read(buff, offset, length); + crc.update(buff, offset, copied); + limit -= copied; + return copied; + } +} diff --git a/src/java/org/apache/cassandra/hints/EncodedHintMessage.java b/src/java/org/apache/cassandra/hints/EncodedHintMessage.java new file mode 100644 index 0000000000..2797495221 --- /dev/null +++ b/src/java/org/apache/cassandra/hints/EncodedHintMessage.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.hints; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.UUID; + +import org.apache.cassandra.db.TypeSizes; +import org.apache.cassandra.io.IVersionedSerializer; +import org.apache.cassandra.io.util.DataInputPlus; +import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.net.MessageOut; +import org.apache.cassandra.net.MessagingService; +import org.apache.cassandra.utils.UUIDSerializer; + +/** + * A specialized version of {@link HintMessage} that takes an already encoded in a bytebuffer hint and sends it verbatim. + * + * An optimization for when dispatching a hint file of the current messaging version to a node of the same messaging version, + * which is the most common case. Saves on extra ByteBuffer allocations one redundant hint deserialization-serialization cycle. + * + * Never deserialized as an EncodedHintMessage - the receiving side will always deserialize the message as vanilla + * {@link HintMessage}. + */ +final class EncodedHintMessage +{ + private static final IVersionedSerializer serializer = new Serializer(); + + private final UUID hostId; + private final ByteBuffer hint; + private final int version; + + EncodedHintMessage(UUID hostId, ByteBuffer hint, int version) + { + this.hostId = hostId; + this.hint = hint; + this.version = version; + } + + MessageOut createMessageOut() + { + return new MessageOut<>(MessagingService.Verb.HINT, this, serializer); + } + + private static class Serializer implements IVersionedSerializer + { + public long serializedSize(EncodedHintMessage message, int version) + { + if (version != message.version) + throw new IllegalArgumentException("serializedSize() called with non-matching version " + version); + + int size = (int) UUIDSerializer.serializer.serializedSize(message.hostId, version); + size += TypeSizes.sizeof(message.hint.remaining()); + size += message.hint.remaining(); + return size; + } + + public void serialize(EncodedHintMessage message, DataOutputPlus out, int version) throws IOException + { + if (version != message.version) + throw new IllegalArgumentException("serialize() called with non-matching version " + version); + + UUIDSerializer.serializer.serialize(message.hostId, out, version); + out.writeInt(message.hint.remaining()); + out.write(message.hint); + } + + public EncodedHintMessage deserialize(DataInputPlus in, int version) throws IOException + { + throw new UnsupportedOperationException(); + } + } +} diff --git a/src/java/org/apache/cassandra/hints/Hint.java b/src/java/org/apache/cassandra/hints/Hint.java new file mode 100644 index 0000000000..d8f85c5bf9 --- /dev/null +++ b/src/java/org/apache/cassandra/hints/Hint.java @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.hints; + +import java.io.IOException; +import java.util.*; +import java.util.concurrent.TimeUnit; + +import org.apache.cassandra.db.*; +import org.apache.cassandra.io.IVersionedSerializer; +import org.apache.cassandra.io.util.DataInputPlus; +import org.apache.cassandra.io.util.DataOutputPlus; + +/** + * Encapsulates the hinted mutation, its creation time, and the gc grace seconds param for each table involved. + * + * - Why do we need to track hint creation time? + * - We must exclude updates for tables that have been truncated after hint's creation, otherwise the result is data corruption. + * + * - Why do we need to track gc grace seconds? + * - Hints can stay in storage for a while before being applied, and without recording gc grace seconds (+ creation time), + * if we apply the mutation blindly, we risk resurrecting a deleted value, a tombstone for which had been already + * compacted away while the hint was in storage. + * + * We also look at the smallest current value of the gcgs param for each affected table when applying the hint, and use + * creation time + min(recorded gc gs, current gcgs + current gc grace) as the overall hint expiration time. + * This allows now to safely reduce gc gs on tables without worrying that an applied old hint might resurrect any data. + */ +public final class Hint +{ + public static final Serializer serializer = new Serializer(); + + final Mutation mutation; + final long creationTime; // time of hint creation (in milliseconds) + final int gcgs; // the smallest gc gs of all involved tables + + private Hint(Mutation mutation, long creationTime, int gcgs) + { + this.mutation = mutation; + this.creationTime = creationTime; + this.gcgs = gcgs; + } + + /** + * @param mutation the hinted mutation + * @param creationTime time of this hint's creation (in milliseconds since epoch) + */ + public static Hint create(Mutation mutation, long creationTime) + { + return new Hint(mutation, creationTime, mutation.smallestGCGS()); + } + + /** + * @param mutation the hinted mutation + * @param creationTime time of this hint's creation (in milliseconds since epoch) + * @param gcgs the smallest gcgs of all tables involved at the time of hint creation (in seconds) + */ + public static Hint create(Mutation mutation, long creationTime, int gcgs) + { + return new Hint(mutation, creationTime, gcgs); + } + + /** + * Applies the contained mutation unless it's expired, filtering out any updates for truncated tables + */ + void apply() + { + if (!isLive()) + return; + + // filter out partition update for table that have been truncated since hint's creation + Mutation filtered = mutation; + for (UUID id : mutation.getColumnFamilyIds()) + if (creationTime <= SystemKeyspace.getTruncatedAt(id)) + filtered = filtered.without(id); + + if (!filtered.isEmpty()) + filtered.apply(); + } + + /** + * @return calculates whether or not it is safe to apply the hint without risking to resurrect any deleted data + */ + boolean isLive() + { + int smallestGCGS = Math.min(gcgs, mutation.smallestGCGS()); + long expirationTime = creationTime + TimeUnit.SECONDS.toMillis(smallestGCGS); + return expirationTime > System.currentTimeMillis(); + } + + static final class Serializer implements IVersionedSerializer + { + public long serializedSize(Hint hint, int version) + { + long size = TypeSizes.sizeof(hint.creationTime); + size += TypeSizes.sizeof(hint.gcgs); + size += Mutation.serializer.serializedSize(hint.mutation, version); + return size; + } + + public void serialize(Hint hint, DataOutputPlus out, int version) throws IOException + { + out.writeLong(hint.creationTime); + out.writeInt(hint.gcgs); + Mutation.serializer.serialize(hint.mutation, out, version); + } + + public Hint deserialize(DataInputPlus in, int version) throws IOException + { + long creationTime = in.readLong(); + int gcgs = in.readInt(); + return new Hint(Mutation.serializer.deserialize(in, version), creationTime, gcgs); + } + } +} diff --git a/src/java/org/apache/cassandra/hints/HintMessage.java b/src/java/org/apache/cassandra/hints/HintMessage.java new file mode 100644 index 0000000000..89baa895f4 --- /dev/null +++ b/src/java/org/apache/cassandra/hints/HintMessage.java @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.cassandra.hints; + +import java.io.IOException; +import java.util.Objects; +import java.util.UUID; + +import javax.annotation.Nullable; + +import org.apache.cassandra.db.TypeSizes; +import org.apache.cassandra.db.UnknownColumnFamilyException; +import org.apache.cassandra.io.IVersionedSerializer; +import org.apache.cassandra.io.util.DataInputPlus; +import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.net.MessageOut; +import org.apache.cassandra.net.MessagingService; +import org.apache.cassandra.utils.BytesReadTracker; +import org.apache.cassandra.utils.UUIDSerializer; + +/** + * The message we use to dispatch and forward hints. + * + * Encodes the host id the hint is meant for and the hint itself. + * We use the host id to determine whether we should store or apply the hint: + * 1. If host id equals to the receiving node host id, then we apply the hint + * 2. If host id is different from the receiving node's host id, then we store the hint + * + * Scenario (1) means that we are dealing with regular hint dispatch. + * Scenario (2) means that we got a hint from a node that's going through decommissioning and is streaming its hints + * elsewhere first. + */ +public final class HintMessage +{ + public static final IVersionedSerializer serializer = new Serializer(); + + final UUID hostId; + + @Nullable // can be null if we fail do decode the hint because of an unknown table id in it + final Hint hint; + + @Nullable // will usually be null, unless a hint deserialization fails due to an unknown table id + final UUID unknownTableID; + + HintMessage(UUID hostId, Hint hint) + { + this.hostId = hostId; + this.hint = hint; + this.unknownTableID = null; + } + + HintMessage(UUID hostId, UUID unknownTableID) + { + this.hostId = hostId; + this.hint = null; + this.unknownTableID = unknownTableID; + } + + public MessageOut createMessageOut() + { + return new MessageOut<>(MessagingService.Verb.HINT, this, serializer); + } + + public static class Serializer implements IVersionedSerializer + { + public long serializedSize(HintMessage message, int version) + { + int size = (int) UUIDSerializer.serializer.serializedSize(message.hostId, version); + + int hintSize = (int) Hint.serializer.serializedSize(message.hint, version); + size += TypeSizes.sizeof(hintSize); + size += hintSize; + + return size; + } + + public void serialize(HintMessage message, DataOutputPlus out, int version) throws IOException + { + Objects.requireNonNull(message.hint); // we should never *send* a HintMessage with null hint + + UUIDSerializer.serializer.serialize(message.hostId, out, version); + + /* + * We are serializing the hint size so that the receiver of the message could gracefully handle + * deserialize failure when a table had been dropped, by simply skipping the unread bytes. + */ + out.writeInt((int) Hint.serializer.serializedSize(message.hint, version)); + + Hint.serializer.serialize(message.hint, out, version); + } + + /* + * It's not an exceptional scenario to have a hints file streamed that have partition updates for tables + * that don't exist anymore. We want to handle that case gracefully instead of dropping the connection for every + * one of them. + */ + public HintMessage deserialize(DataInputPlus in, int version) throws IOException + { + UUID hostId = UUIDSerializer.serializer.deserialize(in, version); + + int hintSize = in.readInt(); + BytesReadTracker countingIn = new BytesReadTracker(in); + try + { + return new HintMessage(hostId, Hint.serializer.deserialize(countingIn, version)); + } + catch (UnknownColumnFamilyException e) + { + in.skipBytes(hintSize - (int) countingIn.getBytesRead()); + return new HintMessage(hostId, e.cfId); + } + } + } +} diff --git a/src/java/org/apache/cassandra/hints/HintResponse.java b/src/java/org/apache/cassandra/hints/HintResponse.java new file mode 100644 index 0000000000..8aa888f551 --- /dev/null +++ b/src/java/org/apache/cassandra/hints/HintResponse.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.cassandra.hints; + +import org.apache.cassandra.io.IVersionedSerializer; +import org.apache.cassandra.io.util.DataInputPlus; +import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.net.MessageOut; +import org.apache.cassandra.net.MessagingService; + +/** + * An empty successful response to a HintMessage. + */ +public final class HintResponse +{ + public static final IVersionedSerializer serializer = new Serializer(); + + static final HintResponse instance = new HintResponse(); + static final MessageOut message = + new MessageOut<>(MessagingService.Verb.REQUEST_RESPONSE, instance, serializer); + + private HintResponse() + { + } + + private static final class Serializer implements IVersionedSerializer + { + public long serializedSize(HintResponse response, int version) + { + return 0; + } + + public void serialize(HintResponse response, DataOutputPlus out, int version) + { + } + + public HintResponse deserialize(DataInputPlus in, int version) + { + return instance; + } + } +} diff --git a/src/java/org/apache/cassandra/hints/HintVerbHandler.java b/src/java/org/apache/cassandra/hints/HintVerbHandler.java new file mode 100644 index 0000000000..458d01ff76 --- /dev/null +++ b/src/java/org/apache/cassandra/hints/HintVerbHandler.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.cassandra.hints; + +import java.net.InetAddress; +import java.util.UUID; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.db.partitions.PartitionUpdate; +import org.apache.cassandra.net.IVerbHandler; +import org.apache.cassandra.net.MessageIn; +import org.apache.cassandra.net.MessagingService; +import org.apache.cassandra.serializers.MarshalException; +import org.apache.cassandra.service.StorageService; + +/** + * Verb handler used both for hint dispatch and streaming. + * + * With the non-sstable format, we cannot just stream hint sstables on node decommission. So sometimes, at decommission + * time, we might have to stream hints to a non-owning host (say, if the owning host B is down during decommission of host A). + * In that case the handler just stores the received hint in its local hint store. + */ +public final class HintVerbHandler implements IVerbHandler +{ + private static final Logger logger = LoggerFactory.getLogger(HintVerbHandler.class); + + public void doVerb(MessageIn message, int id) + { + UUID hostId = message.payload.hostId; + Hint hint = message.payload.hint; + + // If we see an unknown table id, it means the table, or one of the tables in the mutation, had been dropped. + // In that case there is nothing we can really do, or should do, other than log it go on. + // This will *not* happen due to a not-yet-seen table, because we don't transfer hints unless there + // is schema agreement between the sender and the receiver. + if (hint == null) + { + logger.debug("Failed to decode and apply a hint for {} - table with id {} is unknown", + hostId, + message.payload.unknownTableID); + reply(id, message.from); + return; + } + + // We must perform validation before applying the hint, and there is no other place to do it other than here. + try + { + hint.mutation.getPartitionUpdates().forEach(PartitionUpdate::validate); + } + catch (MarshalException e) + { + logger.warn("Failed to validate a hint for {} (table id {}) - skipped", hostId); + reply(id, message.from); + return; + } + + // Apply the hint if this node is the destination, store for future dispatch if this node isn't (must have gotten + // it from a decommissioned node that had streamed it before going out). + if (hostId.equals(StorageService.instance.getLocalHostUUID())) + hint.apply(); + else + HintsService.instance.write(hostId, hint); + + reply(id, message.from); + } + + private static void reply(int id, InetAddress to) + { + MessagingService.instance().sendReply(HintResponse.message, id, to); + } +} diff --git a/src/java/org/apache/cassandra/hints/HintsBuffer.java b/src/java/org/apache/cassandra/hints/HintsBuffer.java new file mode 100644 index 0000000000..097abce738 --- /dev/null +++ b/src/java/org/apache/cassandra/hints/HintsBuffer.java @@ -0,0 +1,261 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.hints; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.zip.CRC32; + +import org.apache.cassandra.io.util.DataOutputBufferFixed; +import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.io.util.FileUtils; +import org.apache.cassandra.net.MessagingService; +import org.apache.cassandra.utils.AbstractIterator; +import org.apache.cassandra.utils.concurrent.OpOrder; + +import static org.apache.cassandra.utils.FBUtilities.updateChecksum; +import static org.apache.cassandra.utils.FBUtilities.updateChecksumInt; + +/** + * A shared buffer that temporarily holds the serialized hints before they are flushed to disk. + * + * Consists of : + * - a ByteBuffer holding the serialized hints (length, length checksum and total checksum included) + * - a pointer to the current allocation offset + * - an {@link OpOrder} appendOrder for {@link HintsWriteExecutor} to wait on for all writes completion + * - a map of (host id -> offset queue) for the hints written + * + * It's possible to write a single hint for two or more hosts at the same time, in which case the same offset will be put + * into two or more offset queues. + */ +final class HintsBuffer +{ + // hint entry overhead in bytes (int length, int length checksum, int body checksum) + static final int ENTRY_OVERHEAD_SIZE = 12; + static final int CLOSED = -1; + + private final ByteBuffer slab; // the underlying backing ByteBuffer for all the serialized hints + private final AtomicInteger position; // the position in the slab that we currently allocate from + + private final ConcurrentMap> offsets; + private final OpOrder appendOrder; + + private HintsBuffer(ByteBuffer slab) + { + this.slab = slab; + + position = new AtomicInteger(); + offsets = new ConcurrentHashMap<>(); + appendOrder = new OpOrder(); + } + + static HintsBuffer create(int slabSize) + { + return new HintsBuffer(ByteBuffer.allocateDirect(slabSize)); + } + + boolean isClosed() + { + return position.get() == CLOSED; + } + + int capacity() + { + return slab.capacity(); + } + + int remaining() + { + int pos = position.get(); + return pos == CLOSED ? 0 : capacity() - pos; + } + + HintsBuffer recycle() + { + slab.clear(); + return new HintsBuffer(slab); + } + + void free() + { + FileUtils.clean(slab); + } + + /** + * Wait for any appends started before this method was called. + */ + void waitForModifications() + { + appendOrder.awaitNewBarrier(); // issue a barrier and wait for it + } + + Set hostIds() + { + return offsets.keySet(); + } + + /** + * Coverts the queue of offsets for the selected host id into an iterator of hints encoded as ByteBuffers. + */ + Iterator consumingHintsIterator(UUID hostId) + { + final Queue bufferOffsets = offsets.get(hostId); + + if (bufferOffsets == null) + return Collections.emptyIterator(); + + return new AbstractIterator() + { + private final ByteBuffer flyweight = slab.duplicate(); + + protected ByteBuffer computeNext() + { + Integer offset = bufferOffsets.poll(); + + if (offset == null) + return endOfData(); + + int totalSize = slab.getInt(offset) + ENTRY_OVERHEAD_SIZE; + + return (ByteBuffer) flyweight.clear().position(offset).limit(offset + totalSize); + } + }; + } + + Allocation allocate(int hintSize) + { + int totalSize = hintSize + ENTRY_OVERHEAD_SIZE; + + if (totalSize > slab.capacity() / 2) + { + throw new IllegalArgumentException(String.format("Hint of %s bytes is too large - the maximum size is %s", + hintSize, + slab.capacity() / 2)); + } + + @SuppressWarnings("resource") + OpOrder.Group opGroup = appendOrder.start(); // will eventually be closed by the receiver of the allocation + try + { + return allocate(totalSize, opGroup); + } + catch (Throwable t) + { + opGroup.close(); + throw t; + } + } + + private Allocation allocate(int totalSize, OpOrder.Group opGroup) + { + int offset = allocateBytes(totalSize); + if (offset < 0) + { + opGroup.close(); + return null; + } + return new Allocation(offset, totalSize, opGroup); + } + + private int allocateBytes(int totalSize) + { + while (true) + { + int prev = position.get(); + int next = prev + totalSize; + + if (prev == CLOSED) // the slab has been 'closed' + return CLOSED; + + if (next > slab.capacity()) + { + position.set(CLOSED); // mark the slab as no longer allocating if we've exceeded its capacity + return CLOSED; + } + + if (position.compareAndSet(prev, next)) + return prev; + } + } + + private void put(UUID hostId, int offset) + { + // we intentionally don't just return offsets.computeIfAbsent() because it's expensive compared to simple get(), + // and the method is on a really hot path + Queue queue = offsets.get(hostId); + if (queue == null) + queue = offsets.computeIfAbsent(hostId, (id) -> new ConcurrentLinkedQueue<>()); + queue.offer(offset); + } + + /** + * A placeholder for hint serialization. Should always be used in a try-with-resources block. + */ + final class Allocation implements AutoCloseable + { + private final Integer offset; + private final int totalSize; + private final OpOrder.Group opGroup; + + Allocation(int offset, int totalSize, OpOrder.Group opGroup) + { + this.offset = offset; + this.totalSize = totalSize; + this.opGroup = opGroup; + } + + void write(Iterable hostIds, Hint hint) + { + write(hint); + for (UUID hostId : hostIds) + put(hostId, offset); + } + + public void close() + { + opGroup.close(); + } + + private void write(Hint hint) + { + ByteBuffer buffer = (ByteBuffer) slab.duplicate().position(offset).limit(offset + totalSize); + DataOutputPlus dop = new DataOutputBufferFixed(buffer); + CRC32 crc = new CRC32(); + int hintSize = totalSize - ENTRY_OVERHEAD_SIZE; + try + { + dop.writeInt(hintSize); + updateChecksumInt(crc, hintSize); + dop.writeInt((int) crc.getValue()); + + Hint.serializer.serialize(hint, dop, MessagingService.current_version); + updateChecksum(crc, buffer, buffer.position() - hintSize, hintSize); + dop.writeInt((int) crc.getValue()); + } + catch (IOException e) + { + throw new AssertionError(); // cannot happen + } + } + } +} diff --git a/src/java/org/apache/cassandra/hints/HintsBufferPool.java b/src/java/org/apache/cassandra/hints/HintsBufferPool.java new file mode 100644 index 0000000000..83b155addd --- /dev/null +++ b/src/java/org/apache/cassandra/hints/HintsBufferPool.java @@ -0,0 +1,120 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.hints; + +import java.util.Queue; +import java.util.UUID; +import java.util.concurrent.ConcurrentLinkedQueue; + +import org.apache.cassandra.net.MessagingService; + +/** + * A primitive pool of {@link HintsBuffer} buffers. Under normal conditions should only hold two buffers - the currently + * written to one, and a reserve buffer to switch to when the first one is beyond capacity. + */ +final class HintsBufferPool +{ + interface FlushCallback + { + void flush(HintsBuffer buffer, HintsBufferPool pool); + } + + private volatile HintsBuffer currentBuffer; + private final Queue reserveBuffers; + private final int bufferSize; + private final FlushCallback flushCallback; + + HintsBufferPool(int bufferSize, FlushCallback flushCallback) + { + reserveBuffers = new ConcurrentLinkedQueue<>(); + + this.bufferSize = bufferSize; + this.flushCallback = flushCallback; + } + + /** + * @param hostIds host ids of the hint's target nodes + * @param hint the hint to store + */ + void write(Iterable hostIds, Hint hint) + { + int hintSize = (int) Hint.serializer.serializedSize(hint, MessagingService.current_version); + try (HintsBuffer.Allocation allocation = allocate(hintSize)) + { + allocation.write(hostIds, hint); + } + } + + private HintsBuffer.Allocation allocate(int hintSize) + { + HintsBuffer current = currentBuffer(); + + while (true) + { + HintsBuffer.Allocation allocation = current.allocate(hintSize); + if (allocation != null) + return allocation; + + // allocation failed due to insufficient size remaining in the buffer + if (switchCurrentBuffer(current)) + flushCallback.flush(current, this); + + current = currentBuffer; + } + } + + boolean offer(HintsBuffer buffer) + { + if (!reserveBuffers.isEmpty()) + return false; + + reserveBuffers.offer(buffer); + return true; + } + + // A wrapper to ensure a non-null currentBuffer value on the first call. + HintsBuffer currentBuffer() + { + if (currentBuffer == null) + initializeCurrentBuffer(); + + return currentBuffer; + } + + private synchronized void initializeCurrentBuffer() + { + if (currentBuffer == null) + currentBuffer = createBuffer(); + } + + private synchronized boolean switchCurrentBuffer(HintsBuffer previous) + { + if (currentBuffer != previous) + return false; + + HintsBuffer buffer = reserveBuffers.poll(); + currentBuffer = buffer == null ? createBuffer() : buffer; + + return true; + } + + private HintsBuffer createBuffer() + { + return HintsBuffer.create(bufferSize); + } +} diff --git a/src/java/org/apache/cassandra/hints/HintsCatalog.java b/src/java/org/apache/cassandra/hints/HintsCatalog.java new file mode 100644 index 0000000000..13404ee6c6 --- /dev/null +++ b/src/java/org/apache/cassandra/hints/HintsCatalog.java @@ -0,0 +1,128 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.hints; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Stream; + +import org.apache.cassandra.io.FSReadError; +import org.apache.cassandra.utils.CLibrary; +import org.apache.cassandra.utils.SyncUtil; + +import static java.util.stream.Collectors.groupingBy; + +/** + * A simple catalog for easy host id -> {@link HintsStore} lookup and manipulation. + */ +final class HintsCatalog +{ + private final File hintsDirectory; + private final Map stores; + + private HintsCatalog(File hintsDirectory, Map> descriptors) + { + this.hintsDirectory = hintsDirectory; + this.stores = new ConcurrentHashMap<>(); + + for (Map.Entry> entry : descriptors.entrySet()) + stores.put(entry.getKey(), HintsStore.create(entry.getKey(), hintsDirectory, entry.getValue())); + } + + /** + * Loads hints stores from a given directory. + */ + static HintsCatalog load(File hintsDirectory) + { + try + { + Map> stores = + Files.list(hintsDirectory.toPath()) + .filter(HintsDescriptor::isHintFileName) + .map(HintsDescriptor::readFromFile) + .collect(groupingBy(h -> h.hostId)); + return new HintsCatalog(hintsDirectory, stores); + } + catch (IOException e) + { + throw new FSReadError(e, hintsDirectory); + } + } + + Stream stores() + { + return stores.values().stream(); + } + + void maybeLoadStores(Iterable hostIds) + { + for (UUID hostId : hostIds) + get(hostId); + } + + HintsStore get(UUID hostId) + { + // we intentionally don't just return stores.computeIfAbsent() because it's expensive compared to simple get(), + // and in this case would also allocate for the capturing lambda; the method is on a really hot path + HintsStore store = stores.get(hostId); + return store == null + ? stores.computeIfAbsent(hostId, (id) -> HintsStore.create(id, hintsDirectory, Collections.emptyList())) + : store; + } + + /** + * Delete all hints for all host ids. + * + * Will not delete the files that are currently being dispatched, or written to. + */ + void deleteAllHints() + { + stores.keySet().forEach(this::deleteAllHints); + } + + /** + * Delete all hints for the specified host id. + * + * Will not delete the files that are currently being dispatched, or written to. + */ + void deleteAllHints(UUID hostId) + { + HintsStore store = stores.get(hostId); + if (store != null) + store.deleteAllHints(); + } + + void exciseStore(UUID hostId) + { + deleteAllHints(hostId); + stores.remove(hostId); + } + + void fsyncDirectory() + { + int fd = CLibrary.tryOpenDirectory(hintsDirectory.getAbsolutePath()); + if (fd != -1) + { + SyncUtil.trySync(fd); + CLibrary.tryCloseFD(fd); + } + } +} diff --git a/src/java/org/apache/cassandra/hints/HintsDescriptor.java b/src/java/org/apache/cassandra/hints/HintsDescriptor.java new file mode 100644 index 0000000000..9c27a232bf --- /dev/null +++ b/src/java/org/apache/cassandra/hints/HintsDescriptor.java @@ -0,0 +1,242 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.hints; + +import java.io.DataInput; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.util.Map; +import java.util.UUID; +import java.util.regex.Pattern; +import java.util.zip.CRC32; + +import com.google.common.base.MoreObjects; +import com.google.common.base.Objects; +import com.google.common.collect.ImmutableMap; + +import org.apache.cassandra.db.TypeSizes; +import org.apache.cassandra.io.FSReadError; +import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.net.MessagingService; +import org.json.simple.JSONValue; + +import static org.apache.cassandra.utils.FBUtilities.updateChecksumInt; + +/** + * Describes the host id, the version, the timestamp of creation, and an arbitrary map of JSON-encoded parameters of a + * hints file. + * + * Written in the beginning of each hints file. + */ +final class HintsDescriptor +{ + static final int VERSION_30 = 1; + static final int CURRENT_VERSION = VERSION_30; + + static final Pattern pattern = + Pattern.compile("^[a-fA-F0-9]{8}\\-[a-fA-F0-9]{4}\\-[a-fA-F0-9]{4}\\-[a-fA-F0-9]{4}\\-[a-fA-F0-9]{12}\\-(\\d+)\\-(\\d+)\\.hints$"); + + final UUID hostId; + final int version; + final long timestamp; + + // implemented for future compression support - see CASSANDRA-9428 + final ImmutableMap parameters; + + HintsDescriptor(UUID hostId, int version, long timestamp, ImmutableMap parameters) + { + this.hostId = hostId; + this.version = version; + this.timestamp = timestamp; + this.parameters = parameters; + } + + HintsDescriptor(UUID hostId, long timestamp) + { + this(hostId, CURRENT_VERSION, timestamp, ImmutableMap.of()); + } + + String fileName() + { + return String.format("%s-%s-%s.hints", hostId, timestamp, version); + } + + String checksumFileName() + { + return String.format("%s-%s-%s.crc32", hostId, timestamp, version); + } + + int messagingVersion() + { + return messagingVersion(version); + } + + static int messagingVersion(int hintsVersion) + { + switch (hintsVersion) + { + case VERSION_30: + return MessagingService.VERSION_30; + default: + throw new AssertionError(); + } + } + + static boolean isHintFileName(Path path) + { + return pattern.matcher(path.getFileName().toString()).matches(); + } + + static HintsDescriptor readFromFile(Path path) + { + try (RandomAccessFile raf = new RandomAccessFile(path.toFile(), "r")) + { + return deserialize(raf); + } + catch (IOException e) + { + throw new FSReadError(e, path.toFile()); + } + } + + @Override + public String toString() + { + return MoreObjects.toStringHelper(this) + .add("hostId", hostId) + .add("version", version) + .add("timestamp", timestamp) + .add("parameters", parameters) + .toString(); + } + + @Override + public boolean equals(Object o) + { + if (this == o) + return true; + + if (!(o instanceof HintsDescriptor)) + return false; + + HintsDescriptor hd = (HintsDescriptor) o; + + return Objects.equal(hostId, hd.hostId) + && Objects.equal(version, hd.version) + && Objects.equal(timestamp, hd.timestamp) + && Objects.equal(parameters, hd.parameters); + } + + @Override + public int hashCode() + { + return Objects.hashCode(hostId, version, timestamp, parameters); + } + + void serialize(DataOutputPlus out) throws IOException + { + CRC32 crc = new CRC32(); + + out.writeInt(version); + updateChecksumInt(crc, version); + + out.writeLong(timestamp); + updateChecksumLong(crc, timestamp); + + out.writeLong(hostId.getMostSignificantBits()); + updateChecksumLong(crc, hostId.getMostSignificantBits()); + out.writeLong(hostId.getLeastSignificantBits()); + updateChecksumLong(crc, hostId.getLeastSignificantBits()); + + byte[] paramsBytes = JSONValue.toJSONString(parameters).getBytes(StandardCharsets.UTF_8); + out.writeInt(paramsBytes.length); + updateChecksumInt(crc, paramsBytes.length); + out.writeInt((int) crc.getValue()); + + out.write(paramsBytes); + crc.update(paramsBytes, 0, paramsBytes.length); + + out.writeInt((int) crc.getValue()); + } + + int serializedSize() + { + int size = TypeSizes.sizeof(version); + size += TypeSizes.sizeof(timestamp); + + size += TypeSizes.sizeof(hostId.getMostSignificantBits()); + size += TypeSizes.sizeof(hostId.getLeastSignificantBits()); + + byte[] paramsBytes = JSONValue.toJSONString(parameters).getBytes(StandardCharsets.UTF_8); + size += TypeSizes.sizeof(paramsBytes.length); + size += 4; // size checksum + size += paramsBytes.length; + size += 4; // total checksum + + return size; + } + + static HintsDescriptor deserialize(DataInput in) throws IOException + { + CRC32 crc = new CRC32(); + + int version = in.readInt(); + updateChecksumInt(crc, version); + + long timestamp = in.readLong(); + updateChecksumLong(crc, timestamp); + + long msb = in.readLong(); + updateChecksumLong(crc, msb); + long lsb = in.readLong(); + updateChecksumLong(crc, lsb); + + UUID hostId = new UUID(msb, lsb); + + int paramsLength = in.readInt(); + updateChecksumInt(crc, paramsLength); + validateCRC(in.readInt(), (int) crc.getValue()); + + byte[] paramsBytes = new byte[paramsLength]; + in.readFully(paramsBytes, 0, paramsLength); + crc.update(paramsBytes, 0, paramsLength); + validateCRC(in.readInt(), (int) crc.getValue()); + + return new HintsDescriptor(hostId, version, timestamp, decodeJSONBytes(paramsBytes)); + } + + @SuppressWarnings("unchecked") + private static ImmutableMap decodeJSONBytes(byte[] bytes) + { + return ImmutableMap.copyOf((Map) JSONValue.parse(new String(bytes, StandardCharsets.UTF_8))); + } + + private static void updateChecksumLong(CRC32 crc, long value) + { + updateChecksumInt(crc, (int) (value & 0xFFFFFFFFL)); + updateChecksumInt(crc, (int) (value >>> 32)); + } + + private static void validateCRC(int expected, int actual) throws IOException + { + if (expected != actual) + throw new IOException("Hints Descriptor CRC Mismatch"); + } +} diff --git a/src/java/org/apache/cassandra/hints/HintsDispatchExecutor.java b/src/java/org/apache/cassandra/hints/HintsDispatchExecutor.java new file mode 100644 index 0000000000..d0fdd04742 --- /dev/null +++ b/src/java/org/apache/cassandra/hints/HintsDispatchExecutor.java @@ -0,0 +1,199 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.hints; + +import java.io.File; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicBoolean; + +import com.google.common.util.concurrent.RateLimiter; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.concurrent.JMXEnabledThreadPoolExecutor; +import org.apache.cassandra.concurrent.NamedThreadFactory; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.io.FSReadError; +import org.apache.cassandra.service.StorageService; + +/** + * A multi-threaded (by default) executor for dispatching hints. + * + * Most of dispatch is triggered by {@link HintsDispatchTrigger} running every ~10 seconds. + */ +final class HintsDispatchExecutor +{ + private static final Logger logger = LoggerFactory.getLogger(HintsDispatchExecutor.class); + + private final File hintsDirectory; + private final ExecutorService executor; + private final AtomicBoolean isPaused; + private final Map scheduledDispatches; + + HintsDispatchExecutor(File hintsDirectory, int maxThreads, AtomicBoolean isPaused) + { + this.hintsDirectory = hintsDirectory; + this.isPaused = isPaused; + + scheduledDispatches = new ConcurrentHashMap<>(); + executor = new JMXEnabledThreadPoolExecutor(1, + maxThreads, + 1, + TimeUnit.MINUTES, + new LinkedBlockingQueue<>(), + new NamedThreadFactory("HintsDispatcher", Thread.MIN_PRIORITY), + "internal"); + } + + /* + * It's safe to terminate dispatch in process and to deschedule dispatch. + */ + void shutdownBlocking() + { + scheduledDispatches.clear(); + executor.shutdownNow(); + } + + boolean isScheduled(HintsStore store) + { + return scheduledDispatches.containsKey(store.hostId); + } + + Future dispatch(HintsStore store) + { + return dispatch(store, store.hostId); + } + + Future dispatch(HintsStore store, UUID hostId) + { + /* + * It is safe to perform dispatch for the same host id concurrently in two or more threads, + * however there is nothing to win from it - so we don't. + * + * Additionally, having just one dispatch task per host id ensures that we'll never violate our per-destination + * rate limit, without having to share a ratelimiter between threads. + * + * It also simplifies reasoning about dispatch sessions. + */ + return scheduledDispatches.computeIfAbsent(store.hostId, uuid -> executor.submit(new DispatchHintsTask(store, hostId))); + } + + void completeDispatchBlockingly(HintsStore store) + { + Future future = scheduledDispatches.get(store.hostId); + try + { + if (future != null) + future.get(); + } + catch (ExecutionException | InterruptedException e) + { + throw new RuntimeException(e); + } + } + + private final class DispatchHintsTask implements Runnable + { + private final HintsStore store; + private final UUID hostId; + private final RateLimiter rateLimiter; + + DispatchHintsTask(HintsStore store, UUID hostId) + { + this.store = store; + this.hostId = hostId; + + // rate limit is in bytes per second. Uses Double.MAX_VALUE if disabled (set to 0 in cassandra.yaml). + // max rate is scaled by the number of nodes in the cluster (CASSANDRA-5272). + // the goal is to bound maximum hints traffic going towards a particular node from the rest of the cluster, + // not total outgoing hints traffic from this node - this is why the rate limiter is not shared between + // all the dispatch tasks (as there will be at most one dispatch task for a particular host id at a time). + int nodesCount = Math.max(1, StorageService.instance.getTokenMetadata().getAllEndpoints().size() - 1); + int throttleInKB = DatabaseDescriptor.getHintedHandoffThrottleInKB() / nodesCount; + this.rateLimiter = RateLimiter.create(throttleInKB == 0 ? Double.MAX_VALUE : throttleInKB * 1024); + } + + public void run() + { + try + { + dispatch(); + } + finally + { + scheduledDispatches.remove(hostId); + } + } + + private void dispatch() + { + while (true) + { + if (isPaused.get()) + break; + + HintsDescriptor descriptor = store.poll(); + if (descriptor == null) + break; + + try + { + dispatch(descriptor); + } + catch (FSReadError e) + { + logger.error("Failed to dispatch hints file {}: file is corrupted ({})", descriptor.fileName(), e); + store.cleanUp(descriptor); + store.blacklist(descriptor); + throw e; + } + } + } + + private void dispatch(HintsDescriptor descriptor) + { + logger.debug("Dispatching hints file {}", descriptor.fileName()); + + File file = new File(hintsDirectory, descriptor.fileName()); + Long offset = store.getDispatchOffset(descriptor).orElse(null); + + try (HintsDispatcher dispatcher = HintsDispatcher.create(file, rateLimiter, hostId, isPaused)) + { + if (offset != null) + dispatcher.seek(offset); + + if (dispatcher.dispatch()) + { + if (!file.delete()) + logger.error("Failed to delete hints file {}", descriptor.fileName()); + store.cleanUp(descriptor); + logger.info("Finished hinted handoff of file {} to endpoint {}", descriptor.fileName(), hostId); + } + else + { + store.markDispatchOffset(descriptor, dispatcher.dispatchOffset()); + store.offerFirst(descriptor); + logger.info("Finished hinted handoff of file {} to endpoint {}, partially", descriptor.fileName(), hostId); + } + } + } + } +} diff --git a/src/java/org/apache/cassandra/hints/HintsDispatchTrigger.java b/src/java/org/apache/cassandra/hints/HintsDispatchTrigger.java new file mode 100644 index 0000000000..5fe0e27b70 --- /dev/null +++ b/src/java/org/apache/cassandra/hints/HintsDispatchTrigger.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.hints; + +import java.util.concurrent.atomic.AtomicBoolean; + +import org.apache.cassandra.gms.ApplicationState; +import org.apache.cassandra.gms.FailureDetector; +import org.apache.cassandra.gms.Gossiper; + +import static org.apache.cassandra.utils.FBUtilities.getBroadcastAddress; + +/** + * A simple dispatch trigger that's being run every 10 seconds. + * + * Goes through all hint stores and schedules for dispatch all the hints for hosts that are: + * 1. Not currently scheduled for dispatch, and + * 2. Either have some hint files, or an active hint writer, and + * 3. Are live, and + * 4. Have matching schema versions + * + * What does triggering a hints store for dispatch mean? + * - If there are existing hint files, it means submitting them for dispatch; + * - If there is an active writer, closing it, for the next run to pick it up. + */ +final class HintsDispatchTrigger implements Runnable +{ + private final HintsCatalog catalog; + private final HintsWriteExecutor writeExecutor; + private final HintsDispatchExecutor dispatchExecutor; + private final AtomicBoolean isPaused; + + HintsDispatchTrigger(HintsCatalog catalog, + HintsWriteExecutor writeExecutor, + HintsDispatchExecutor dispatchExecutor, + AtomicBoolean isPaused) + { + this.catalog = catalog; + this.writeExecutor = writeExecutor; + this.dispatchExecutor = dispatchExecutor; + this.isPaused = isPaused; + } + + public void run() + { + if (isPaused.get()) + return; + + catalog.stores() + .filter(store -> !isScheduled(store)) + .filter(HintsStore::isLive) + .filter(store -> store.isWriting() || store.hasFiles()) + .filter(store -> Gossiper.instance.valuesEqual(getBroadcastAddress(), store.address(), ApplicationState.SCHEMA)) + .forEach(this::schedule); + } + + private void schedule(HintsStore store) + { + if (store.hasFiles()) + dispatchExecutor.dispatch(store); + + if (store.isWriting()) + writeExecutor.closeWriter(store); + } + + private boolean isScheduled(HintsStore store) + { + return dispatchExecutor.isScheduled(store); + } +} diff --git a/src/java/org/apache/cassandra/hints/HintsDispatcher.java b/src/java/org/apache/cassandra/hints/HintsDispatcher.java new file mode 100644 index 0000000000..f769e09b5b --- /dev/null +++ b/src/java/org/apache/cassandra/hints/HintsDispatcher.java @@ -0,0 +1,228 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.hints; + +import java.io.File; +import java.net.InetAddress; +import java.nio.ByteBuffer; +import java.util.*; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Function; + +import com.google.common.util.concurrent.RateLimiter; + +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.gms.FailureDetector; +import org.apache.cassandra.net.IAsyncCallbackWithFailure; +import org.apache.cassandra.net.MessageIn; +import org.apache.cassandra.net.MessagingService; +import org.apache.cassandra.service.StorageService; +import org.apache.cassandra.utils.concurrent.SimpleCondition; + +/** + * Dispatches a single hints file to a specified node in a batched manner. + * + * Uses either {@link EncodedHintMessage} - when dispatching hints into a node with the same messaging version as the hints file, + * or {@link HintMessage}, when conversion is required. + */ +final class HintsDispatcher implements AutoCloseable +{ + private enum Action { CONTINUE, ABORT, RETRY } + + private final HintsReader reader; + private final UUID hostId; + private final InetAddress address; + private final int messagingVersion; + private final AtomicBoolean isPaused; + + private long currentPageOffset; + + private HintsDispatcher(HintsReader reader, UUID hostId, InetAddress address, int messagingVersion, AtomicBoolean isPaused) + { + currentPageOffset = 0L; + + this.reader = reader; + this.hostId = hostId; + this.address = address; + this.messagingVersion = messagingVersion; + this.isPaused = isPaused; + } + + static HintsDispatcher create(File file, RateLimiter rateLimiter, UUID hostId, AtomicBoolean isPaused) + { + InetAddress address = StorageService.instance.getEndpointForHostId(hostId); + int messagingVersion = MessagingService.instance().getVersion(address); + return new HintsDispatcher(HintsReader.open(file, rateLimiter), hostId, address, messagingVersion, isPaused); + } + + public void close() + { + reader.close(); + } + + void seek(long bytes) + { + reader.seek(bytes); + currentPageOffset = 0L; + } + + /** + * @return whether or not dispatch completed entirely and successfully + */ + boolean dispatch() + { + for (HintsReader.Page page : reader) + { + currentPageOffset = page.offset; + if (dispatch(page) != Action.CONTINUE) + return false; + } + + return true; + } + + /** + * @return offset of the first non-delivered page + */ + long dispatchOffset() + { + return currentPageOffset; + } + + private boolean isHostAlive() + { + return FailureDetector.instance.isAlive(address); + } + + private boolean isPaused() + { + return isPaused.get(); + } + + // retry in case of a timeout; stop in case of a failure, host going down, or delivery paused + private Action dispatch(HintsReader.Page page) + { + Action action = sendHintsAndAwait(page); + return action == Action.RETRY + ? dispatch(page) + : action; + } + + private Action sendHintsAndAwait(HintsReader.Page page) + { + Collection callbacks = new ArrayList<>(); + + /* + * If hints file messaging version matches the version of the target host, we'll use the optimised path - + * skipping the redundant decoding/encoding cycle of the already encoded hint. + * + * If that is not the case, we'll need to perform conversion to a newer (or an older) format, and decoding the hint + * is an unavoidable intermediate step. + */ + Action action = reader.descriptor().messagingVersion() == messagingVersion + ? sendHints(page.buffersIterator(), callbacks, this::sendEncodedHint) + : sendHints(page.hintsIterator(), callbacks, this::sendHint); + + if (action == Action.ABORT) + return action; + + for (Callback cb : callbacks) + if (cb.await() != Callback.Outcome.SUCCESS) + return Action.RETRY; + + return Action.CONTINUE; + } + + /* + * Sending hints in compatibility mode. + */ + private Action sendHints(Iterator hints, Collection callbacks, Function sendFunction) + { + while (hints.hasNext()) + { + if (!isHostAlive() || isPaused()) + return Action.ABORT; + callbacks.add(sendFunction.apply(hints.next())); + } + return Action.CONTINUE; + } + + private Callback sendHint(Hint hint) + { + Callback callback = new Callback(); + HintMessage message = new HintMessage(hostId, hint); + MessagingService.instance().sendRRWithFailure(message.createMessageOut(), address, callback); + return callback; + } + + /* + * Sending hints in raw mode. + */ + + private Callback sendEncodedHint(ByteBuffer hint) + { + Callback callback = new Callback(); + EncodedHintMessage message = new EncodedHintMessage(hostId, hint, messagingVersion); + MessagingService.instance().sendRRWithFailure(message.createMessageOut(), address, callback); + return callback; + } + + private static final class Callback implements IAsyncCallbackWithFailure + { + enum Outcome { SUCCESS, TIMEOUT, FAILURE } + + private final long start = System.nanoTime(); + private final SimpleCondition condition = new SimpleCondition(); + private volatile Outcome outcome; + + Outcome await() + { + long timeout = TimeUnit.MILLISECONDS.toNanos(DatabaseDescriptor.getTimeout(MessagingService.Verb.HINT)) - (System.nanoTime() - start); + boolean timedOut; + + try + { + timedOut = !condition.await(timeout, TimeUnit.NANOSECONDS); + } + catch (InterruptedException e) + { + throw new AssertionError(e); + } + + return timedOut ? Outcome.TIMEOUT : outcome; + } + + public void onFailure(InetAddress from) + { + outcome = Outcome.FAILURE; + condition.signalAll(); + } + + public void response(MessageIn msg) + { + outcome = Outcome.SUCCESS; + condition.signalAll(); + } + + public boolean isLatencyForSnitch() + { + return false; + } + } +} diff --git a/src/java/org/apache/cassandra/hints/HintsReader.java b/src/java/org/apache/cassandra/hints/HintsReader.java new file mode 100644 index 0000000000..7d164b454f --- /dev/null +++ b/src/java/org/apache/cassandra/hints/HintsReader.java @@ -0,0 +1,312 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.hints; + +import java.io.File; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Iterator; + +import javax.annotation.Nullable; + +import com.google.common.util.concurrent.RateLimiter; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.db.UnknownColumnFamilyException; +import org.apache.cassandra.io.FSReadError; +import org.apache.cassandra.io.util.FileUtils; +import org.apache.cassandra.io.util.RandomAccessReader; +import org.apache.cassandra.utils.AbstractIterator; +import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.utils.CLibrary; + +/** + * A paged non-compressed hints reader that provides two iterators: + * - a 'raw' ByteBuffer iterator that doesn't deserialize the hints, but returns the pre-encoded hints verbatim + * - a decoded iterator, that deserializes the underlying bytes into {@link Hint} instances. + * + * The former is an optimisation for when the messaging version of the file matches the messaging version of the destination + * node. Extra decoding and reencoding is a waste of effort in this scenario, so we avoid it. + * + * The latter is required for dispatch of hints to nodes that have a different messaging version, and in general is just an + * easy way to enable backward and future compatibilty. + */ +final class HintsReader implements AutoCloseable, Iterable +{ + private static final Logger logger = LoggerFactory.getLogger(HintsReader.class); + + // don't read more than 512 KB of hints at a time. + private static final int PAGE_SIZE = 512 << 10; + + private final HintsDescriptor descriptor; + private final File file; + private final RandomAccessReader reader; + private final ChecksummedDataInput crcInput; + + // we pass the RateLimiter into HintsReader itself because it's cheaper to calculate the size before the hint is deserialized + @Nullable + private final RateLimiter rateLimiter; + + private HintsReader(HintsDescriptor descriptor, File file, RandomAccessReader reader, RateLimiter rateLimiter) + { + this.descriptor = descriptor; + this.file = file; + this.reader = reader; + this.crcInput = ChecksummedDataInput.wrap(reader); + this.rateLimiter = rateLimiter; + } + + static HintsReader open(File file, RateLimiter rateLimiter) + { + RandomAccessReader reader = RandomAccessReader.open(file); + try + { + HintsDescriptor descriptor = HintsDescriptor.deserialize(reader); + return new HintsReader(descriptor, file, reader, rateLimiter); + } + catch (IOException e) + { + reader.close(); + throw new FSReadError(e, file); + } + } + + static HintsReader open(File file) + { + return open(file, null); + } + + public void close() + { + FileUtils.closeQuietly(reader); + } + + public HintsDescriptor descriptor() + { + return descriptor; + } + + void seek(long newPosition) + { + reader.seek(newPosition); + } + + public Iterator iterator() + { + return new PagesIterator(); + } + + final class Page + { + public final long offset; + + private Page(long offset) + { + this.offset = offset; + } + + Iterator hintsIterator() + { + return new HintsIterator(offset); + } + + Iterator buffersIterator() + { + return new BuffersIterator(offset); + } + } + + final class PagesIterator extends AbstractIterator + { + @SuppressWarnings("resource") + protected Page computeNext() + { + CLibrary.trySkipCache(reader.getChannel().getFileDescriptor(), 0, reader.getFilePointer(), reader.getPath()); + + if (reader.length() == reader.getFilePointer()) + return endOfData(); + + return new Page(reader.getFilePointer()); + } + } + + /** + * A decoding iterator that deserializes the hints as it goes. + */ + final class HintsIterator extends AbstractIterator + { + private final long offset; + + HintsIterator(long offset) + { + super(); + this.offset = offset; + } + + protected Hint computeNext() + { + Hint hint; + + do + { + long position = reader.getFilePointer(); + + if (reader.length() == position) + return endOfData(); // reached EOF + + if (position - offset >= PAGE_SIZE) + return endOfData(); // read page size or more bytes + + try + { + hint = computeNextInternal(); + } + catch (IOException e) + { + throw new FSReadError(e, file); + } + } + while (hint == null); + + return hint; + } + + private Hint computeNextInternal() throws IOException + { + crcInput.resetCrc(); + crcInput.resetLimit(); + + int size = crcInput.readInt(); + + // if we cannot corroborate the size via crc, then we cannot safely skip this hint + if (reader.readInt() != crcInput.getCrc()) + throw new IOException("Digest mismatch exception"); + + return readHint(size); + } + + private Hint readHint(int size) throws IOException + { + if (rateLimiter != null) + rateLimiter.acquire(size); + crcInput.limit(size); + + Hint hint; + try + { + hint = Hint.serializer.deserialize(crcInput, descriptor.messagingVersion()); + } + catch (UnknownColumnFamilyException e) + { + logger.warn("Failed to read a hint for {} - table with id {} is unknown in file {}", + descriptor.hostId, + e.cfId, + descriptor.fileName()); + reader.skipBytes(crcInput.bytesRemaining()); + + return null; + } + + if (reader.readInt() == crcInput.getCrc()) + return hint; + + // log a warning and skip the corrupted entry + logger.warn("Failed to read a hint for {} - digest mismatch for hint at position {} in file {}", + descriptor.hostId, + crcInput.getPosition() - size - 4, + descriptor.fileName()); + return null; + } + } + + /** + * A verbatim iterator that simply returns the underlying ByteBuffers. + */ + final class BuffersIterator extends AbstractIterator + { + private final long offset; + + BuffersIterator(long offset) + { + super(); + this.offset = offset; + } + + protected ByteBuffer computeNext() + { + ByteBuffer buffer; + + do + { + long position = reader.getFilePointer(); + + if (reader.length() == position) + return endOfData(); // reached EOF + + if (position - offset >= PAGE_SIZE) + return endOfData(); // read page size or more bytes + + try + { + buffer = computeNextInternal(); + } + catch (IOException e) + { + throw new FSReadError(e, file); + } + } + while (buffer == null); + + return buffer; + } + + private ByteBuffer computeNextInternal() throws IOException + { + crcInput.resetCrc(); + crcInput.resetLimit(); + + int size = crcInput.readInt(); + + // if we cannot corroborate the size via crc, then we cannot safely skip this hint + if (reader.readInt() != crcInput.getCrc()) + throw new IOException("Digest mismatch exception"); + + return readBuffer(size); + } + + private ByteBuffer readBuffer(int size) throws IOException + { + if (rateLimiter != null) + rateLimiter.acquire(size); + crcInput.limit(size); + + ByteBuffer buffer = ByteBufferUtil.read(crcInput, size); + if (reader.readInt() == crcInput.getCrc()) + return buffer; + + // log a warning and skip the corrupted entry + logger.warn("Failed to read a hint for {} - digest mismatch for hint at position {} in file {}", + descriptor.hostId, + crcInput.getPosition() - size - 4, + descriptor.fileName()); + return null; + } + } +} diff --git a/src/java/org/apache/cassandra/hints/HintsService.java b/src/java/org/apache/cassandra/hints/HintsService.java new file mode 100644 index 0000000000..3f30c1d567 --- /dev/null +++ b/src/java/org/apache/cassandra/hints/HintsService.java @@ -0,0 +1,291 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.hints; + +import java.io.File; +import java.lang.management.ManagementFactory; +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.Collections; +import java.util.UUID; +import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicBoolean; + +import javax.management.MBeanServer; +import javax.management.ObjectName; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.concurrent.ScheduledExecutors; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.metrics.HintedHandoffMetrics; +import org.apache.cassandra.metrics.StorageMetrics; +import org.apache.cassandra.service.StorageService; + +import static com.google.common.collect.Iterables.transform; +import static com.google.common.collect.Iterables.size; + +/** + * A singleton-ish wrapper over various hints components: + * - a catalog of all hints stores + * - a single-threaded write executor + * - a multi-threaded dispatch executor + * - the buffer pool for writing hints into + * + * The front-end for everything hints related. + */ +public final class HintsService implements HintsServiceMBean +{ + private static final Logger logger = LoggerFactory.getLogger(HintsService.class); + + public static final HintsService instance = new HintsService(); + + private static final String MBEAN_NAME = "org.apache.cassandra.hints:type=HintsService"; + + private static final int MIN_BUFFER_SIZE = 32 << 20; + + private final HintsCatalog catalog; + private final HintsWriteExecutor writeExecutor; + private final HintsBufferPool bufferPool; + private final HintsDispatchExecutor dispatchExecutor; + private final AtomicBoolean isDispatchPaused; + + private volatile boolean isShutDown = false; + + private final ScheduledFuture triggerFlushingFuture; + private volatile ScheduledFuture triggerDispatchFuture; + + public final HintedHandoffMetrics metrics; + + private HintsService() + { + File hintsDirectory = DatabaseDescriptor.getHintsDirectory(); + int maxDeliveryThreads = DatabaseDescriptor.getMaxHintsDeliveryThreads(); + + catalog = HintsCatalog.load(hintsDirectory); + writeExecutor = new HintsWriteExecutor(catalog); + + int bufferSize = Math.max(DatabaseDescriptor.getMaxMutationSize() * 2, MIN_BUFFER_SIZE); + bufferPool = new HintsBufferPool(bufferSize, writeExecutor::flushBuffer); + + isDispatchPaused = new AtomicBoolean(true); + dispatchExecutor = new HintsDispatchExecutor(hintsDirectory, maxDeliveryThreads, isDispatchPaused); + + // periodically empty the current content of the buffers + int flushPeriod = DatabaseDescriptor.getHintsFlushPeriodInMS(); + triggerFlushingFuture = ScheduledExecutors.optionalTasks.scheduleWithFixedDelay(() -> writeExecutor.flushBufferPool(bufferPool), + flushPeriod, + flushPeriod, + TimeUnit.MILLISECONDS); + metrics = new HintedHandoffMetrics(); + } + + public void registerMBean() + { + MBeanServer mbs = ManagementFactory.getPlatformMBeanServer(); + try + { + mbs.registerMBean(this, new ObjectName(MBEAN_NAME)); + } + catch (Exception e) + { + throw new RuntimeException(e); + } + } + + /** + * Write a hint for a iterable of nodes. + * + * @param hostIds host ids of the hint's target nodes + * @param hint the hint to store + */ + public void write(Iterable hostIds, Hint hint) + { + if (isShutDown) + throw new IllegalStateException("HintsService is shut down and can't accept new hints"); + + // we have to make sure that the HintsStore instances get properly initialized - otherwise dispatch will not trigger + catalog.maybeLoadStores(hostIds); + + if (hint.isLive()) + bufferPool.write(hostIds, hint); + + StorageMetrics.totalHints.inc(size(hostIds)); + } + + /** + * Write a hint for a single node. + * + * @param hostId host id of the hint's target node + * @param hint the hint to store + */ + public void write(UUID hostId, Hint hint) + { + write(Collections.singleton(hostId), hint); + } + + /** + * Flush the buffer pool for the selected target nodes, then fsync their writers. + * + * @param hostIds host ids of the nodes to flush and fsync hints for + */ + public void flushAndFsyncBlockingly(Iterable hostIds) + { + Iterable stores = transform(hostIds, catalog::get); + writeExecutor.flushBufferPool(bufferPool, stores); + writeExecutor.fsyncWritersBlockingly(stores); + } + + public synchronized void startDispatch() + { + if (isShutDown) + throw new IllegalStateException("HintsService is shut down and cannot be restarted"); + + isDispatchPaused.set(false); + + HintsDispatchTrigger trigger = new HintsDispatchTrigger(catalog, writeExecutor, dispatchExecutor, isDispatchPaused); + // triggering hint dispatch is now very cheap, so we can do it more often - every 10 seconds vs. every 10 minutes, + // previously; this reduces mean time to delivery, and positively affects batchlog delivery latencies, too + triggerDispatchFuture = ScheduledExecutors.scheduledTasks.scheduleWithFixedDelay(trigger, 10, 10, TimeUnit.SECONDS); + } + + public void pauseDispatch() + { + logger.info("Paused hints dispatch"); + isDispatchPaused.set(true); + } + + public void resumeDispatch() + { + logger.info("Resumed hints dispatch"); + isDispatchPaused.set(false); + } + + /** + * Gracefully and blockingly shut down the service. + * + * Will abort dispatch sessions that are currently in progress (which is okay, it's idempotent), + * and make sure the buffers are flushed, hints files written and fsynced. + */ + public synchronized void shutdownBlocking() + { + if (isShutDown) + throw new IllegalStateException("HintsService has already been shut down"); + isShutDown = true; + + if (triggerDispatchFuture != null) + triggerDispatchFuture.cancel(false); + pauseDispatch(); + + triggerFlushingFuture.cancel(false); + + writeExecutor.flushBufferPool(bufferPool); + writeExecutor.closeAllWriters(); + + dispatchExecutor.shutdownBlocking(); + writeExecutor.shutdownBlocking(); + } + + public void decommission() + { + resumeDispatch(); + } + + /** + * Deletes all hints for all destinations. Doesn't make snapshots - should be used with care. + */ + public void deleteAllHints() + { + catalog.deleteAllHints(); + } + + /** + * Deletes all hints for the provided destination. Doesn't make snapshots - should be used with care. + * + * @param address inet address of the target node - encoded as a string for easier JMX consumption + */ + public void deleteAllHintsForEndpoint(String address) + { + InetAddress target; + try + { + target = InetAddress.getByName(address); + } + catch (UnknownHostException e) + { + throw new IllegalArgumentException(e); + } + deleteAllHintsForEndpoint(target); + } + + /** + * Deletes all hints for the provided destination. Doesn't make snapshots - should be used with care. + * + * @param target inet address of the target node + */ + public void deleteAllHintsForEndpoint(InetAddress target) + { + UUID hostId = StorageService.instance.getHostIdForEndpoint(target); + if (hostId == null) + throw new IllegalArgumentException("Can't delete hints for unknown address " + target); + catalog.deleteAllHints(hostId); + } + + /** + * Cleans up hints-related state after a node with id = hostId left. + * + * Dispatcher should stop itself (isHostAlive() will start returning false for the leaving host), but we'll wait for + * completion anyway. + * + * We should also flush the buffer is there are any thints for the node there, and close the writer (if any), + * so that we don't leave any hint files lying around. + * + * Once that is done, we can simply delete all hint files and remove the host id from the catalog. + * + * The worst that can happen if we don't get everything right is a hints file (or two) remaining undeleted. + * + * @param hostId id of the node being excised + */ + public void excise(UUID hostId) + { + HintsStore store = catalog.get(hostId); + if (store == null) + return; + + // flush the buffer and then close the writer for the excised host id, to make sure that no new files will appear + // for this host id after we are done + Future flushFuture = writeExecutor.flushBufferPool(bufferPool, Collections.singleton(store)); + Future closeFuture = writeExecutor.closeWriter(store); + try + { + flushFuture.get(); + closeFuture.get(); + } + catch (InterruptedException | ExecutionException e) + { + throw new RuntimeException(e); + } + + // wait for the current dispatch session to end (if any), so that the currently dispatched file gets removed + dispatchExecutor.completeDispatchBlockingly(store); + + // delete all the hints files and remove the HintsStore instance from the map in the catalog + catalog.exciseStore(hostId); + } +} diff --git a/src/java/org/apache/cassandra/hints/HintsServiceMBean.java b/src/java/org/apache/cassandra/hints/HintsServiceMBean.java new file mode 100644 index 0000000000..fe0abcc186 --- /dev/null +++ b/src/java/org/apache/cassandra/hints/HintsServiceMBean.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.hints; + +public interface HintsServiceMBean +{ + /** + * Pause dispatch of all hints. Does not affect the creation of hints. + */ + void pauseDispatch(); + + /** + * Resume dispatch of all hints. Does not affect the creation of hints. + */ + void resumeDispatch(); + + /** + * Irrevocably deletes all the stored hints files (with the exception of those that are being dispatched right now, + * or being written to). + */ + void deleteAllHints(); + + /** + * Irrevocably deletes all the stored hints files for the target address (with the exception of those that are + * being dispatched right now, or being written to). + */ + void deleteAllHintsForEndpoint(String address); +} diff --git a/src/java/org/apache/cassandra/hints/HintsStore.java b/src/java/org/apache/cassandra/hints/HintsStore.java new file mode 100644 index 0000000000..e19de9951a --- /dev/null +++ b/src/java/org/apache/cassandra/hints/HintsStore.java @@ -0,0 +1,210 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.hints; + +import java.io.File; +import java.io.IOException; +import java.net.InetAddress; +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedDeque; +import java.util.concurrent.ConcurrentLinkedQueue; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.gms.FailureDetector; +import org.apache.cassandra.io.FSWriteError; +import org.apache.cassandra.service.StorageService; +import org.apache.cassandra.utils.SyncUtil; + +/** + * Encapsulates the state of a peer's hints: the queue of hints files for dispatch, and the current writer (if any). + * + * The queue for dispatch is multi-threading safe. + * + * The writer MUST only be accessed by {@link HintsWriteExecutor}. + */ +final class HintsStore +{ + private static final Logger logger = LoggerFactory.getLogger(HintsStore.class); + + public final UUID hostId; + private final File hintsDirectory; + + private final Map dispatchOffsets; + private final Deque dispatchDequeue; + private final Queue blacklistedFiles; + + // last timestamp used in a descriptor; make sure to not reuse the same timestamp for new descriptors. + private volatile long lastUsedTimestamp; + private volatile HintsWriter hintsWriter; + + private HintsStore(UUID hostId, File hintsDirectory, List descriptors) + { + this.hostId = hostId; + this.hintsDirectory = hintsDirectory; + + dispatchOffsets = new ConcurrentHashMap<>(); + dispatchDequeue = new ConcurrentLinkedDeque<>(descriptors); + blacklistedFiles = new ConcurrentLinkedQueue<>(); + + //noinspection resource + lastUsedTimestamp = descriptors.stream().mapToLong(d -> d.timestamp).max().orElse(0L); + } + + static HintsStore create(UUID hostId, File hintsDirectory, List descriptors) + { + descriptors.sort((d1, d2) -> Long.compare(d1.timestamp, d2.timestamp)); + return new HintsStore(hostId, hintsDirectory, descriptors); + } + + InetAddress address() + { + return StorageService.instance.getEndpointForHostId(hostId); + } + + boolean isLive() + { + InetAddress address = address(); + return address != null && FailureDetector.instance.isAlive(address); + } + + HintsDescriptor poll() + { + return dispatchDequeue.poll(); + } + + void offerFirst(HintsDescriptor descriptor) + { + dispatchDequeue.offerFirst(descriptor); + } + + void offerLast(HintsDescriptor descriptor) + { + dispatchDequeue.offerLast(descriptor); + } + + void deleteAllHints() + { + HintsDescriptor descriptor; + while ((descriptor = poll()) != null) + { + cleanUp(descriptor); + delete(descriptor); + } + + while ((descriptor = blacklistedFiles.poll()) != null) + { + cleanUp(descriptor); + delete(descriptor); + } + } + + private void delete(HintsDescriptor descriptor) + { + File hintsFile = new File(hintsDirectory, descriptor.fileName()); + if (hintsFile.delete()) + logger.info("Deleted hint file {}", descriptor.fileName()); + else + logger.error("Failed to delete hint file {}", descriptor.fileName()); + + //noinspection ResultOfMethodCallIgnored + new File(hintsDirectory, descriptor.checksumFileName()).delete(); + } + + boolean hasFiles() + { + return !dispatchDequeue.isEmpty(); + } + + Optional getDispatchOffset(HintsDescriptor descriptor) + { + return Optional.ofNullable(dispatchOffsets.get(descriptor)); + } + + void markDispatchOffset(HintsDescriptor descriptor, long mark) + { + dispatchOffsets.put(descriptor, mark); + } + + void cleanUp(HintsDescriptor descriptor) + { + dispatchOffsets.remove(descriptor); + } + + void blacklist(HintsDescriptor descriptor) + { + blacklistedFiles.add(descriptor); + } + + /* + * Methods dealing with HintsWriter. + * + * All of these, with the exception of isWriting(), are for exclusively single-threaded use by HintsWriteExecutor. + */ + + boolean isWriting() + { + return hintsWriter != null; + } + + HintsWriter getOrOpenWriter() + { + if (hintsWriter == null) + hintsWriter = openWriter(); + return hintsWriter; + } + + HintsWriter getWriter() + { + return hintsWriter; + } + + private HintsWriter openWriter() + { + lastUsedTimestamp = Math.max(System.currentTimeMillis(), lastUsedTimestamp + 1); + HintsDescriptor descriptor = new HintsDescriptor(hostId, lastUsedTimestamp); + + try + { + return HintsWriter.create(hintsDirectory, descriptor); + } + catch (IOException e) + { + throw new FSWriteError(e, descriptor.fileName()); + } + } + + void closeWriter() + { + if (hintsWriter != null) + { + hintsWriter.close(); + offerLast(hintsWriter.descriptor()); + hintsWriter = null; + SyncUtil.trySyncDir(hintsDirectory); + } + } + + void fsyncWriter() + { + if (hintsWriter != null) + hintsWriter.fsync(); + } +} diff --git a/src/java/org/apache/cassandra/hints/HintsWriteExecutor.java b/src/java/org/apache/cassandra/hints/HintsWriteExecutor.java new file mode 100644 index 0000000000..be52f925ee --- /dev/null +++ b/src/java/org/apache/cassandra/hints/HintsWriteExecutor.java @@ -0,0 +1,235 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.hints; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Iterator; +import java.util.concurrent.*; + +import org.apache.cassandra.concurrent.DebuggableThreadPoolExecutor; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.io.FSWriteError; + +/** + * A single threaded executor that exclusively writes all the hints and otherwise manipulate the writers. + * + * Flushing demultiplexes the provided {@link HintsBuffer} and sequentially writes to each {@link HintsWriter}, + * using the same shared write buffer. In the near future, when CASSANDRA-9428 (compression) is implemented, + * will also share a compression buffer. + */ +final class HintsWriteExecutor +{ + private static final int WRITE_BUFFER_SIZE = 256 << 10; + + private final HintsCatalog catalog; + private final ByteBuffer writeBuffer; + private final ExecutorService executor; + + HintsWriteExecutor(HintsCatalog catalog) + { + this.catalog = catalog; + + writeBuffer = ByteBuffer.allocateDirect(WRITE_BUFFER_SIZE); + executor = DebuggableThreadPoolExecutor.createWithFixedPoolSize("HintsWriteExecutor", 1); + } + + /* + * Should be very fast (worst case scenario - write a few 10s of megabytes to disk). + */ + void shutdownBlocking() + { + executor.shutdown(); + try + { + executor.awaitTermination(1, TimeUnit.MINUTES); + } + catch (InterruptedException e) + { + throw new AssertionError(e); + } + } + + /** + * Flush the provided buffer, recycle it and offer it back to the pool. + */ + Future flushBuffer(HintsBuffer buffer, HintsBufferPool bufferPool) + { + return executor.submit(new FlushBufferTask(buffer, bufferPool)); + } + + /** + * Flush the current buffer, but without clearing/recycling it. + */ + Future flushBufferPool(HintsBufferPool bufferPool) + { + return executor.submit(new FlushBufferPoolTask(bufferPool)); + } + + /** + * Flush the current buffer just for the specified hints stores. Without clearing/recycling it. + */ + Future flushBufferPool(HintsBufferPool bufferPool, Iterable stores) + { + return executor.submit(new PartiallyFlushBufferPoolTask(bufferPool, stores)); + } + + void fsyncWritersBlockingly(Iterable stores) + { + try + { + executor.submit(new FsyncWritersTask(stores)).get(); + } + catch (InterruptedException | ExecutionException e) + { + throw new RuntimeException(e); + } + } + + Future closeWriter(HintsStore store) + { + return executor.submit(store::closeWriter); + } + + Future closeAllWriters() + { + return executor.submit(() -> catalog.stores().forEach(HintsStore::closeWriter)); + } + + private final class FlushBufferTask implements Runnable + { + private final HintsBuffer buffer; + private final HintsBufferPool bufferPool; + + FlushBufferTask(HintsBuffer buffer, HintsBufferPool bufferPool) + { + this.buffer = buffer; + this.bufferPool = bufferPool; + } + + public void run() + { + buffer.waitForModifications(); + + try + { + flush(buffer); + } + finally + { + HintsBuffer recycledBuffer = buffer.recycle(); + if (!bufferPool.offer(recycledBuffer)) + recycledBuffer.free(); + } + } + } + + private final class FlushBufferPoolTask implements Runnable + { + private final HintsBufferPool bufferPool; + + FlushBufferPoolTask(HintsBufferPool bufferPool) + { + this.bufferPool = bufferPool; + } + + public void run() + { + HintsBuffer buffer = bufferPool.currentBuffer(); + buffer.waitForModifications(); + flush(buffer); + } + } + + private final class PartiallyFlushBufferPoolTask implements Runnable + { + private final HintsBufferPool bufferPool; + private final Iterable stores; + + PartiallyFlushBufferPoolTask(HintsBufferPool bufferPool, Iterable stores) + { + this.bufferPool = bufferPool; + this.stores = stores; + } + + public void run() + { + HintsBuffer buffer = bufferPool.currentBuffer(); + buffer.waitForModifications(); + stores.forEach(store -> flush(buffer.consumingHintsIterator(store.hostId), store)); + } + } + + private final class FsyncWritersTask implements Runnable + { + private final Iterable stores; + + FsyncWritersTask(Iterable stores) + { + this.stores = stores; + } + + public void run() + { + stores.forEach(HintsStore::fsyncWriter); + catalog.fsyncDirectory(); + } + } + + private void flush(HintsBuffer buffer) + { + buffer.hostIds().forEach(hostId -> flush(buffer.consumingHintsIterator(hostId), catalog.get(hostId))); + } + + private void flush(Iterator iterator, HintsStore store) + { + while (true) + { + flushInternal(iterator, store); + + if (!iterator.hasNext()) + break; + + // exceeded the size limit for an individual file, but still have more to write + // close the current writer and continue flushing to a new one in the next iteration + store.closeWriter(); + } + } + + private void flushInternal(Iterator iterator, HintsStore store) + { + long maxHintsFileSize = DatabaseDescriptor.getMaxHintsFileSize(); + + @SuppressWarnings("resource") + HintsWriter writer = store.getOrOpenWriter(); + + try (HintsWriter.Session session = writer.newSession(writeBuffer)) + { + while (iterator.hasNext()) + { + session.append(iterator.next()); + if (session.position() >= maxHintsFileSize) + break; + } + } + catch (IOException e) + { + throw new FSWriteError(e, writer.descriptor().fileName()); + } + } +} diff --git a/src/java/org/apache/cassandra/hints/HintsWriter.java b/src/java/org/apache/cassandra/hints/HintsWriter.java new file mode 100644 index 0000000000..300d9ccba4 --- /dev/null +++ b/src/java/org/apache/cassandra/hints/HintsWriter.java @@ -0,0 +1,272 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.hints; + +import java.io.File; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.file.Files; +import java.nio.file.StandardOpenOption; +import java.util.zip.CRC32; + +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.io.FSWriteError; +import org.apache.cassandra.io.util.DataOutputBuffer; +import org.apache.cassandra.io.util.DataOutputBufferFixed; +import org.apache.cassandra.utils.CLibrary; +import org.apache.cassandra.utils.SyncUtil; +import org.apache.cassandra.utils.Throwables; + +import static org.apache.cassandra.utils.FBUtilities.updateChecksum; +import static org.apache.cassandra.utils.FBUtilities.updateChecksumInt; +import static org.apache.cassandra.utils.Throwables.perform; + +final class HintsWriter implements AutoCloseable +{ + static final int PAGE_SIZE = 4096; + + private final File directory; + private final HintsDescriptor descriptor; + private final File file; + private final FileChannel channel; + private final int fd; + private final CRC32 globalCRC; + + private volatile long lastSyncPosition = 0L; + + private HintsWriter(File directory, HintsDescriptor descriptor, File file, FileChannel channel, int fd, CRC32 globalCRC) + { + this.directory = directory; + this.descriptor = descriptor; + this.file = file; + this.channel = channel; + this.fd = fd; + this.globalCRC = globalCRC; + } + + static HintsWriter create(File directory, HintsDescriptor descriptor) throws IOException + { + File file = new File(directory, descriptor.fileName()); + + FileChannel channel = FileChannel.open(file.toPath(), StandardOpenOption.WRITE, StandardOpenOption.CREATE_NEW); + int fd = CLibrary.getfd(channel); + + CRC32 crc = new CRC32(); + + try + { + // write the descriptor + DataOutputBuffer dob = new DataOutputBuffer(); + descriptor.serialize(dob); + ByteBuffer descriptorBytes = dob.buffer(); + updateChecksum(crc, descriptorBytes); + channel.write(descriptorBytes); + } + catch (Throwable e) + { + channel.close(); + throw e; + } + + return new HintsWriter(directory, descriptor, file, channel, fd, crc); + } + + HintsDescriptor descriptor() + { + return descriptor; + } + + private void writeChecksum() + { + File checksumFile = new File(directory, descriptor.checksumFileName()); + try (OutputStream out = Files.newOutputStream(checksumFile.toPath())) + { + out.write(Integer.toHexString((int) globalCRC.getValue()).getBytes()); + } + catch (IOException e) + { + throw new FSWriteError(e, checksumFile); + } + } + + public void close() + { + perform(file, Throwables.FileOpType.WRITE, this::doFsync, channel::close); + + writeChecksum(); + } + + public void fsync() + { + perform(file, Throwables.FileOpType.WRITE, this::doFsync); + } + + private void doFsync() throws IOException + { + SyncUtil.force(channel, true); + lastSyncPosition = channel.position(); + } + + Session newSession(ByteBuffer buffer) + { + try + { + return new Session(buffer, channel.size()); + } + catch (IOException e) + { + throw new FSWriteError(e, file); + } + } + + /** + * The primary goal of the Session class is to be able to share the same buffers among potentially dozens or hundreds + * of hints writers, and ensure that their contents are always written to the underlying channels in the end. + */ + final class Session implements AutoCloseable + { + private final ByteBuffer buffer; + + private final long initialSize; + private long bytesWritten; + + Session(ByteBuffer buffer, long initialSize) + { + buffer.clear(); + bytesWritten = 0L; + + this.buffer = buffer; + this.initialSize = initialSize; + } + + long position() + { + return initialSize + bytesWritten; + } + + /** + * Appends the serialized hint (with CRC included) to this session's aggregation buffer, + * writes to the underlying channel when the buffer is overflown. + * + * @param hint the serialized hint (with CRC included) + * @throws IOException + */ + void append(ByteBuffer hint) throws IOException + { + bytesWritten += hint.remaining(); + + // if the hint fits in the aggregation buffer, then just update the aggregation buffer, + // otherwise write both the aggregation buffer and the new buffer to the channel + if (hint.remaining() <= buffer.remaining()) + { + buffer.put(hint); + return; + } + + buffer.flip(); + + // update file-global CRC checksum + updateChecksum(globalCRC, buffer); + updateChecksum(globalCRC, hint); + + channel.write(new ByteBuffer[] { buffer, hint }); + buffer.clear(); + } + + /** + * Serializes and appends the hint (with CRC included) to this session's aggregation buffer, + * writes to the underlying channel when the buffer is overflown. + * + * Used mainly by tests and {@link LegacyHintsMigrator} + * + * @param hint the unserialized hint + * @throws IOException + */ + void append(Hint hint) throws IOException + { + int hintSize = (int) Hint.serializer.serializedSize(hint, descriptor.messagingVersion()); + int totalSize = hintSize + HintsBuffer.ENTRY_OVERHEAD_SIZE; + + if (totalSize > buffer.remaining()) + flushBuffer(); + + ByteBuffer hintBuffer = totalSize <= buffer.remaining() + ? buffer + : ByteBuffer.allocate(totalSize); + + CRC32 crc = new CRC32(); + try (DataOutputBufferFixed out = new DataOutputBufferFixed(hintBuffer)) + { + out.writeInt(hintSize); + updateChecksumInt(crc, hintSize); + out.writeInt((int) crc.getValue()); + + Hint.serializer.serialize(hint, out, descriptor.messagingVersion()); + updateChecksum(crc, hintBuffer, hintBuffer.position() - hintSize, hintSize); + out.writeInt((int) crc.getValue()); + } + + if (hintBuffer == buffer) + bytesWritten += totalSize; + else + append((ByteBuffer) hintBuffer.flip()); + } + + /** + * Closes the session - flushes the aggregation buffer (if not empty), does page aligning, and potentially fsyncs. + * @throws IOException + */ + public void close() throws IOException + { + flushBuffer(); + maybeFsync(); + maybeSkipCache(); + } + + private void flushBuffer() throws IOException + { + buffer.flip(); + + if (buffer.remaining() > 0) + { + updateChecksum(globalCRC, buffer); + channel.write(buffer); + } + + buffer.clear(); + } + + private void maybeFsync() + { + if (position() >= lastSyncPosition + DatabaseDescriptor.getTrickleFsyncIntervalInKb() * 1024) + fsync(); + } + + private void maybeSkipCache() + { + long position = position(); + + // don't skip page cache for tiny files, on the assumption that if they are tiny, the target node is probably + // alive, and if so, the file will be closed and dispatched shortly (within a minute), and the file will be dropped. + if (position >= DatabaseDescriptor.getTrickleFsyncIntervalInKb() * 1024) + CLibrary.trySkipCache(fd, 0, position - (position % PAGE_SIZE), file.getPath()); + } + } +} diff --git a/src/java/org/apache/cassandra/hints/LegacyHintsMigrator.java b/src/java/org/apache/cassandra/hints/LegacyHintsMigrator.java new file mode 100644 index 0000000000..082e307337 --- /dev/null +++ b/src/java/org/apache/cassandra/hints/LegacyHintsMigrator.java @@ -0,0 +1,243 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.hints; + +import java.io.File; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.*; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.cql3.QueryProcessor; +import org.apache.cassandra.cql3.UntypedResultSet; +import org.apache.cassandra.db.*; +import org.apache.cassandra.db.compaction.CompactionManager; +import org.apache.cassandra.db.marshal.UUIDType; +import org.apache.cassandra.db.partitions.PartitionUpdate; +import org.apache.cassandra.io.FSWriteError; +import org.apache.cassandra.io.sstable.Descriptor; +import org.apache.cassandra.io.util.DataInputBuffer; +import org.apache.cassandra.io.util.FileUtils; +import org.apache.cassandra.serializers.MarshalException; +import org.apache.cassandra.utils.FBUtilities; + +/** + * A migrator that goes through the legacy system.hints table and writes all the hints to the new hints storage format. + */ +@SuppressWarnings("deprecation") +public final class LegacyHintsMigrator +{ + private static final Logger logger = LoggerFactory.getLogger(LegacyHintsMigrator.class); + + private final File hintsDirectory; + private final long maxHintsFileSize; + + private final ColumnFamilyStore legacyHintsTable; + private final int pageSize; + + public LegacyHintsMigrator(File hintsDirectory, long maxHintsFileSize) + { + this.hintsDirectory = hintsDirectory; + this.maxHintsFileSize = maxHintsFileSize; + + legacyHintsTable = Keyspace.open(SystemKeyspace.NAME).getColumnFamilyStore(SystemKeyspace.LEGACY_HINTS); + pageSize = calculatePageSize(legacyHintsTable); + } + + // read fewer columns (mutations) per page if they are very large + private static int calculatePageSize(ColumnFamilyStore legacyHintsTable) + { + int size = 128; + + int meanCellCount = legacyHintsTable.getMeanColumns(); + double meanPartitionSize = legacyHintsTable.getMeanPartitionSize(); + + if (meanCellCount != 0 || meanPartitionSize != 0) + { + int avgHintSize = (int) meanPartitionSize / meanCellCount; + size = Math.max(2, Math.min(size, (512 << 10) / avgHintSize)); + } + + return size; + } + + public void migrate() + { + // nothing to migrate + if (legacyHintsTable.isEmpty()) + return; + logger.info("Migrating legacy hints to new storage"); + + // major-compact all of the existing sstables to get rid of the tombstones + expired hints + logger.info("Forcing a major compaction of {}.{} table", SystemKeyspace.NAME, SystemKeyspace.LEGACY_HINTS); + compactLegacyHints(); + + // paginate over legacy hints and write them to the new storage + logger.info("Migrating legacy hints to the new storage"); + migrateLegacyHints(); + + // truncate the legacy hints table + logger.info("Truncating {}.{} table", SystemKeyspace.NAME, SystemKeyspace.LEGACY_HINTS); + legacyHintsTable.truncateBlocking(); + } + + private void compactLegacyHints() + { + Collection descriptors = new ArrayList<>(); + legacyHintsTable.getTracker().getUncompacting().forEach(sstable -> descriptors.add(sstable.descriptor)); + if (!descriptors.isEmpty()) + forceCompaction(descriptors); + } + + private void forceCompaction(Collection descriptors) + { + try + { + CompactionManager.instance.submitUserDefined(legacyHintsTable, descriptors, FBUtilities.nowInSeconds()).get(); + } + catch (InterruptedException | ExecutionException e) + { + throw new RuntimeException(e); + } + } + + private void migrateLegacyHints() + { + ByteBuffer buffer = ByteBuffer.allocateDirect(256 * 1024); + String query = String.format("SELECT DISTINCT target_id FROM %s.%s", SystemKeyspace.NAME, SystemKeyspace.LEGACY_HINTS); + //noinspection ConstantConditions + QueryProcessor.executeInternal(query).forEach(row -> migrateLegacyHints(row.getUUID("target_id"), buffer)); + FileUtils.clean(buffer); + } + + private void migrateLegacyHints(UUID hostId, ByteBuffer buffer) + { + String query = String.format("SELECT target_id, hint_id, message_version, mutation, ttl(mutation) AS ttl, writeTime(mutation) AS write_time " + + "FROM %s.%s " + + "WHERE target_id = ?", + SystemKeyspace.NAME, + SystemKeyspace.LEGACY_HINTS); + + // read all the old hints (paged iterator), write them in the new format + UntypedResultSet rows = QueryProcessor.executeInternalWithPaging(query, pageSize, hostId); + migrateLegacyHints(hostId, rows, buffer); + + // delete the whole partition in the legacy table; we would truncate the whole table afterwards, but this allows + // to not lose progress in case of a terminated conversion + deleteLegacyHintsPartition(hostId); + } + + private void migrateLegacyHints(UUID hostId, UntypedResultSet rows, ByteBuffer buffer) + { + migrateLegacyHints(hostId, rows.iterator(), buffer); + } + + private void migrateLegacyHints(UUID hostId, Iterator iterator, ByteBuffer buffer) + { + do + { + migrateLegacyHintsInternal(hostId, iterator, buffer); + // if there are hints that didn't fit in the previous file, keep calling the method to write to a new + // file until we get everything written. + } + while (iterator.hasNext()); + } + + private void migrateLegacyHintsInternal(UUID hostId, Iterator iterator, ByteBuffer buffer) + { + HintsDescriptor descriptor = new HintsDescriptor(hostId, System.currentTimeMillis()); + + try (HintsWriter writer = HintsWriter.create(hintsDirectory, descriptor)) + { + try (HintsWriter.Session session = writer.newSession(buffer)) + { + while (iterator.hasNext()) + { + Hint hint = convertLegacyHint(iterator.next()); + if (hint != null) + session.append(hint); + + if (session.position() >= maxHintsFileSize) + break; + } + } + } + catch (IOException e) + { + throw new FSWriteError(e, descriptor.fileName()); + } + } + + private static Hint convertLegacyHint(UntypedResultSet.Row row) + { + Mutation mutation = deserializeLegacyMutation(row); + if (mutation == null) + return null; + + long creationTime = row.getLong("write_time"); // milliseconds, not micros, for the hints table + int expirationTime = FBUtilities.nowInSeconds() + row.getInt("ttl"); + int originalGCGS = expirationTime - (int) TimeUnit.MILLISECONDS.toSeconds(creationTime); + + int gcgs = Math.min(originalGCGS, mutation.smallestGCGS()); + + return Hint.create(mutation, creationTime, gcgs); + } + + private static Mutation deserializeLegacyMutation(UntypedResultSet.Row row) + { + try + { + Mutation mutation = Mutation.serializer.deserialize(new DataInputBuffer(row.getBlob("mutation"), true), + row.getInt("message_version")); + mutation.getPartitionUpdates().forEach(PartitionUpdate::validate); + return mutation; + } + catch (IOException e) + { + logger.error("Failed to migrate a hint for {} from legacy {}.{} table: {}", + row.getUUID("target_id"), + SystemKeyspace.NAME, + SystemKeyspace.LEGACY_HINTS, + e); + return null; + } + catch (MarshalException e) + { + logger.warn("Failed to validate a hint for {} (table id {}) from legacy {}.{} table - skipping: {})", + row.getUUID("target_id"), + SystemKeyspace.NAME, + SystemKeyspace.LEGACY_HINTS, + e); + return null; + } + } + + private static void deleteLegacyHintsPartition(UUID hostId) + { + // intentionally use millis, like the rest of the legacy implementation did, just in case + Mutation mutation = new Mutation(PartitionUpdate.fullPartitionDelete(SystemKeyspace.LegacyHints, + UUIDType.instance.decompose(hostId), + System.currentTimeMillis(), + FBUtilities.nowInSeconds())); + mutation.applyUnsafe(); + } +} diff --git a/src/java/org/apache/cassandra/hints/package-info.java b/src/java/org/apache/cassandra/hints/package-info.java new file mode 100644 index 0000000000..faa7b9fa70 --- /dev/null +++ b/src/java/org/apache/cassandra/hints/package-info.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * Hints subsystem consists of several components. + * + * {@link org.apache.cassandra.hints.Hint} encodes all the required metadata and the mutation being hinted. + * + * {@link org.apache.cassandra.hints.HintsBuffer} provides a temporary buffer for writing the hints to in a concurrent manner, + * before we flush them to disk. + * + * {@link org.apache.cassandra.hints.HintsBufferPool} is responsible for submitting {@link org.apache.cassandra.hints.HintsBuffer} + * instances for flushing when they exceed their capacity, and for maitaining a reserve {@link org.apache.cassandra.hints.HintsBuffer} + * instance, and creating extra ones if flushing cannot keep up with arrival rate. + * + * {@link org.apache.cassandra.hints.HintsWriteExecutor} is a single-threaded executor that performs all the writing to disk. + * + * {@link org.apache.cassandra.hints.HintsDispatchExecutor} is a multi-threaded executor responsible for dispatch of + * the hints to their destinations. + * + * {@link org.apache.cassandra.hints.HintsStore} tracks the state of all hints files (written and being written to) + * for a given host id destination. + * + * {@link org.apache.cassandra.hints.HintsCatalog} maintains the mapping of host ids to {@link org.apache.cassandra.hints.HintsStore} + * instances, and provides some aggregate APIs. + * + * {@link org.apache.cassandra.hints.HintsService} wraps the catalog, the pool, and the two executors, acting as a front-end + * for hints. + */ +package org.apache.cassandra.hints; \ No newline at end of file diff --git a/src/java/org/apache/cassandra/metrics/HintedHandoffMetrics.java b/src/java/org/apache/cassandra/metrics/HintedHandoffMetrics.java index e44279a339..51f6569f54 100644 --- a/src/java/org/apache/cassandra/metrics/HintedHandoffMetrics.java +++ b/src/java/org/apache/cassandra/metrics/HintedHandoffMetrics.java @@ -21,7 +21,6 @@ import java.net.InetAddress; import java.util.Map.Entry; import com.codahale.metrics.Counter; -import org.apache.cassandra.db.HintedHandOffManager; import org.apache.cassandra.db.SystemKeyspace; import org.apache.cassandra.utils.UUIDGen; import org.slf4j.Logger; @@ -34,7 +33,7 @@ import com.google.common.cache.LoadingCache; import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics; /** - * Metrics for {@link HintedHandOffManager}. + * Metrics for {@link org.apache.cassandra.hints.HintsService}. */ public class HintedHandoffMetrics { diff --git a/src/java/org/apache/cassandra/metrics/HintsServiceMetrics.java b/src/java/org/apache/cassandra/metrics/HintsServiceMetrics.java new file mode 100644 index 0000000000..062f67db7f --- /dev/null +++ b/src/java/org/apache/cassandra/metrics/HintsServiceMetrics.java @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.metrics; + +/** + * Metrics for {@link HintsService}. + */ +public final class HintsServiceMetrics +{ +} diff --git a/src/java/org/apache/cassandra/net/MessagingService.java b/src/java/org/apache/cassandra/net/MessagingService.java index b057d9879b..13632ace6a 100644 --- a/src/java/org/apache/cassandra/net/MessagingService.java +++ b/src/java/org/apache/cassandra/net/MessagingService.java @@ -55,6 +55,8 @@ import org.apache.cassandra.gms.EchoMessage; import org.apache.cassandra.gms.GossipDigestAck; import org.apache.cassandra.gms.GossipDigestAck2; import org.apache.cassandra.gms.GossipDigestSyn; +import org.apache.cassandra.hints.HintMessage; +import org.apache.cassandra.hints.HintResponse; import org.apache.cassandra.io.IVersionedSerializer; import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputPlus; @@ -97,7 +99,7 @@ public final class MessagingService implements MessagingServiceMBean public enum Verb { MUTATION, - @Deprecated BINARY, + HINT, READ_REPAIR, READ, REQUEST_RESPONSE, // client-initiated reads and writes @@ -129,7 +131,6 @@ public final class MessagingService implements MessagingServiceMBean _TRACE, // dummy verb so we can use MS.droppedMessagesMap ECHO, REPAIR_MESSAGE, - // use as padding for backwards compatability where a previous version needs to validate a verb from the future. PAXOS_PREPARE, PAXOS_PROPOSE, PAXOS_COMMIT, @@ -150,6 +151,7 @@ public final class MessagingService implements MessagingServiceMBean put(Verb.COUNTER_MUTATION, Stage.COUNTER_MUTATION); put(Verb.BATCHLOG_MUTATION, Stage.BATCHLOG_MUTATION); put(Verb.READ_REPAIR, Stage.MUTATION); + put(Verb.HINT, Stage.MUTATION); put(Verb.TRUNCATE, Stage.MUTATION); put(Verb.PAXOS_PREPARE, Stage.MUTATION); put(Verb.PAXOS_PROPOSE, Stage.MUTATION); @@ -226,6 +228,7 @@ public final class MessagingService implements MessagingServiceMBean put(Verb.PAXOS_PREPARE, Commit.serializer); put(Verb.PAXOS_PROPOSE, Commit.serializer); put(Verb.PAXOS_COMMIT, Commit.serializer); + put(Verb.HINT, HintMessage.serializer); }}; /** @@ -234,6 +237,7 @@ public final class MessagingService implements MessagingServiceMBean public static final EnumMap> callbackDeserializers = new EnumMap>(Verb.class) {{ put(Verb.MUTATION, WriteResponse.serializer); + put(Verb.HINT, HintResponse.serializer); put(Verb.BATCHLOG_MUTATION, WriteResponse.serializer); put(Verb.READ_REPAIR, WriteResponse.serializer); put(Verb.COUNTER_MUTATION, WriteResponse.serializer); @@ -299,6 +303,7 @@ public final class MessagingService implements MessagingServiceMBean Verb.MUTATION, Verb.BATCHLOG_MUTATION, //FIXME: should this be droppable?? Verb.COUNTER_MUTATION, + Verb.HINT, Verb.READ_REPAIR, Verb.READ, Verb.RANGE_SLICE, diff --git a/src/java/org/apache/cassandra/service/CassandraDaemon.java b/src/java/org/apache/cassandra/service/CassandraDaemon.java index 3b123c32c7..cf1e021f12 100644 --- a/src/java/org/apache/cassandra/service/CassandraDaemon.java +++ b/src/java/org/apache/cassandra/service/CassandraDaemon.java @@ -56,6 +56,7 @@ import org.apache.cassandra.db.commitlog.CommitLog; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.exceptions.StartupException; import org.apache.cassandra.gms.Gossiper; +import org.apache.cassandra.hints.LegacyHintsMigrator; import org.apache.cassandra.io.FSError; import org.apache.cassandra.io.sstable.CorruptSSTableException; import org.apache.cassandra.io.util.FileUtils; @@ -276,6 +277,9 @@ public class CassandraDaemon throw new RuntimeException(e); } + // migrate any legacy (pre-3.0) hints from system.hints table into the new store + new LegacyHintsMigrator(DatabaseDescriptor.getHintsDirectory(), DatabaseDescriptor.getMaxHintsFileSize()).migrate(); + // enable auto compaction for (Keyspace keyspace : Keyspace.all()) { diff --git a/src/java/org/apache/cassandra/service/StartupChecks.java b/src/java/org/apache/cassandra/service/StartupChecks.java index 9ffef9670d..1c07b5202f 100644 --- a/src/java/org/apache/cassandra/service/StartupChecks.java +++ b/src/java/org/apache/cassandra/service/StartupChecks.java @@ -179,33 +179,30 @@ public class StartupChecks } }; - public static final StartupCheck checkDataDirs = new StartupCheck() + public static final StartupCheck checkDataDirs = () -> { - public void execute() throws StartupException + // check all directories(data, commitlog, saved cache) for existence and permission + Iterable dirs = Iterables.concat(Arrays.asList(DatabaseDescriptor.getAllDataFileLocations()), + Arrays.asList(DatabaseDescriptor.getCommitLogLocation(), + DatabaseDescriptor.getSavedCachesLocation(), + DatabaseDescriptor.getHintsDirectory().getAbsolutePath())); + for (String dataDir : dirs) { - // check all directories(data, commitlog, saved cache) for existence and permission - Iterable dirs = Iterables.concat(Arrays.asList(DatabaseDescriptor.getAllDataFileLocations()), - Arrays.asList(DatabaseDescriptor.getCommitLogLocation(), - DatabaseDescriptor.getSavedCachesLocation())); - for (String dataDir : dirs) + logger.debug("Checking directory {}", dataDir); + File dir = new File(dataDir); + + // check that directories exist. + if (!dir.exists()) { - logger.debug("Checking directory {}", dataDir); - File dir = new File(dataDir); - - // check that directories exist. - if (!dir.exists()) - { - logger.error("Directory {} doesn't exist", dataDir); - // if they don't, failing their creation, stop cassandra. - if (!dir.mkdirs()) - throw new StartupException(3, "Has no permission to create directory "+ dataDir); - } - - // if directories exist verify their permissions - if (!Directories.verifyFullPermissions(dir, dataDir)) - throw new StartupException(3, "Insufficient permissions on directory " + dataDir); - + logger.error("Directory {} doesn't exist", dataDir); + // if they don't, failing their creation, stop cassandra. + if (!dir.mkdirs()) + throw new StartupException(3, "Has no permission to create directory "+ dataDir); } + + // if directories exist verify their permissions + if (!Directories.verifyFullPermissions(dir, dataDir)) + throw new StartupException(3, "Insufficient permissions on directory " + dataDir); } }; diff --git a/src/java/org/apache/cassandra/service/StorageProxy.java b/src/java/org/apache/cassandra/service/StorageProxy.java index fc917f0d64..12c2c241fa 100644 --- a/src/java/org/apache/cassandra/service/StorageProxy.java +++ b/src/java/org/apache/cassandra/service/StorageProxy.java @@ -34,6 +34,7 @@ import com.google.common.util.concurrent.Uninterruptibles; import org.apache.cassandra.db.view.MaterializedViewManager; import org.apache.cassandra.db.view.MaterializedViewUtils; +import org.apache.cassandra.db.HintedHandOffManager; import org.apache.cassandra.metrics.*; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; @@ -57,6 +58,8 @@ import org.apache.cassandra.dht.Token; import org.apache.cassandra.exceptions.*; import org.apache.cassandra.gms.FailureDetector; import org.apache.cassandra.gms.Gossiper; +import org.apache.cassandra.hints.Hint; +import org.apache.cassandra.hints.HintsService; import org.apache.cassandra.io.util.DataOutputBuffer; import org.apache.cassandra.locator.AbstractReplicationStrategy; import org.apache.cassandra.locator.IEndpointSnitch; @@ -99,7 +102,9 @@ public class StorageProxy implements StorageProxyMBean private static final double CONCURRENT_SUBREQUESTS_MARGIN = 0.10; - private StorageProxy() {} + private StorageProxy() + { + } static { @@ -113,6 +118,9 @@ public class StorageProxy implements StorageProxyMBean throw new RuntimeException(e); } + HintsService.instance.registerMBean(); + HintedHandOffManager.instance.registerMBean(); + standardWritePerformer = new WritePerformer() { public void apply(IMutation mutation, @@ -603,32 +611,41 @@ public class StorageProxy implements StorageProxyMBean } } - /** hint all the mutations (except counters, which can't be safely retried). This means - * we'll re-hint any successful ones; doesn't seem worth it to track individual success - * just for this unusual case. - - * @param mutations the mutations that require hints - */ + /** + * Hint all the mutations (except counters, which can't be safely retried). This means + * we'll re-hint any successful ones; doesn't seem worth it to track individual success + * just for this unusual case. + * + * Only used for CL.ANY + * + * @param mutations the mutations that require hints + */ private static void hintMutations(Collection mutations) { for (IMutation mutation : mutations) - { - if (mutation instanceof CounterMutation) - continue; + if (!(mutation instanceof CounterMutation)) + hintMutation((Mutation) mutation); - Token tk = mutation.key().getToken(); - List naturalEndpoints = StorageService.instance.getNaturalEndpoints(mutation.getKeyspaceName(), tk); - Collection pendingEndpoints = StorageService.instance.getTokenMetadata().pendingEndpointsFor(tk, mutation.getKeyspaceName()); - for (InetAddress target : Iterables.concat(naturalEndpoints, pendingEndpoints)) - { - // local writes can timeout, but cannot be dropped (see LocalMutationRunnable and - // CASSANDRA-6510), so there is no need to hint or retry - if (!target.equals(FBUtilities.getBroadcastAddress()) && shouldHint(target)) - submitHint((Mutation) mutation, target, null); - } - } + Tracing.trace("Wrote hints to satisfy CL.ANY after no replicas acknowledged the write"); + } - Tracing.trace("Wrote hint to satisfy CL.ANY after no replicas acknowledged the write"); + private static void hintMutation(Mutation mutation) + { + Token tk = DatabaseDescriptor.getPartitioner().getToken(mutation.key().getKey()); + List naturalEndpoints = StorageService.instance.getNaturalEndpoints(mutation.getKeyspaceName(), tk); + Collection pendingEndpoints = + StorageService.instance.getTokenMetadata().pendingEndpointsFor(tk, mutation.getKeyspaceName()); + + Iterable endpoints = Iterables.concat(naturalEndpoints, pendingEndpoints); + ArrayList endpointsToHint = new ArrayList<>(Iterables.size(endpoints)); + + // local writes can timeout, but cannot be dropped (see LocalMutationRunnable and CASSANDRA-6510), + // so there is no need to hint or retry. + for (InetAddress target : endpoints) + if (!target.equals(FBUtilities.getBroadcastAddress()) && shouldHint(target)) + endpointsToHint.add(target); + + submitHint(mutation, endpointsToHint, null); } /** @@ -1071,7 +1088,7 @@ public class StorageProxy implements StorageProxyMBean MessageOut message = null; boolean insertLocal = false; - + ArrayList endpointsToHint = null; for (InetAddress destination : targets) { @@ -1115,16 +1132,21 @@ public class StorageProxy implements StorageProxyMBean messages.add(destination); } } - } else + } + else { - if (!shouldHint(destination)) - continue; - - // Schedule a local hint - submitHint(mutation, destination, responseHandler); + if (shouldHint(destination)) + { + if (endpointsToHint == null) + endpointsToHint = new ArrayList<>(Iterables.size(targets)); + endpointsToHint.add(destination); + } } } + if (endpointsToHint != null) + submitHint(mutation, endpointsToHint, responseHandler); + if (insertLocal) insertLocal(stage, mutation, responseHandler); @@ -1139,66 +1161,6 @@ public class StorageProxy implements StorageProxyMBean } } - private static AtomicInteger getHintsInProgressFor(InetAddress destination) - { - try - { - return hintsInProgress.load(destination); - } - catch (Exception e) - { - throw new AssertionError(e); - } - } - - public static Future submitHint(final Mutation mutation, - final InetAddress target, - final AbstractWriteResponseHandler responseHandler) - { - // local write that time out should be handled by LocalMutationRunnable - assert !target.equals(FBUtilities.getBroadcastAddress()) : target; - - HintRunnable runnable = new HintRunnable(target) - { - public void runMayThrow() - { - int ttl = HintedHandOffManager.calculateHintTTL(mutation); - if (ttl > 0) - { - logger.debug("Adding hint for {}", target); - writeHintForMutation(mutation, System.currentTimeMillis(), ttl, target); - // Notify the handler only for CL == ANY - if (responseHandler != null && responseHandler.consistencyLevel == ConsistencyLevel.ANY) - responseHandler.response(null); - } else - { - logger.debug("Skipped writing hint for {} (ttl {})", target, ttl); - } - } - }; - - return submitHint(runnable); - } - - private static Future submitHint(HintRunnable runnable) - { - StorageMetrics.totalHintsInProgress.inc(); - getHintsInProgressFor(runnable.target).incrementAndGet(); - return (Future) StageManager.getStage(Stage.MUTATION).submit(runnable); - } - - /** - * @param now current time in milliseconds - relevant for hint replay handling of truncated CFs - */ - public static void writeHintForMutation(Mutation mutation, long now, int ttl, InetAddress target) - { - assert ttl > 0; - UUID hostId = StorageService.instance.getTokenMetadata().getHostId(target); - assert hostId != null : "Missing host ID for " + target.getHostAddress(); - HintedHandOffManager.instance.hintFor(mutation, now, ttl, hostId).apply(); - StorageMetrics.totalHints.inc(); - } - private static void sendMessagesToNonlocalDC(MessageOut message, Collection targets, AbstractWriteResponseHandler handler) @@ -2209,7 +2171,7 @@ public class StorageProxy implements StorageProxyMBean { if (!DatabaseDescriptor.hintedHandoffEnabled()) { - HintedHandOffManager.instance.metrics.incrPastWindow(ep); + HintsService.instance.metrics.incrPastWindow(ep); return false; } @@ -2220,7 +2182,7 @@ public class StorageProxy implements StorageProxyMBean if (disabledDCs.contains(dc)) { Tracing.trace("Not hinting {} since its data center {} has been disabled {}", ep, dc, disabledDCs); - HintedHandOffManager.instance.metrics.incrPastWindow(ep); + HintsService.instance.metrics.incrPastWindow(ep); return false; } } @@ -2228,7 +2190,7 @@ public class StorageProxy implements StorageProxyMBean boolean hintWindowExpired = Gossiper.instance.getEndpointDowntime(ep) > DatabaseDescriptor.getMaxHintWindow(); if (hintWindowExpired) { - HintedHandOffManager.instance.metrics.incrPastWindow(ep); + HintsService.instance.metrics.incrPastWindow(ep); Tracing.trace("Not hinting {} which has been down {} ms", ep, Gossiper.instance.getEndpointDowntime(ep)); } return !hintWindowExpired; @@ -2363,7 +2325,7 @@ public class StorageProxy implements StorageProxyMBean if (System.currentTimeMillis() > constructionTime + DatabaseDescriptor.getTimeout(MessagingService.Verb.MUTATION)) { MessagingService.instance().incrementDroppedMessages(MessagingService.Verb.MUTATION); - HintRunnable runnable = new HintRunnable(FBUtilities.getBroadcastAddress()) + HintRunnable runnable = new HintRunnable(Collections.singleton(FBUtilities.getBroadcastAddress())) { protected void runMayThrow() throws Exception { @@ -2393,11 +2355,11 @@ public class StorageProxy implements StorageProxyMBean */ private abstract static class HintRunnable implements Runnable { - public final InetAddress target; + public final Collection targets; - protected HintRunnable(InetAddress target) + protected HintRunnable(Collection targets) { - this.target = target; + this.targets = targets; } public void run() @@ -2412,8 +2374,9 @@ public class StorageProxy implements StorageProxyMBean } finally { - StorageMetrics.totalHintsInProgress.dec(); - getHintsInProgressFor(target).decrementAndGet(); + StorageMetrics.totalHintsInProgress.dec(targets.size()); + for (InetAddress target : targets) + getHintsInProgressFor(target).decrementAndGet(); } } @@ -2446,6 +2409,52 @@ public class StorageProxy implements StorageProxyMBean logger.warn("Some hints were not written before shutdown. This is not supposed to happen. You should (a) run repair, and (b) file a bug report"); } + private static AtomicInteger getHintsInProgressFor(InetAddress destination) + { + try + { + return hintsInProgress.load(destination); + } + catch (Exception e) + { + throw new AssertionError(e); + } + } + + public static Future submitHint(Mutation mutation, InetAddress target, AbstractWriteResponseHandler responseHandler) + { + return submitHint(mutation, Collections.singleton(target), responseHandler); + } + + public static Future submitHint(Mutation mutation, + Collection targets, + AbstractWriteResponseHandler responseHandler) + { + HintRunnable runnable = new HintRunnable(targets) + { + public void runMayThrow() + { + logger.debug("Adding hints for {}", targets); + HintsService.instance.write(Iterables.transform(targets, StorageService.instance::getHostIdForEndpoint), + Hint.create(mutation, System.currentTimeMillis())); + targets.forEach(HintsService.instance.metrics::incrCreatedHints); + // Notify the handler only for CL == ANY + if (responseHandler != null && responseHandler.consistencyLevel == ConsistencyLevel.ANY) + responseHandler.response(null); + } + }; + + return submitHint(runnable); + } + + private static Future submitHint(HintRunnable runnable) + { + StorageMetrics.totalHintsInProgress.inc(runnable.targets.size()); + for (InetAddress target : runnable.targets) + getHintsInProgressFor(target).incrementAndGet(); + return (Future) StageManager.getStage(Stage.MUTATION).submit(runnable); + } + public Long getRpcTimeout() { return DatabaseDescriptor.getRpcTimeout(); } public void setRpcTimeout(Long timeoutInMillis) { DatabaseDescriptor.setRpcTimeout(timeoutInMillis); } @@ -2478,11 +2487,11 @@ public class StorageProxy implements StorageProxyMBean public long getReadRepairAttempted() { return ReadRepairMetrics.attempted.getCount(); } - + public long getReadRepairRepairedBlocking() { return ReadRepairMetrics.repairedBlocking.getCount(); } - + public long getReadRepairRepairedBackground() { return ReadRepairMetrics.repairedBackground.getCount(); } diff --git a/src/java/org/apache/cassandra/service/StorageService.java b/src/java/org/apache/cassandra/service/StorageService.java index 5966e49d53..034664575a 100644 --- a/src/java/org/apache/cassandra/service/StorageService.java +++ b/src/java/org/apache/cassandra/service/StorageService.java @@ -28,22 +28,8 @@ import java.lang.management.ManagementFactory; import java.net.InetAddress; import java.net.UnknownHostException; import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.LinkedHashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.Map.Entry; -import java.util.Set; -import java.util.SortedMap; -import java.util.TreeMap; -import java.util.UUID; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; @@ -54,6 +40,7 @@ import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; +import javax.annotation.Nullable; import javax.management.JMX; import javax.management.MBeanServer; import javax.management.NotificationBroadcasterSupport; @@ -95,6 +82,8 @@ import org.apache.cassandra.gms.IEndpointStateChangeSubscriber; import org.apache.cassandra.gms.IFailureDetector; import org.apache.cassandra.gms.TokenSerializer; import org.apache.cassandra.gms.VersionedValue; +import org.apache.cassandra.hints.HintVerbHandler; +import org.apache.cassandra.hints.HintsService; import org.apache.cassandra.io.sstable.SSTableLoader; import org.apache.cassandra.io.util.FileUtils; import org.apache.cassandra.locator.AbstractReplicationStrategy; @@ -303,6 +292,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE MessagingService.instance().registerVerbHandlers(MessagingService.Verb.PAXOS_PREPARE, new PrepareVerbHandler()); MessagingService.instance().registerVerbHandlers(MessagingService.Verb.PAXOS_PROPOSE, new ProposeVerbHandler()); MessagingService.instance().registerVerbHandlers(MessagingService.Verb.PAXOS_COMMIT, new CommitVerbHandler()); + MessagingService.instance().registerVerbHandlers(MessagingService.Verb.HINT, new HintVerbHandler()); // see BootStrapper for a summary of how the bootstrap verbs interact MessagingService.instance().registerVerbHandlers(MessagingService.Verb.REPLICATION_FINISHED, new ReplicationFinishedVerbHandler()); @@ -585,7 +575,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE logger.info("Cassandra version: {}", FBUtilities.getReleaseVersionString()); logger.info("Thrift API version: {}", cassandraConstants.VERSION); logger.info("CQL supported versions: {} (default: {})", - StringUtils.join(ClientState.getCQLSupportedVersion(), ","), ClientState.DEFAULT_CQL_VERSION); + StringUtils.join(ClientState.getCQLSupportedVersion(), ","), ClientState.DEFAULT_CQL_VERSION); initialized = true; @@ -649,6 +639,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE MessagingService.instance().shutdown(); materializedViewMutationStage.shutdown(); batchlogMutationStage.shutdown(); + HintsService.instance.pauseDispatch(); counterMutationStage.shutdown(); mutationStage.shutdown(); materializedViewMutationStage.awaitTermination(3600, TimeUnit.SECONDS); @@ -681,6 +672,8 @@ public class StorageService extends NotificationBroadcasterSupport implements IE if (FBUtilities.isWindows()) WindowsTimer.endTimerPeriod(DatabaseDescriptor.getWindowsTimerInterval()); + HintsService.instance.shutdownBlocking(); + // wait for miscellaneous tasks like sstable and commitlog segment deletion ScheduledExecutors.nonPeriodicTasks.shutdown(); if (!ScheduledExecutors.nonPeriodicTasks.awaitTermination(1, TimeUnit.MINUTES)) @@ -792,7 +785,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE MessagingService.instance().listen(FBUtilities.getLocalAddress()); LoadBroadcaster.instance.startBroadcasting(); - HintedHandOffManager.instance.start(); + HintsService.instance.startDispatch(); BatchlogManager.instance.start(); } } @@ -1564,6 +1557,11 @@ public class StorageService extends NotificationBroadcasterSupport implements IE return getTokenMetadata().getHostId(FBUtilities.getBroadcastAddress()).toString(); } + public UUID getLocalHostUUID() + { + return getTokenMetadata().getHostId(FBUtilities.getBroadcastAddress()); + } + public Map getHostIdMap() { Map mapOut = new HashMap<>(); @@ -2119,11 +2117,13 @@ public class StorageService extends NotificationBroadcasterSupport implements IE private void excise(Collection tokens, InetAddress endpoint) { logger.info("Removing tokens {} for {}", tokens, endpoint); - HintedHandOffManager.instance.deleteHintsForEndpoint(endpoint); + + if (tokenMetadata.isMember(endpoint)) + HintsService.instance.excise(tokenMetadata.getHostId(endpoint)); + removeEndpoint(endpoint); tokenMetadata.removeEndpoint(endpoint); tokenMetadata.removeBootstrapTokens(tokens); - notifyLeft(endpoint); PendingRangeCalculatorService.instance.update(); } @@ -2337,10 +2337,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE MigrationManager.instance.scheduleSchemaPull(endpoint, state); if (tokenMetadata.isMember(endpoint)) - { - HintedHandOffManager.instance.scheduleHintDelivery(endpoint, true); notifyUp(endpoint); - } } public void onRemove(InetAddress endpoint) @@ -2380,9 +2377,10 @@ public class StorageService extends NotificationBroadcasterSupport implements IE return map; } + // TODO public final void deliverHints(String host) throws UnknownHostException { - HintedHandOffManager.instance.scheduleHintDelivery(host); + throw new UnsupportedOperationException(); } public Collection getLocalTokens() @@ -2392,6 +2390,18 @@ public class StorageService extends NotificationBroadcasterSupport implements IE return tokens; } + @Nullable + public InetAddress getEndpointForHostId(UUID hostId) + { + return tokenMetadata.getEndpointForHostId(hostId); + } + + @Nullable + public UUID getHostIdForEndpoint(InetAddress address) + { + return tokenMetadata.getHostId(address); + } + /* These methods belong to the MBean interface */ public List getTokens() @@ -3418,7 +3428,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE private Future streamHints() { // StreamPlan will not fail if there are zero files to transfer, so flush anyway (need to get any in-memory hints, as well) - ColumnFamilyStore hintsCF = Keyspace.open(SystemKeyspace.NAME).getColumnFamilyStore(SystemKeyspace.HINTS); + ColumnFamilyStore hintsCF = Keyspace.open(SystemKeyspace.NAME).getColumnFamilyStore(SystemKeyspace.LEGACY_HINTS); FBUtilities.waitOnFuture(hintsCF.forceFlush()); // gather all live nodes in the cluster that aren't also leaving @@ -3451,7 +3461,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE preferred, SystemKeyspace.NAME, ranges, - SystemKeyspace.HINTS) + SystemKeyspace.LEGACY_HINTS) .execute(); } } @@ -3835,7 +3845,11 @@ public class StorageService extends NotificationBroadcasterSupport implements IE public synchronized void drain() throws IOException, InterruptedException, ExecutionException { inShutdownHook = true; - + + BatchlogManager.shutdown(); + + HintsService.instance.pauseDispatch(); + ExecutorService counterMutationStage = StageManager.getStage(Stage.COUNTER_MUTATION); ExecutorService batchlogMutationStage = StageManager.getStage(Stage.BATCHLOG_MUTATION); ExecutorService materializedViewMutationStage = StageManager.getStage(Stage.MATERIALIZED_VIEW_MUTATION); @@ -3899,7 +3913,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE } FBUtilities.waitOnFutures(flushes); - BatchlogManager.shutdown(); + HintsService.instance.shutdownBlocking(); // whilst we've flushed all the CFs, which will have recycled all completed segments, we want to ensure // there are no segments to replay, so we force the recycling of any remaining (should be at most one) diff --git a/src/java/org/apache/cassandra/tools/NodeProbe.java b/src/java/org/apache/cassandra/tools/NodeProbe.java index 22b34558b4..c1e0a0d425 100644 --- a/src/java/org/apache/cassandra/tools/NodeProbe.java +++ b/src/java/org/apache/cassandra/tools/NodeProbe.java @@ -57,7 +57,6 @@ import org.apache.cassandra.concurrent.Stage; import org.apache.cassandra.db.BatchlogManager; import org.apache.cassandra.db.BatchlogManagerMBean; import org.apache.cassandra.db.ColumnFamilyStoreMBean; -import org.apache.cassandra.db.HintedHandOffManager; import org.apache.cassandra.db.HintedHandOffManagerMBean; import org.apache.cassandra.db.compaction.CompactionManager; import org.apache.cassandra.db.compaction.CompactionManagerMBean; @@ -65,6 +64,7 @@ import org.apache.cassandra.gms.FailureDetector; import org.apache.cassandra.gms.FailureDetectorMBean; import org.apache.cassandra.gms.Gossiper; import org.apache.cassandra.gms.GossiperMBean; +import org.apache.cassandra.db.HintedHandOffManager; import org.apache.cassandra.locator.EndpointSnitchInfoMBean; import org.apache.cassandra.metrics.CassandraMetricsRegistry; import org.apache.cassandra.metrics.TableMetrics.Sampler; diff --git a/src/java/org/apache/cassandra/utils/FBUtilities.java b/src/java/org/apache/cassandra/utils/FBUtilities.java index d1462d228c..4f7a97a83e 100644 --- a/src/java/org/apache/cassandra/utils/FBUtilities.java +++ b/src/java/org/apache/cassandra/utils/FBUtilities.java @@ -26,10 +26,10 @@ import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.util.*; import java.util.concurrent.*; +import java.util.zip.CRC32; import java.util.zip.Checksum; import com.google.common.base.Joiner; -import org.apache.cassandra.utils.AbstractIterator; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -332,7 +332,7 @@ public class FBUtilities public static int nowInSeconds() { - return (int)(System.currentTimeMillis() / 1000); + return (int) (System.currentTimeMillis() / 1000); } public static void waitOnFutures(Iterable> futures) @@ -595,6 +595,34 @@ public class FBUtilities checksum.update((v >>> 0) & 0xFF); } + /** + * Updates checksum with the provided ByteBuffer at the given offset + length. + * Resets position and limit back to their original values on return. + * This method is *NOT* thread-safe. + */ + public static void updateChecksum(CRC32 checksum, ByteBuffer buffer, int offset, int length) + { + int position = buffer.position(); + int limit = buffer.limit(); + + buffer.position(offset).limit(offset + length); + checksum.update(buffer); + + buffer.position(position).limit(limit); + } + + /** + * Updates checksum with the provided ByteBuffer. + * Resets position back to its original values on return. + * This method is *NOT* thread-safe. + */ + public static void updateChecksum(CRC32 checksum, ByteBuffer buffer) + { + int position = buffer.position(); + checksum.update(buffer); + buffer.position(position); + } + private static final ThreadLocal threadLocalScratchBuffer = new ThreadLocal() { @Override diff --git a/src/java/org/apache/cassandra/utils/Throwables.java b/src/java/org/apache/cassandra/utils/Throwables.java index 0a2bd2834a..d6ce7b4535 100644 --- a/src/java/org/apache/cassandra/utils/Throwables.java +++ b/src/java/org/apache/cassandra/utils/Throwables.java @@ -18,10 +18,25 @@ */ package org.apache.cassandra.utils; -public class Throwables -{ +import java.io.File; +import java.io.IOException; +import java.util.Arrays; +import java.util.Iterator; +import java.util.stream.Stream; - public static Throwable merge(Throwable existingFail, Throwable newFail) +import org.apache.cassandra.io.FSReadError; +import org.apache.cassandra.io.FSWriteError; + +public final class Throwables +{ + public enum FileOpType { READ, WRITE } + + public interface DiscreteAction + { + void perform() throws E; + } + + public static T merge(T existingFail, T newFail) { if (existingFail == null) return newFail; @@ -31,7 +46,74 @@ public class Throwables public static void maybeFail(Throwable fail) { - if (fail != null) - com.google.common.base.Throwables.propagate(fail); + if (failIfCanCast(fail, null)) + throw new RuntimeException(fail); + } + + public static void maybeFail(Throwable fail, Class checked) throws T + { + if (failIfCanCast(fail, checked)) + throw new RuntimeException(fail); + } + + public static boolean failIfCanCast(Throwable fail, Class checked) throws T + { + if (fail == null) + return false; + + if (fail instanceof Error) + throw (Error) fail; + + if (fail instanceof RuntimeException) + throw (RuntimeException) fail; + + if (checked != null && checked.isInstance(fail)) + throw checked.cast(fail); + + return true; + } + + @SafeVarargs + public static void perform(DiscreteAction ... actions) throws E + { + perform(Arrays.stream(actions)); + } + + @SuppressWarnings("unchecked") + public static void perform(Stream> actions) throws E + { + Throwable fail = null; + Iterator> iter = actions.iterator(); + while (iter.hasNext()) + { + DiscreteAction action = iter.next(); + try + { + action.perform(); + } + catch (Throwable t) + { + fail = merge(fail, t); + } + } + + if (failIfCanCast(fail, null)) + throw (E) fail; + } + + @SafeVarargs + public static void perform(File against, FileOpType opType, DiscreteAction ... actions) + { + perform(Arrays.stream(actions).map((action) -> () -> + { + try + { + action.perform(); + } + catch (IOException e) + { + throw (opType == FileOpType.WRITE) ? new FSWriteError(e, against) : new FSReadError(e, against); + } + })); } } diff --git a/test/conf/cassandra.yaml b/test/conf/cassandra.yaml index 0bbaee4cc9..1dba2845a0 100644 --- a/test/conf/cassandra.yaml +++ b/test/conf/cassandra.yaml @@ -8,6 +8,7 @@ commitlog_sync: batch commitlog_sync_batch_window_in_ms: 1.0 commitlog_segment_size_in_mb: 5 commitlog_directory: build/test/cassandra/commitlog +hints_directory: build/test/cassandra/hints partitioner: org.apache.cassandra.dht.ByteOrderedPartitioner listen_address: 127.0.0.1 storage_port: 7010 diff --git a/test/long/org/apache/cassandra/hints/HintsWriteThenReadTest.java b/test/long/org/apache/cassandra/hints/HintsWriteThenReadTest.java new file mode 100644 index 0000000000..fd880cb70e --- /dev/null +++ b/test/long/org/apache/cassandra/hints/HintsWriteThenReadTest.java @@ -0,0 +1,191 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.hints; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.nio.file.Files; +import java.util.Iterator; +import java.util.UUID; +import java.util.concurrent.TimeUnit; +import java.util.zip.CRC32; + +import com.google.common.collect.Iterables; +import org.junit.Test; + +import org.apache.cassandra.SchemaLoader; +import org.apache.cassandra.config.CFMetaData; +import org.apache.cassandra.config.Schema; +import org.apache.cassandra.db.Mutation; +import org.apache.cassandra.db.RowUpdateBuilder; +import org.apache.cassandra.db.rows.Cell; +import org.apache.cassandra.db.rows.Row; +import org.apache.cassandra.io.util.FileUtils; +import org.apache.cassandra.schema.KeyspaceParams; +import org.apache.cassandra.utils.FBUtilities; + +import static junit.framework.Assert.assertEquals; +import static junit.framework.Assert.assertNotNull; +import static junit.framework.Assert.assertTrue; + +import static org.apache.cassandra.Util.dk; +import static org.apache.cassandra.utils.ByteBufferUtil.bytes; + +public class HintsWriteThenReadTest +{ + private static final String KEYSPACE = "hints_write_then_read_test"; + private static final String TABLE = "table"; + + private static final int HINTS_COUNT = 10_000_000; + + @Test + public void testWriteReadCycle() throws IOException + { + SchemaLoader.prepareServer(); + SchemaLoader.createKeyspace(KEYSPACE, KeyspaceParams.simple(1), SchemaLoader.standardCFMD(KEYSPACE, TABLE)); + + HintsDescriptor descriptor = new HintsDescriptor(UUID.randomUUID(), System.currentTimeMillis()); + + File directory = Files.createTempDirectory(null).toFile(); + try + { + testWriteReadCycle(directory, descriptor); + } + finally + { + directory.deleteOnExit(); + } + } + + private void testWriteReadCycle(File directory, HintsDescriptor descriptor) throws IOException + { + // write HINTS_COUNT hints to a file + writeHints(directory, descriptor); + + // calculate the checksum of the file, then compare to the .crc32 checksum file content + verifyChecksum(directory, descriptor); + + // iterate over the written hints, make sure they are all present + verifyHints(directory, descriptor); + } + + private void writeHints(File directory, HintsDescriptor descriptor) throws IOException + { + try (HintsWriter writer = HintsWriter.create(directory, descriptor)) + { + write(writer, descriptor.timestamp); + } + } + + private static void verifyChecksum(File directory, HintsDescriptor descriptor) throws IOException + { + File hintsFile = new File(directory, descriptor.fileName()); + File checksumFile = new File(directory, descriptor.checksumFileName()); + + assertTrue(checksumFile.exists()); + + String actualChecksum = Integer.toHexString(calculateChecksum(hintsFile)); + String expectedChecksum = Files.readAllLines(checksumFile.toPath()).iterator().next(); + + assertEquals(expectedChecksum, actualChecksum); + } + + private void verifyHints(File directory, HintsDescriptor descriptor) + { + long baseTimestamp = descriptor.timestamp; + int index = 0; + + try (HintsReader reader = HintsReader.open(new File(directory, descriptor.fileName()))) + { + for (HintsReader.Page page : reader) + { + Iterator hints = page.hintsIterator(); + while (hints.hasNext()) + { + Hint hint = hints.next(); + + long timestamp = baseTimestamp + index; + Mutation mutation = hint.mutation; + + assertEquals(timestamp, hint.creationTime); + assertEquals(dk(bytes(index)), mutation.key()); + + Row row = mutation.getPartitionUpdates().iterator().next().iterator().next(); + assertEquals(1, Iterables.size(row.cells())); + assertEquals(bytes(index), row.clustering().get(0)); + Cell cell = row.cells().iterator().next(); + assertNotNull(cell); + assertEquals(bytes(index), cell.value()); + assertEquals(timestamp * 1000, cell.timestamp()); + + index++; + } + } + } + + assertEquals(index, HINTS_COUNT); + } + + private void write(HintsWriter writer, long timestamp) throws IOException + { + ByteBuffer buffer = ByteBuffer.allocateDirect(256 * 1024); + try (HintsWriter.Session session = writer.newSession(buffer)) + { + write(session, timestamp); + } + FileUtils.clean(buffer); + } + + private void write(HintsWriter.Session session, long timestamp) throws IOException + { + for (int i = 0; i < HINTS_COUNT; i++) + session.append(createHint(i, timestamp)); + } + + private static Hint createHint(int idx, long baseTimestamp) + { + long timestamp = baseTimestamp + idx; + return Hint.create(createMutation(idx, TimeUnit.MILLISECONDS.toMicros(timestamp)), timestamp); + } + + private static Mutation createMutation(int index, long timestamp) + { + CFMetaData table = Schema.instance.getCFMetaData(KEYSPACE, TABLE); + return new RowUpdateBuilder(table, timestamp, bytes(index)) + .clustering(bytes(index)) + .add("val", bytes(index)) + .build(); + } + + private static int calculateChecksum(File file) throws IOException + { + CRC32 crc = new CRC32(); + byte[] buffer = new byte[FBUtilities.MAX_UNSIGNED_SHORT]; + + try (InputStream in = Files.newInputStream(file.toPath())) + { + int bytesRead; + while((bytesRead = in.read(buffer)) != -1) + crc.update(buffer, 0, bytesRead); + } + + return (int) crc.getValue(); + } +} diff --git a/test/unit/org/apache/cassandra/OffsetAwareConfigurationLoader.java b/test/unit/org/apache/cassandra/OffsetAwareConfigurationLoader.java index 9023b11ef2..3bdb192caf 100644 --- a/test/unit/org/apache/cassandra/OffsetAwareConfigurationLoader.java +++ b/test/unit/org/apache/cassandra/OffsetAwareConfigurationLoader.java @@ -54,6 +54,7 @@ public class OffsetAwareConfigurationLoader extends YamlConfigurationLoader config.commitlog_directory += File.pathSeparator + offset; config.saved_caches_directory += File.pathSeparator + offset; + config.hints_directory += File.pathSeparator + offset; for (int i = 0; i < config.data_file_directories.length; i++) config.data_file_directories[i] += File.pathSeparator + offset; diff --git a/test/unit/org/apache/cassandra/db/HintedHandOffTest.java b/test/unit/org/apache/cassandra/db/HintedHandOffTest.java deleted file mode 100644 index e06c95aeb3..0000000000 --- a/test/unit/org/apache/cassandra/db/HintedHandOffTest.java +++ /dev/null @@ -1,150 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * - */ -package org.apache.cassandra.db; - -import java.net.InetAddress; -import java.util.Map; -import java.util.UUID; - -import com.google.common.collect.Iterators; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; - -import org.apache.cassandra.SchemaLoader; -import org.apache.cassandra.cql3.UntypedResultSet; -import org.apache.cassandra.db.marshal.Int32Type; -import org.apache.cassandra.db.marshal.UUIDType; -import org.apache.cassandra.exceptions.ConfigurationException; -import org.apache.cassandra.schema.KeyspaceParams; -import org.apache.cassandra.utils.ByteBufferUtil; -import org.apache.cassandra.utils.FBUtilities; - -import static org.apache.cassandra.cql3.QueryProcessor.executeInternal; -import static org.junit.Assert.assertEquals; - -public class HintedHandOffTest -{ - - public static final String KEYSPACE4 = "HintedHandOffTest4"; - public static final String STANDARD1_CF = "Standard1"; - - @BeforeClass - public static void defineSchema() throws ConfigurationException - { - SchemaLoader.prepareServer(); - SchemaLoader.createKeyspace(KEYSPACE4, - KeyspaceParams.simple(1), - SchemaLoader.standardCFMD(KEYSPACE4, STANDARD1_CF)); - } - - @Before - public void clearHints() - { - Keyspace systemKeyspace = Keyspace.open("system"); - ColumnFamilyStore hintStore = systemKeyspace.getColumnFamilyStore(SystemKeyspace.HINTS); - hintStore.clearUnsafe(); - } - - // Test compaction of hints column family. It shouldn't remove all columns on compaction. - @Test - public void testCompactionOfHintsCF() throws Exception - { - // prepare hints column family - Keyspace systemKeyspace = Keyspace.open("system"); - ColumnFamilyStore hintStore = systemKeyspace.getColumnFamilyStore(SystemKeyspace.HINTS); - hintStore.clearUnsafe(); - hintStore.metadata.gcGraceSeconds(36000); // 10 hours - hintStore.disableAutoCompaction(); - - // insert 1 hint - Mutation rm = mutation(); - HintedHandOffManager.instance.hintFor(rm, - System.currentTimeMillis(), - HintedHandOffManager.calculateHintTTL(rm), - UUID.randomUUID()) - .applyUnsafe(); - - // flush data to disk - hintStore.forceBlockingFlush(); - assertEquals(1, hintStore.getLiveSSTables().size()); - - // submit compaction - HintedHandOffManager.instance.compact(); - - // single row should not be removed because of gc_grace_seconds - // is 10 hours and there are no any tombstones in sstable - assertEquals(1, hintStore.getLiveSSTables().size()); - } - - @Test - public void testHintsMetrics() throws Exception - { - for (int i = 0; i < 99; i++) - HintedHandOffManager.instance.metrics.incrPastWindow(InetAddress.getLocalHost()); - HintedHandOffManager.instance.metrics.log(); - - UntypedResultSet rows = executeInternal("SELECT hints_dropped FROM system." + SystemKeyspace.PEER_EVENTS); - Map returned = rows.one().getMap("hints_dropped", UUIDType.instance, Int32Type.instance); - assertEquals(Iterators.getLast(returned.values().iterator()).intValue(), 99); - } - - @Test(timeout = 5000) - public void testTruncateHints() throws Exception - { - // insert 1 hint - Mutation rm = mutation(); - HintedHandOffManager.instance.hintFor(rm, - System.currentTimeMillis(), - HintedHandOffManager.calculateHintTTL(rm), - UUID.randomUUID()) - .applyUnsafe(); - - assert getNoOfHints() == 1; - - HintedHandOffManager.instance.truncateAllHints(); - - while(getNoOfHints() > 0) - { - Thread.sleep(100); - } - - assert getNoOfHints() == 0; - } - - private Mutation mutation() - { - // get a random mutation to write a hint for - return new RowUpdateBuilder(Keyspace.open(KEYSPACE4).getColumnFamilyStore(STANDARD1_CF).metadata, - FBUtilities.timestampMicros(), - ByteBufferUtil.bytes(1)) - .clustering("cluster_col0") - .add("val", "value0") - .build(); - } - - private int getNoOfHints() - { - String req = "SELECT * FROM system.%s"; - UntypedResultSet resultSet = executeInternal(String.format(req, SystemKeyspace.HINTS)); - return resultSet.size(); - } -} diff --git a/test/unit/org/apache/cassandra/db/commitlog/CommitLogTest.java b/test/unit/org/apache/cassandra/db/commitlog/CommitLogTest.java index b41b7b30dc..fcdab62d99 100644 --- a/test/unit/org/apache/cassandra/db/commitlog/CommitLogTest.java +++ b/test/unit/org/apache/cassandra/db/commitlog/CommitLogTest.java @@ -250,7 +250,7 @@ public class CommitLogTest // Adding new mutation on another CF, large enough (including CL entry overhead) that a new segment is created Mutation rm2 = new RowUpdateBuilder(cfs2.metadata, 0, "k") .clustering("bytes") - .add("val", ByteBuffer.allocate((DatabaseDescriptor.getCommitLogSegmentSize()/2) - 200)) + .add("val", ByteBuffer.allocate(DatabaseDescriptor.getMaxMutationSize() - 200)) .build(); CommitLog.instance.add(rm2); // also forces a new segment, since each entry-with-overhead is just under half the CL size @@ -280,7 +280,7 @@ public class CommitLogTest .clustering(colName) .add("val", ByteBuffer.allocate(allocSize)).build(); - int max = (DatabaseDescriptor.getCommitLogSegmentSize() / 2); + int max = DatabaseDescriptor.getMaxMutationSize(); max -= CommitLogSegment.ENTRY_OVERHEAD_SIZE; // log entry overhead // Note that the size of the value if vint encoded. So we first compute the ovehead of the mutation without the value and it's size diff --git a/test/unit/org/apache/cassandra/hints/ChecksummedDataInputTest.java b/test/unit/org/apache/cassandra/hints/ChecksummedDataInputTest.java new file mode 100644 index 0000000000..e431924f32 --- /dev/null +++ b/test/unit/org/apache/cassandra/hints/ChecksummedDataInputTest.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.hints; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.zip.CRC32; + +import org.junit.Test; + +import org.apache.cassandra.hints.ChecksummedDataInput; +import org.apache.cassandra.io.util.AbstractDataInput; +import org.apache.cassandra.io.util.DataOutputBuffer; +import org.apache.cassandra.utils.FBUtilities; + +import static junit.framework.Assert.assertEquals; +import static junit.framework.Assert.assertTrue; + +public class ChecksummedDataInputTest +{ + @Test + public void testThatItWorks() throws IOException + { + // fill a bytebuffer with some input + DataOutputBuffer out = new DataOutputBuffer(); + out.write(127); + out.write(new byte[]{ 0, 1, 2, 3, 4, 5, 6 }); + out.writeBoolean(false); + out.writeByte(10); + out.writeChar('t'); + out.writeDouble(3.3); + out.writeFloat(2.2f); + out.writeInt(42); + out.writeLong(Long.MAX_VALUE); + out.writeShort(Short.MIN_VALUE); + out.writeUTF("utf"); + ByteBuffer buffer = out.buffer(); + + // calculate resulting CRC + CRC32 crc = new CRC32(); + FBUtilities.updateChecksum(crc, buffer); + int expectedCRC = (int) crc.getValue(); + + ChecksummedDataInput crcInput = ChecksummedDataInput.wrap(new DummyByteBufferDataInput(buffer.duplicate())); + crcInput.limit(buffer.remaining()); + + // assert that we read all the right values back + assertEquals(127, crcInput.read()); + byte[] bytes = new byte[7]; + crcInput.readFully(bytes); + assertTrue(Arrays.equals(new byte[]{ 0, 1, 2, 3, 4, 5, 6 }, bytes)); + assertEquals(false, crcInput.readBoolean()); + assertEquals(10, crcInput.readByte()); + assertEquals('t', crcInput.readChar()); + assertEquals(3.3, crcInput.readDouble()); + assertEquals(2.2f, crcInput.readFloat()); + assertEquals(42, crcInput.readInt()); + assertEquals(Long.MAX_VALUE, crcInput.readLong()); + assertEquals(Short.MIN_VALUE, crcInput.readShort()); + assertEquals("utf", crcInput.readUTF()); + + // assert that the crc matches, and that we've read exactly as many bytes as expected + assertEquals(0, crcInput.bytesRemaining()); + assertEquals(expectedCRC, crcInput.getCrc()); + } + + private static final class DummyByteBufferDataInput extends AbstractDataInput + { + private final ByteBuffer buffer; + + DummyByteBufferDataInput(ByteBuffer buffer) + { + this.buffer = buffer; + } + + public void seek(long position) + { + throw new UnsupportedOperationException(); + } + + public long getPosition() + { + throw new UnsupportedOperationException(); + } + + public long getPositionLimit() + { + throw new UnsupportedOperationException(); + } + + public int read() + { + return buffer.get() & 0xFF; + } + } +} diff --git a/test/unit/org/apache/cassandra/hints/HintMessageTest.java b/test/unit/org/apache/cassandra/hints/HintMessageTest.java new file mode 100644 index 0000000000..7ffaa54194 --- /dev/null +++ b/test/unit/org/apache/cassandra/hints/HintMessageTest.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.hints; + +import java.io.IOException; +import java.util.UUID; + +import org.junit.Test; + +import org.apache.cassandra.SchemaLoader; +import org.apache.cassandra.config.CFMetaData; +import org.apache.cassandra.config.Schema; +import org.apache.cassandra.db.Mutation; +import org.apache.cassandra.db.RowUpdateBuilder; +import org.apache.cassandra.io.util.DataInputBuffer; +import org.apache.cassandra.io.util.DataInputPlus; +import org.apache.cassandra.io.util.DataOutputBuffer; +import org.apache.cassandra.net.MessagingService; +import org.apache.cassandra.schema.KeyspaceParams; +import org.apache.cassandra.utils.FBUtilities; + +import static junit.framework.Assert.assertEquals; + +import static org.apache.cassandra.hints.HintsTestUtil.assertHintsEqual; +import static org.apache.cassandra.utils.ByteBufferUtil.bytes; + +public class HintMessageTest +{ + private static final String KEYSPACE = "hint_message_test"; + private static final String TABLE = "table"; + + @Test + public void testSerializer() throws IOException + { + SchemaLoader.prepareServer(); + SchemaLoader.createKeyspace(KEYSPACE, KeyspaceParams.simple(1), SchemaLoader.standardCFMD(KEYSPACE, TABLE)); + + UUID hostId = UUID.randomUUID(); + long now = FBUtilities.timestampMicros(); + + CFMetaData table = Schema.instance.getCFMetaData(KEYSPACE, TABLE); + Mutation mutation = + new RowUpdateBuilder(table, now, bytes("key")) + .clustering("column") + .add("val", "val" + 1234) + .build(); + Hint hint = Hint.create(mutation, now / 1000); + HintMessage message = new HintMessage(hostId, hint); + + // serialize + int serializedSize = (int) HintMessage.serializer.serializedSize(message, MessagingService.current_version); + DataOutputBuffer dob = new DataOutputBuffer(); + HintMessage.serializer.serialize(message, dob, MessagingService.current_version); + assertEquals(serializedSize, dob.getLength()); + + // deserialize + DataInputPlus di = new DataInputBuffer(dob.buffer(), true); + HintMessage deserializedMessage = HintMessage.serializer.deserialize(di, MessagingService.current_version); + + // compare before/after + assertEquals(hostId, deserializedMessage.hostId); + assertHintsEqual(message.hint, deserializedMessage.hint); + } +} diff --git a/test/unit/org/apache/cassandra/hints/HintTest.java b/test/unit/org/apache/cassandra/hints/HintTest.java new file mode 100644 index 0000000000..c198149493 --- /dev/null +++ b/test/unit/org/apache/cassandra/hints/HintTest.java @@ -0,0 +1,231 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.hints; + +import java.io.IOException; + +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import org.apache.cassandra.SchemaLoader; +import org.apache.cassandra.Util; +import org.apache.cassandra.config.CFMetaData; +import org.apache.cassandra.config.Schema; +import org.apache.cassandra.db.*; +import org.apache.cassandra.db.partitions.FilteredPartition; +import org.apache.cassandra.db.partitions.PartitionIterator; +import org.apache.cassandra.db.partitions.PartitionUpdate; +import org.apache.cassandra.io.util.DataInputBuffer; +import org.apache.cassandra.io.util.DataInputPlus; +import org.apache.cassandra.io.util.DataOutputBuffer; +import org.apache.cassandra.net.MessagingService; +import org.apache.cassandra.schema.KeyspaceParams; +import org.apache.cassandra.schema.TableParams; +import org.apache.cassandra.utils.FBUtilities; + +import static junit.framework.Assert.*; + +import static org.apache.cassandra.Util.dk; +import static org.apache.cassandra.hints.HintsTestUtil.assertHintsEqual; +import static org.apache.cassandra.hints.HintsTestUtil.assertPartitionsEqual; +import static org.apache.cassandra.utils.ByteBufferUtil.bytes; + +public class HintTest +{ + private static final String KEYSPACE = "hint_test"; + private static final String TABLE0 = "table_0"; + private static final String TABLE1 = "table_1"; + private static final String TABLE2 = "table_2"; + + @BeforeClass + public static void defineSchema() + { + SchemaLoader.prepareServer(); + SchemaLoader.createKeyspace(KEYSPACE, + KeyspaceParams.simple(1), + SchemaLoader.standardCFMD(KEYSPACE, TABLE0), + SchemaLoader.standardCFMD(KEYSPACE, TABLE1), + SchemaLoader.standardCFMD(KEYSPACE, TABLE2)); + } + + @Before + public void resetGcGraceSeconds() + { + for (CFMetaData table : Schema.instance.getTables(KEYSPACE)) + table.gcGraceSeconds(TableParams.DEFAULT_GC_GRACE_SECONDS); + } + + @Test + public void testSerializer() throws IOException + { + long now = FBUtilities.timestampMicros(); + Mutation mutation = createMutation("testSerializer", now); + Hint hint = Hint.create(mutation, now / 1000); + + // serialize + int serializedSize = (int) Hint.serializer.serializedSize(hint, MessagingService.current_version); + DataOutputBuffer dob = new DataOutputBuffer(); + Hint.serializer.serialize(hint, dob, MessagingService.current_version); + assertEquals(serializedSize, dob.getLength()); + + // deserialize + DataInputPlus di = new DataInputBuffer(dob.buffer(), true); + Hint deserializedHint = Hint.serializer.deserialize(di, MessagingService.current_version); + + // compare before/after + assertHintsEqual(hint, deserializedHint); + } + + @Test + public void testApply() + { + long now = FBUtilities.timestampMicros(); + String key = "testApply"; + Mutation mutation = createMutation(key, now); + Hint hint = Hint.create(mutation, now / 1000); + + // sanity check that there is no data inside yet + assertNoPartitions(key, TABLE0); + assertNoPartitions(key, TABLE1); + assertNoPartitions(key, TABLE2); + + hint.apply(); + + // assert that we can read the inserted partitions + for (PartitionUpdate partition : mutation.getPartitionUpdates()) + assertPartitionsEqual(partition, readPartition(key, partition.metadata().cfName)); + } + + @Test + public void testApplyWithTruncation() + { + long now = FBUtilities.timestampMicros(); + String key = "testApplyWithTruncation"; + Mutation mutation = createMutation(key, now); + + // sanity check that there is no data inside yet + assertNoPartitions(key, TABLE0); + assertNoPartitions(key, TABLE1); + assertNoPartitions(key, TABLE2); + + // truncate TABLE1 + Keyspace.open(KEYSPACE).getColumnFamilyStore(TABLE1).truncateBlocking(); + + // create and apply a hint with creation time in the past (one second before the truncation) + Hint.create(mutation, now / 1000 - 1).apply(); + + // TABLE1 update should have been skipped and not applied, as expired + assertNoPartitions(key, TABLE1); + + // TABLE0 and TABLE2 updates should have been applied successfully + assertPartitionsEqual(mutation.getPartitionUpdate(Schema.instance.getId(KEYSPACE, TABLE0)), readPartition(key, TABLE0)); + assertPartitionsEqual(mutation.getPartitionUpdate(Schema.instance.getId(KEYSPACE, TABLE2)), readPartition(key, TABLE2)); + } + + @Test + public void testApplyWithRegularExpiration() + { + long now = FBUtilities.timestampMicros(); + String key = "testApplyWithRegularExpiration"; + Mutation mutation = createMutation(key, now); + + // sanity check that there is no data inside yet + assertNoPartitions(key, TABLE0); + assertNoPartitions(key, TABLE1); + assertNoPartitions(key, TABLE2); + + // lower the GC GS on TABLE0 to 0 BEFORE the hint is created + Schema.instance.getCFMetaData(KEYSPACE, TABLE0).gcGraceSeconds(0); + + Hint.create(mutation, now / 1000).apply(); + + // all updates should have been skipped and not applied, as expired + assertNoPartitions(key, TABLE0); + assertNoPartitions(key, TABLE1); + assertNoPartitions(key, TABLE2); + } + + @Test + public void testApplyWithGCGSReducedLater() + { + long now = FBUtilities.timestampMicros(); + String key = "testApplyWithGCGSReducedLater"; + Mutation mutation = createMutation(key, now); + Hint hint = Hint.create(mutation, now / 1000); + + // sanity check that there is no data inside yet + assertNoPartitions(key, TABLE0); + assertNoPartitions(key, TABLE1); + assertNoPartitions(key, TABLE2); + + // lower the GC GS on TABLE0 AFTER the hint is already created + Schema.instance.getCFMetaData(KEYSPACE, TABLE0).gcGraceSeconds(0); + + hint.apply(); + + // all updates should have been skipped and not applied, as expired + assertNoPartitions(key, TABLE0); + assertNoPartitions(key, TABLE1); + assertNoPartitions(key, TABLE2); + } + + private static Mutation createMutation(String key, long now) + { + Mutation mutation = new Mutation(KEYSPACE, dk(key)); + + new RowUpdateBuilder(Schema.instance.getCFMetaData(KEYSPACE, TABLE0), now, mutation) + .clustering("column0") + .add("val", "value0") + .build(); + + new RowUpdateBuilder(Schema.instance.getCFMetaData(KEYSPACE, TABLE1), now + 1, mutation) + .clustering("column1") + .add("val", "value1") + .build(); + + new RowUpdateBuilder(Schema.instance.getCFMetaData(KEYSPACE, TABLE2), now + 2, mutation) + .clustering("column2") + .add("val", "value2") + .build(); + + return mutation; + } + + private static SinglePartitionReadCommand cmd(String key, String table) + { + CFMetaData meta = Schema.instance.getCFMetaData(KEYSPACE, table); + return SinglePartitionReadCommand.fullPartitionRead(meta, FBUtilities.nowInSeconds(), bytes(key)); + } + + private static FilteredPartition readPartition(String key, String table) + { + return Util.getOnlyPartition(cmd(key, table)); + } + + private static void assertNoPartitions(String key, String table) + { + ReadCommand cmd = cmd(key, table); + + try (ReadOrderGroup orderGroup = cmd.startOrderGroup(); + PartitionIterator iterator = cmd.executeInternal(orderGroup)) + { + assertFalse(iterator.hasNext()); + } + } +} diff --git a/test/unit/org/apache/cassandra/hints/HintsBufferTest.java b/test/unit/org/apache/cassandra/hints/HintsBufferTest.java new file mode 100644 index 0000000000..ebc333add9 --- /dev/null +++ b/test/unit/org/apache/cassandra/hints/HintsBufferTest.java @@ -0,0 +1,236 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.hints; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.*; +import java.util.concurrent.TimeUnit; +import java.util.zip.CRC32; + +import com.google.common.collect.Iterables; +import org.junit.BeforeClass; +import org.junit.Test; + +import org.apache.cassandra.SchemaLoader; +import org.apache.cassandra.config.CFMetaData; +import org.apache.cassandra.config.Schema; +import org.apache.cassandra.db.Mutation; +import org.apache.cassandra.db.RowUpdateBuilder; +import org.apache.cassandra.db.rows.Cell; +import org.apache.cassandra.db.rows.Row; +import org.apache.cassandra.io.util.DataInputBuffer; +import org.apache.cassandra.io.util.DataInputPlus; +import org.apache.cassandra.net.MessagingService; +import org.apache.cassandra.schema.KeyspaceParams; + +import static junit.framework.Assert.*; + +import static org.apache.cassandra.utils.ByteBufferUtil.bytes; +import static org.apache.cassandra.utils.FBUtilities.updateChecksum; + +public class HintsBufferTest +{ + private static final String KEYSPACE = "hints_buffer_test"; + private static final String TABLE = "table"; + + private static final int HINTS_COUNT = 300_000; + private static final int HINT_THREADS_COUNT = 10; + private static final int HOST_ID_COUNT = 10; + + @BeforeClass + public static void defineSchema() + { + SchemaLoader.prepareServer(); + SchemaLoader.createKeyspace(KEYSPACE, KeyspaceParams.simple(1), SchemaLoader.standardCFMD(KEYSPACE, TABLE)); + } + + @Test + @SuppressWarnings("resource") + public void testOverlyLargeAllocation() + { + // create a small, 128 bytes buffer + HintsBuffer buffer = HintsBuffer.create(128); + + // try allocating an entry of 65 bytes (53 bytes hint + 12 bytes of overhead) + try + { + buffer.allocate(65 - HintsBuffer.ENTRY_OVERHEAD_SIZE); + fail("Allocation of the buffer should have failed but hasn't"); + } + catch (IllegalArgumentException e) + { + assertEquals(String.format("Hint of %s bytes is too large - the maximum size is 64", 65 - HintsBuffer.ENTRY_OVERHEAD_SIZE), + e.getMessage()); + } + + // assert that a 1-byte smaller allocation fits properly + try (HintsBuffer.Allocation allocation = buffer.allocate(64 - HintsBuffer.ENTRY_OVERHEAD_SIZE)) + { + assertNotNull(allocation); + } + } + + @Test + public void testWrite() throws IOException, InterruptedException + { + // generate 10 random host ids to choose from + UUID[] hostIds = new UUID[HOST_ID_COUNT]; + for (int i = 0; i < hostIds.length; i++) + hostIds[i] = UUID.randomUUID(); + + // map each index to one random UUID from the previously created UUID array + Random random = new Random(System.currentTimeMillis()); + UUID[] load = new UUID[HINTS_COUNT]; + for (int i = 0; i < load.length; i++) + load[i] = hostIds[random.nextInt(HOST_ID_COUNT)]; + + // calculate the size of a single hint (they will all have an equal size in this test) + int hintSize = (int) Hint.serializer.serializedSize(createHint(0, System.currentTimeMillis()), MessagingService.current_version); + int entrySize = hintSize + HintsBuffer.ENTRY_OVERHEAD_SIZE; + + // allocate a slab to fit *precisely* HINTS_COUNT hints + int slabSize = entrySize * HINTS_COUNT; + HintsBuffer buffer = HintsBuffer.create(slabSize); + + // use a fixed timestamp base for all mutation timestamps + long baseTimestamp = System.currentTimeMillis(); + + // create HINT_THREADS_COUNT, start them, and wait for them to finish + List threads = new ArrayList<>(HINT_THREADS_COUNT); + for (int i = 0; i < HINT_THREADS_COUNT; i ++) + threads.add(new Thread(new Writer(buffer, load, hintSize, i, baseTimestamp))); + threads.forEach(java.lang.Thread::start); + for (Thread thread : threads) + thread.join(); + + // sanity check that we are full + assertEquals(slabSize, buffer.capacity()); + assertEquals(0, buffer.remaining()); + + // try to allocate more bytes, ensure that the allocation fails + assertNull(buffer.allocate(1)); + + // a failed allocation should automatically close the oporder + buffer.waitForModifications(); + + // a failed allocation should also automatically make the buffer as closed + assertTrue(buffer.isClosed()); + + // assert that host id set in the buffer equals to hostIds + assertEquals(HOST_ID_COUNT, buffer.hostIds().size()); + assertEquals(new HashSet<>(Arrays.asList(hostIds)), buffer.hostIds()); + + // iterate over *every written hint*, validate its content + for (UUID hostId : hostIds) + { + Iterator iter = buffer.consumingHintsIterator(hostId); + while (iter.hasNext()) + { + int idx = validateEntry(hostId, iter.next(), baseTimestamp, load); + load[idx] = null; // nullify each visited entry + } + } + + // assert that all the entries in load array have been visited and nullified + for (UUID hostId : load) + assertNull(hostId); + + // free the buffer + buffer.free(); + } + + private static int validateEntry(UUID hostId, ByteBuffer buffer, long baseTimestamp, UUID[] load) throws IOException + { + CRC32 crc = new CRC32(); + DataInputPlus di = new DataInputBuffer(buffer, true); + + // read and validate size + int hintSize = di.readInt(); + assertEquals(hintSize + HintsBuffer.ENTRY_OVERHEAD_SIZE, buffer.remaining()); + + // read and validate size crc + updateChecksum(crc, buffer, buffer.position(), 4); + assertEquals((int) crc.getValue(), di.readInt()); + + // read the hint and update/validate overall crc + Hint hint = Hint.serializer.deserialize(di, MessagingService.current_version); + updateChecksum(crc, buffer, buffer.position() + 8, hintSize); + assertEquals((int) crc.getValue(), di.readInt()); + + // further validate hint correctness + int idx = (int) (hint.creationTime - baseTimestamp); + assertEquals(hostId, load[idx]); + + Row row = hint.mutation.getPartitionUpdates().iterator().next().iterator().next(); + assertEquals(1, Iterables.size(row.cells())); + + assertEquals(bytes(idx), row.clustering().get(0)); + Cell cell = row.cells().iterator().next(); + assertEquals(TimeUnit.MILLISECONDS.toMicros(baseTimestamp + idx), cell.timestamp()); + assertEquals(bytes(idx), cell.value()); + + return idx; + } + + private static Hint createHint(int idx, long baseTimestamp) + { + long timestamp = baseTimestamp + idx; + return Hint.create(createMutation(idx, TimeUnit.MILLISECONDS.toMicros(timestamp)), timestamp); + } + + private static Mutation createMutation(int index, long timestamp) + { + CFMetaData table = Schema.instance.getCFMetaData(KEYSPACE, TABLE); + return new RowUpdateBuilder(table, timestamp, bytes(index)) + .clustering(bytes(index)) + .add("val", bytes(index)) + .build(); + } + + static class Writer implements Runnable + { + final HintsBuffer buffer; + final UUID[] load; + final int hintSize; + final int index; + final long baseTimestamp; + + Writer(HintsBuffer buffer, UUID[] load, int hintSize, int index, long baseTimestamp) + { + this.buffer = buffer; + this.load = load; + this.hintSize = hintSize; + this.index = index; + this.baseTimestamp = baseTimestamp; + } + + public void run() + { + int hintsPerThread = HINTS_COUNT / HINT_THREADS_COUNT; + for (int i = index * hintsPerThread; i < (index + 1) * hintsPerThread; i++) + { + try (HintsBuffer.Allocation allocation = buffer.allocate(hintSize)) + { + Hint hint = createHint(i, baseTimestamp); + allocation.write(Collections.singleton(load[i]), hint); + } + } + } + } +} diff --git a/test/unit/org/apache/cassandra/hints/HintsCatalogTest.java b/test/unit/org/apache/cassandra/hints/HintsCatalogTest.java new file mode 100644 index 0000000000..d627fcfa26 --- /dev/null +++ b/test/unit/org/apache/cassandra/hints/HintsCatalogTest.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.hints; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.util.*; + +import org.junit.Test; + +import static junit.framework.Assert.*; + +public class HintsCatalogTest +{ + @Test + public void loadCompletenessAndOrderTest() throws IOException + { + File directory = Files.createTempDirectory(null).toFile(); + try + { + loadCompletenessAndOrderTest(directory); + } + finally + { + directory.deleteOnExit(); + } + } + + public static void loadCompletenessAndOrderTest(File directory) throws IOException + { + UUID hostId1 = UUID.randomUUID(); + UUID hostId2 = UUID.randomUUID(); + + long timestamp1 = System.currentTimeMillis(); + long timestamp2 = System.currentTimeMillis() + 1; + long timestamp3 = System.currentTimeMillis() + 2; + long timestamp4 = System.currentTimeMillis() + 3; + + HintsDescriptor descriptor1 = new HintsDescriptor(hostId1, timestamp1); + HintsDescriptor descriptor2 = new HintsDescriptor(hostId2, timestamp3); + HintsDescriptor descriptor3 = new HintsDescriptor(hostId2, timestamp2); + HintsDescriptor descriptor4 = new HintsDescriptor(hostId1, timestamp4); + + writeDescriptor(directory, descriptor1); + writeDescriptor(directory, descriptor2); + writeDescriptor(directory, descriptor3); + writeDescriptor(directory, descriptor4); + + HintsCatalog catalog = HintsCatalog.load(directory); + assertEquals(2, catalog.stores().count()); + + HintsStore store1 = catalog.get(hostId1); + assertNotNull(store1); + assertEquals(descriptor1, store1.poll()); + assertEquals(descriptor4, store1.poll()); + assertNull(store1.poll()); + + HintsStore store2 = catalog.get(hostId2); + assertNotNull(store2); + assertEquals(descriptor3, store2.poll()); + assertEquals(descriptor2, store2.poll()); + assertNull(store2.poll()); + } + + @SuppressWarnings("EmptyTryBlock") + private static void writeDescriptor(File directory, HintsDescriptor descriptor) throws IOException + { + try (HintsWriter ignored = HintsWriter.create(directory, descriptor)) + { + } + } +} diff --git a/test/unit/org/apache/cassandra/hints/HintsDescriptorTest.java b/test/unit/org/apache/cassandra/hints/HintsDescriptorTest.java new file mode 100644 index 0000000000..08487d1874 --- /dev/null +++ b/test/unit/org/apache/cassandra/hints/HintsDescriptorTest.java @@ -0,0 +1,153 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.hints; + +import java.io.DataInput; +import java.io.File; +import java.io.IOException; +import java.util.UUID; + +import com.google.common.collect.ImmutableMap; +import com.google.common.io.ByteStreams; +import com.google.common.io.Files; +import org.junit.Test; + +import org.apache.cassandra.io.compress.LZ4Compressor; +import org.apache.cassandra.io.util.DataOutputBuffer; + +import static junit.framework.Assert.assertEquals; +import static junit.framework.Assert.assertNotSame; +import static junit.framework.Assert.fail; + +public class HintsDescriptorTest +{ + @Test + public void testSerializerNormal() throws IOException + { + UUID hostId = UUID.randomUUID(); + int version = HintsDescriptor.CURRENT_VERSION; + long timestamp = System.currentTimeMillis(); + ImmutableMap parameters = + ImmutableMap.of("compression", (Object) ImmutableMap.of("class_name", LZ4Compressor.class.getName())); + HintsDescriptor descriptor = new HintsDescriptor(hostId, version, timestamp, parameters); + + testSerializeDeserializeLoop(descriptor); + } + + @Test + public void testSerializerWithEmptyParameters() throws IOException + { + UUID hostId = UUID.randomUUID(); + int version = HintsDescriptor.CURRENT_VERSION; + long timestamp = System.currentTimeMillis(); + ImmutableMap parameters = ImmutableMap.of(); + HintsDescriptor descriptor = new HintsDescriptor(hostId, version, timestamp, parameters); + + testSerializeDeserializeLoop(descriptor); + } + + @Test + public void testCorruptedDeserialize() throws IOException + { + UUID hostId = UUID.randomUUID(); + int version = HintsDescriptor.CURRENT_VERSION; + long timestamp = System.currentTimeMillis(); + ImmutableMap parameters = ImmutableMap.of(); + HintsDescriptor descriptor = new HintsDescriptor(hostId, version, timestamp, parameters); + + byte[] bytes = serializeDescriptor(descriptor); + + // mess up the parameters size + bytes[28] = (byte) 0xFF; + bytes[29] = (byte) 0xFF; + bytes[30] = (byte) 0xFF; + bytes[31] = (byte) 0x7F; + + // attempt to deserialize + try + { + deserializeDescriptor(bytes); + fail("Deserializing the descriptor should but didn't"); + } + catch (IOException e) + { + assertEquals("Hints Descriptor CRC Mismatch", e.getMessage()); + } + } + + @Test + @SuppressWarnings("EmptyTryBlock") + public void testReadFromFile() throws IOException + { + UUID hostId = UUID.randomUUID(); + int version = HintsDescriptor.CURRENT_VERSION; + long timestamp = System.currentTimeMillis(); + ImmutableMap parameters = ImmutableMap.of(); + HintsDescriptor expected = new HintsDescriptor(hostId, version, timestamp, parameters); + + File directory = Files.createTempDir(); + try + { + try (HintsWriter ignored = HintsWriter.create(directory, expected)) + { + } + HintsDescriptor actual = HintsDescriptor.readFromFile(new File(directory, expected.fileName()).toPath()); + assertEquals(expected, actual); + } + finally + { + directory.deleteOnExit(); + } + } + + private static void testSerializeDeserializeLoop(HintsDescriptor descriptor) throws IOException + { + // serialize to a byte array + byte[] bytes = serializeDescriptor(descriptor); + // make sure the sizes match + assertEquals(bytes.length, descriptor.serializedSize()); + // deserialize back + HintsDescriptor deserializedDescriptor = deserializeDescriptor(bytes); + // compare equality + assertDescriptorsEqual(descriptor, deserializedDescriptor); + } + + private static byte[] serializeDescriptor(HintsDescriptor descriptor) throws IOException + { + DataOutputBuffer dob = new DataOutputBuffer(); + descriptor.serialize(dob); + return dob.toByteArray(); + } + + private static HintsDescriptor deserializeDescriptor(byte[] bytes) throws IOException + { + DataInput in = ByteStreams.newDataInput(bytes); + return HintsDescriptor.deserialize(in); + } + + private static void assertDescriptorsEqual(HintsDescriptor expected, HintsDescriptor actual) + { + assertNotSame(expected, actual); + assertEquals(expected, actual); + assertEquals(expected.hashCode(), actual.hashCode()); + assertEquals(expected.hostId, actual.hostId); + assertEquals(expected.version, actual.version); + assertEquals(expected.timestamp, actual.timestamp); + assertEquals(expected.parameters, actual.parameters); + } +} diff --git a/test/unit/org/apache/cassandra/hints/HintsTestUtil.java b/test/unit/org/apache/cassandra/hints/HintsTestUtil.java new file mode 100644 index 0000000000..89b532f155 --- /dev/null +++ b/test/unit/org/apache/cassandra/hints/HintsTestUtil.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.hints; + +import java.util.UUID; + +import com.google.common.collect.Iterators; + +import org.apache.cassandra.db.Mutation; +import org.apache.cassandra.db.partitions.AbstractBTreePartition; +import org.apache.cassandra.db.partitions.PartitionUpdate; + +import static junit.framework.Assert.assertEquals; +import static junit.framework.Assert.assertTrue; + +final class HintsTestUtil +{ + static void assertMutationsEqual(Mutation expected, Mutation actual) + { + assertEquals(expected.key(), actual.key()); + assertEquals(expected.getPartitionUpdates().size(), actual.getPartitionUpdates().size()); + + for (UUID id : expected.getColumnFamilyIds()) + assertPartitionsEqual(expected.getPartitionUpdate(id), actual.getPartitionUpdate(id)); + } + + static void assertPartitionsEqual(AbstractBTreePartition expected, AbstractBTreePartition actual) + { + assertEquals(expected.partitionKey(), actual.partitionKey()); + assertEquals(expected.deletionInfo(), actual.deletionInfo()); + assertEquals(expected.columns(), actual.columns()); + assertTrue(Iterators.elementsEqual(expected.iterator(), actual.iterator())); + } + + static void assertHintsEqual(Hint expected, Hint actual) + { + assertEquals(expected.mutation.getKeyspaceName(), actual.mutation.getKeyspaceName()); + assertEquals(expected.mutation.key(), actual.mutation.key()); + assertEquals(expected.mutation.getColumnFamilyIds(), actual.mutation.getColumnFamilyIds()); + for (PartitionUpdate partitionUpdate : expected.mutation.getPartitionUpdates()) + assertPartitionsEqual(partitionUpdate, actual.mutation.getPartitionUpdate(partitionUpdate.metadata().cfId)); + assertEquals(expected.creationTime, actual.creationTime); + assertEquals(expected.gcgs, actual.gcgs); + } +} diff --git a/test/unit/org/apache/cassandra/hints/LegacyHintsMigratorTest.java b/test/unit/org/apache/cassandra/hints/LegacyHintsMigratorTest.java new file mode 100644 index 0000000000..85e4b69ac0 --- /dev/null +++ b/test/unit/org/apache/cassandra/hints/LegacyHintsMigratorTest.java @@ -0,0 +1,195 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.hints; + +import java.io.File; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.file.Files; +import java.util.*; + +import org.junit.BeforeClass; +import org.junit.Test; + +import org.apache.cassandra.SchemaLoader; +import org.apache.cassandra.config.CFMetaData; +import org.apache.cassandra.config.Schema; +import org.apache.cassandra.db.*; +import org.apache.cassandra.db.marshal.UUIDType; +import org.apache.cassandra.db.partitions.PartitionUpdate; +import org.apache.cassandra.db.rows.BTreeRow; +import org.apache.cassandra.db.rows.BufferCell; +import org.apache.cassandra.db.rows.Cell; +import org.apache.cassandra.net.MessagingService; +import org.apache.cassandra.schema.KeyspaceParams; +import org.apache.cassandra.utils.FBUtilities; +import org.apache.cassandra.utils.UUIDGen; + +import static junit.framework.Assert.assertEquals; +import static junit.framework.Assert.assertNotNull; +import static junit.framework.Assert.assertTrue; + +import static org.apache.cassandra.hints.HintsTestUtil.assertMutationsEqual; +import static org.apache.cassandra.utils.ByteBufferUtil.bytes; + +// TODO: test split into several files +@SuppressWarnings("deprecation") +public class LegacyHintsMigratorTest +{ + private static final String KEYSPACE = "legacy_hints_migrator_test"; + private static final String TABLE = "table"; + + @BeforeClass + public static void defineSchema() + { + SchemaLoader.prepareServer(); + SchemaLoader.createKeyspace(KEYSPACE, KeyspaceParams.simple(1), SchemaLoader.standardCFMD(KEYSPACE, TABLE)); + } + + @Test + public void testNothingToMigrate() throws IOException + { + File directory = Files.createTempDirectory(null).toFile(); + try + { + testNothingToMigrate(directory); + } + finally + { + directory.deleteOnExit(); + } + } + + private static void testNothingToMigrate(File directory) + { + // truncate system.hints to enseure nothing inside + Keyspace.open(SystemKeyspace.NAME).getColumnFamilyStore(SystemKeyspace.LEGACY_HINTS).truncateBlocking(); + new LegacyHintsMigrator(directory, 128 * 1024 * 1024).migrate(); + HintsCatalog catalog = HintsCatalog.load(directory); + assertEquals(0, catalog.stores().count()); + } + + @Test + public void testMigrationIsComplete() throws IOException + { + File directory = Files.createTempDirectory(null).toFile(); + try + { + testMigrationIsComplete(directory); + } + finally + { + directory.deleteOnExit(); + } + } + + private static void testMigrationIsComplete(File directory) + { + long timestamp = System.currentTimeMillis(); + + // write 100 mutations for each of the 10 generated endpoints + Map> mutations = new HashMap<>(); + for (int i = 0; i < 10; i++) + { + UUID hostId = UUID.randomUUID(); + Queue queue = new LinkedList<>(); + mutations.put(hostId, queue); + + for (int j = 0; j < 100; j++) + { + Mutation mutation = createMutation(j, timestamp + j); + queue.offer(mutation); + Mutation legacyHint = createLegacyHint(mutation, timestamp, hostId); + legacyHint.applyUnsafe(); + } + } + + // run the migration + new LegacyHintsMigrator(directory, 128 * 1024 * 1024).migrate(); + + // validate that the hints table is truncated now + assertTrue(Keyspace.open(SystemKeyspace.NAME).getColumnFamilyStore(SystemKeyspace.LEGACY_HINTS).isEmpty()); + + HintsCatalog catalog = HintsCatalog.load(directory); + + // assert that we've correctly loaded 10 hints stores + assertEquals(10, catalog.stores().count()); + + // for each of the 10 stores, make sure the mutations have been migrated correctly + for (Map.Entry> entry : mutations.entrySet()) + { + HintsStore store = catalog.get(entry.getKey()); + assertNotNull(store); + + HintsDescriptor descriptor = store.poll(); + assertNotNull(descriptor); + + // read all the hints + Queue actualHints = new LinkedList<>(); + try (HintsReader reader = HintsReader.open(new File(directory, descriptor.fileName()))) + { + for (HintsReader.Page page : reader) + page.hintsIterator().forEachRemaining(actualHints::offer); + } + + // assert the size matches + assertEquals(100, actualHints.size()); + + // compare expected hints to actual hints + for (int i = 0; i < 100; i++) + { + Hint hint = actualHints.poll(); + Mutation mutation = entry.getValue().poll(); + int ttl = mutation.smallestGCGS(); + + assertEquals(timestamp, hint.creationTime); + assertEquals(ttl, hint.gcgs); + assertMutationsEqual(mutation, hint.mutation); + } + } + } + + // legacy hint mutation creation code, copied more or less verbatim from the previous implementation + private static Mutation createLegacyHint(Mutation mutation, long now, UUID targetId) + { + int version = MessagingService.VERSION_21; + int ttl = mutation.smallestGCGS(); + UUID hintId = UUIDGen.getTimeUUID(); + + ByteBuffer key = UUIDType.instance.decompose(targetId); + Clustering clustering = SystemKeyspace.LegacyHints.comparator.make(hintId, version); + ByteBuffer value = ByteBuffer.wrap(FBUtilities.serialize(mutation, Mutation.serializer, version)); + Cell cell = BufferCell.expiring(SystemKeyspace.LegacyHints.compactValueColumn(), + now, + ttl, + FBUtilities.nowInSeconds(), + value); + return new Mutation(PartitionUpdate.singleRowUpdate(SystemKeyspace.LegacyHints, + key, + BTreeRow.singleCellRow(clustering, cell))); + } + + private static Mutation createMutation(int index, long timestamp) + { + CFMetaData table = Schema.instance.getCFMetaData(KEYSPACE, TABLE); + return new RowUpdateBuilder(table, timestamp, bytes(index)) + .clustering(bytes(index)) + .add("val", bytes(index)) + .build(); + } +} diff --git a/test/unit/org/apache/cassandra/metrics/HintedHandOffMetricsTest.java b/test/unit/org/apache/cassandra/metrics/HintedHandOffMetricsTest.java new file mode 100644 index 0000000000..6f76db4f64 --- /dev/null +++ b/test/unit/org/apache/cassandra/metrics/HintedHandOffMetricsTest.java @@ -0,0 +1,56 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +package org.apache.cassandra.metrics; + +import java.net.InetAddress; +import java.util.Map; +import java.util.UUID; + +import org.junit.Test; + +import com.google.common.collect.Iterators; + +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.cql3.UntypedResultSet; +import org.apache.cassandra.db.SystemKeyspace; +import org.apache.cassandra.db.marshal.Int32Type; +import org.apache.cassandra.db.marshal.UUIDType; +import org.apache.cassandra.hints.HintsService; + +import static org.junit.Assert.assertEquals; +import static org.apache.cassandra.cql3.QueryProcessor.executeInternal; + +public class HintedHandOffMetricsTest +{ + @Test + public void testHintsMetrics() throws Exception + { + DatabaseDescriptor.getHintsDirectory().mkdirs(); + + for (int i = 0; i < 99; i++) + HintsService.instance.metrics.incrPastWindow(InetAddress.getLocalHost()); + HintsService.instance.metrics.log(); + + UntypedResultSet rows = executeInternal("SELECT hints_dropped FROM system." + SystemKeyspace.PEER_EVENTS); + Map returned = rows.one().getMap("hints_dropped", UUIDType.instance, Int32Type.instance); + assertEquals(Iterators.getLast(returned.values().iterator()).intValue(), 99); + } +} diff --git a/test/unit/org/apache/cassandra/service/StorageProxyTest.java b/test/unit/org/apache/cassandra/service/StorageProxyTest.java index 801fc53e00..42eb1f583c 100644 --- a/test/unit/org/apache/cassandra/service/StorageProxyTest.java +++ b/test/unit/org/apache/cassandra/service/StorageProxyTest.java @@ -25,9 +25,9 @@ import org.junit.BeforeClass; import org.junit.Test; import org.apache.cassandra.db.PartitionPosition; +import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.dht.*; import org.apache.cassandra.locator.TokenMetadata; -import org.apache.cassandra.utils.ByteBufferUtil; import static org.apache.cassandra.Util.rp; import static org.apache.cassandra.Util.token; @@ -78,6 +78,7 @@ public class StorageProxyTest @BeforeClass public static void beforeClass() throws Throwable { + DatabaseDescriptor.getHintsDirectory().mkdir(); TokenMetadata tmd = StorageService.instance.getTokenMetadata(); tmd.updateNormalToken(token("1"), InetAddress.getByName("127.0.0.1")); tmd.updateNormalToken(token("6"), InetAddress.getByName("127.0.0.6"));