Merge branch 'cassandra-5.0' into trunk

CHANGES.txt

@@ -3,6 +3,7 @@
  * Add the ability to disable bulk loading of SSTables (CASSANDRA-18781)
  * Clean up obsolete functions and simplify cql_version handling in cqlsh (CASSANDRA-18787)
 Merged from 5.0:
+ * Add support for repair coordinator to retry messages that timeout (CASSANDRA-18816)
  * Upgrade slf4j-api to 1.7.36 (CASSANDRA-18882)
  * Make the output of ON/OFF commands in cqlsh consistent (CASSANDRA-18547)
  * Do not create sstable files before registering in txn (CASSANDRA-18737)

conf/cassandra.yaml

@@ -723,6 +723,17 @@ memtable_allocation_type: heap_buffers
 # Min unit: MiB
 # repair_session_space:
 
+# repair:
+#   # Configure the retries for each of the repair messages that support it.  As of this moment retries use an exponential algorithm where each attempt sleeps longer based off the base_sleep_time and attempt.
+#   retries:
+#     max_attempts: 10
+#     base_sleep_time: 200ms
+#     max_sleep_time: 1s
+#     # Increase the timeout of validation responses due to them containing the merkle tree
+#     merkle_tree_response:
+#       base_sleep_time: 30s
+#       max_sleep_time: 1m
+
 # Total space to use for commit logs on disk.
 #
 # If space gets above this value, Cassandra will flush every dirty CF

src/java/org/apache/cassandra/concurrent/Stage.java

@@ -144,6 +144,12 @@ public enum Stage
         return executor;
     }
 
+    @VisibleForTesting
+    public void unsafeSetExecutor(ExecutorPlus executor)
+    {
+        this.executor = executor;
+    }
+
     private static List<ExecutorPlus> executors()
     {
         return Stream.of(Stage.values())

src/java/org/apache/cassandra/config/Config.java

@@ -1106,6 +1106,8 @@ public class Config
     public volatile long min_tracked_partition_tombstone_count = 5000;
     public volatile boolean top_partitions_enabled = true;
 
+    public final RepairConfig repair = new RepairConfig();
+
     /**
      * Default compaction configuration, used if a table does not specify any.
      */

src/java/org/apache/cassandra/config/DatabaseDescriptor.java

@@ -3695,6 +3695,12 @@ public class DatabaseDescriptor
         return localDC;
     }
 
+    @VisibleForTesting
+    public static void setLocalDataCenter(String value)
+    {
+        localDC = value;
+    }
+
     public static Comparator<Replica> getLocalComparator()
     {
         return localComparator;
@@ -4906,4 +4912,9 @@ public class DatabaseDescriptor
     {
         return conf.sai_options.segment_write_buffer_size;
     }
+
+    public static RepairRetrySpec getRepairRetrySpec()
+    {
+        return conf == null ? new RepairRetrySpec() : conf.repair.retries;
+    }
 }

src/java/org/apache/cassandra/config/RepairConfig.java

@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.config;
+
+public class RepairConfig
+{
+    public final RepairRetrySpec retries = new RepairRetrySpec();
+}

src/java/org/apache/cassandra/config/RepairRetrySpec.java

@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.config;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+
+public class RepairRetrySpec extends RetrySpec
+{
+    public RetrySpec.Partial merkle_tree_response = null;
+
+    public boolean isMerkleTreeRetriesEnabled()
+    {
+        RetrySpec.Partial partial = merkle_tree_response;
+        if (partial == null || partial.maxAttempts == null)
+            return isEnabled();
+        return partial.isEnabled();
+    }
+
+    @JsonIgnore
+    public RetrySpec getMerkleTreeResponseSpec()
+    {
+        RetrySpec.Partial partial = merkle_tree_response;
+        if (partial == null)
+            return this;
+        return partial.withDefaults(this);
+    }
+}

src/java/org/apache/cassandra/config/RetrySpec.java

@@ -0,0 +1,165 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.config;
+
+import java.util.Objects;
+
+import javax.annotation.Nullable;
+
+import org.apache.cassandra.config.DurationSpec.LongMillisecondsBound;
+
+public class RetrySpec
+{
+    public static class MaxAttempt
+    {
+        public static final MaxAttempt DISABLED = new MaxAttempt();
+
+        public final int value;
+
+        public MaxAttempt(int value)
+        {
+            if (value < 1)
+                throw new IllegalArgumentException("max attempt must be positive; but given " + value);
+            this.value = value;
+        }
+
+        private MaxAttempt()
+        {
+            value = 0;
+        }
+
+        @Override
+        public boolean equals(Object o)
+        {
+            if (this == o) return true;
+            if (o == null) return false;
+            if (o instanceof Integer) return this.value == ((Integer) o).intValue();
+            if (getClass() != o.getClass()) return false;
+            MaxAttempt that = (MaxAttempt) o;
+            return value == that.value;
+        }
+
+        @Override
+        public int hashCode()
+        {
+            return Objects.hash(value);
+        }
+
+        @Override
+        public String toString()
+        {
+            return Integer.toString(value);
+        }
+    }
+
+    public static class Partial extends RetrySpec
+    {
+        public Partial()
+        {
+            this.maxAttempts = null;
+            this.baseSleepTime = null;
+            this.maxSleepTime = null;
+        }
+
+        public RetrySpec withDefaults(RetrySpec defaultValues)
+        {
+            MaxAttempt maxAttempts = nonNull(this.maxAttempts, defaultValues.getMaxAttempts(), DEFAULT_MAX_ATTEMPTS);
+            LongMillisecondsBound baseSleepTime = nonNull(this.baseSleepTime, defaultValues.getBaseSleepTime(), DEFAULT_BASE_SLEEP);
+            LongMillisecondsBound maxSleepTime = nonNull(this.maxSleepTime, defaultValues.getMaxSleepTime(), DEFAULT_MAX_SLEEP);
+            return new RetrySpec(maxAttempts, baseSleepTime, maxSleepTime);
+        }
+
+        private static <T> T nonNull(@Nullable T left, @Nullable T right, T defaultValue)
+        {
+            if (left != null)
+                return left;
+            if (right != null)
+                return right;
+            return defaultValue;
+        }
+    }
+
+    public static final MaxAttempt DEFAULT_MAX_ATTEMPTS = MaxAttempt.DISABLED;
+    public static final LongMillisecondsBound DEFAULT_BASE_SLEEP = new LongMillisecondsBound("200ms");
+    public static final LongMillisecondsBound DEFAULT_MAX_SLEEP = new LongMillisecondsBound("1s");
+
+    /**
+     * Represents how many retry attempts are allowed.  If the value is 2, this will cause 2 retries + 1 original request, for a total of 3 requests!
+     * <p/>
+     * To disable, set to 0.
+     */
+    public MaxAttempt maxAttempts = DEFAULT_MAX_ATTEMPTS; // 2 retries, 1 original request; so 3 total
+    public LongMillisecondsBound baseSleepTime = DEFAULT_BASE_SLEEP;
+    public LongMillisecondsBound maxSleepTime = DEFAULT_MAX_SLEEP;
+
+    public RetrySpec()
+    {
+    }
+
+    public RetrySpec(MaxAttempt maxAttempts, LongMillisecondsBound baseSleepTime, LongMillisecondsBound maxSleepTime)
+    {
+        this.maxAttempts = maxAttempts;
+        this.baseSleepTime = baseSleepTime;
+        this.maxSleepTime = maxSleepTime;
+    }
+
+    public boolean isEnabled()
+    {
+        return maxAttempts != MaxAttempt.DISABLED;
+    }
+
+    public void setEnabled(boolean enabled)
+    {
+        if (!enabled)
+        {
+            maxAttempts = MaxAttempt.DISABLED;
+        }
+        else if (maxAttempts == MaxAttempt.DISABLED)
+        {
+            maxAttempts = new MaxAttempt(2);
+        }
+    }
+
+    @Nullable
+    public MaxAttempt getMaxAttempts()
+    {
+        return !isEnabled() ? null : maxAttempts;
+    }
+
+    @Nullable
+    public LongMillisecondsBound getBaseSleepTime()
+    {
+        return !isEnabled() ? null : baseSleepTime;
+    }
+
+    public LongMillisecondsBound getMaxSleepTime()
+    {
+        return !isEnabled() ? null : maxSleepTime;
+    }
+
+    @Override
+    public String toString()
+    {
+        return "RetrySpec{" +
+               "maxAttempts=" + maxAttempts +
+               ", baseSleepTime=" + baseSleepTime +
+               ", maxSleepTime=" + maxSleepTime +
+               '}';
+    }
+}

src/java/org/apache/cassandra/cql3/QueryProcessor.java

@@ -478,7 +478,7 @@ public class QueryProcessor implements QueryHandler
             }
             Future<List<Message<ReadResponse>>> future = FutureCombiner.allOf(commands.stream()
                                                                                       .map(rc -> Message.out(rc.verb(), rc))
-                                                                                      .map(m -> MessagingService.instance().<ReadResponse>sendWithResult(m, address))
+                                                                                      .map(m -> MessagingService.instance().<ReadCommand, ReadResponse>sendWithResult(m, address))
                                                                                       .collect(Collectors.toList()));
 
             ResultSetBuilder result = new ResultSetBuilder(select.getResultMetadata(), select.getSelection().newSelectors(options), false);

src/java/org/apache/cassandra/db/ColumnFamilyStore.java

@@ -2726,7 +2726,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner
                 // since truncation can happen at different times on different nodes, we need to make sure
                 // that any repairs are aborted, otherwise we might clear the data on one node and then
                 // stream in data that is actually supposed to have been deleted
-                ActiveRepairService.instance.abort((prs) -> prs.getTableIds().contains(metadata.id),
+                ActiveRepairService.instance().abort((prs) -> prs.getTableIds().contains(metadata.id),
                                                    "Stopping parent sessions {} due to truncation of tableId="+metadata.id);
                 data.notifyTruncated(truncatedAt);
 

src/java/org/apache/cassandra/db/Keyspace.java

@@ -674,7 +674,7 @@ public class Keyspace
 
     public Iterable<ColumnFamilyStore> getValidColumnFamilies(boolean allowIndexes,
                                                               boolean autoAddIndexes,
-                                                              String... cfNames) throws IOException
+                                                              String... cfNames)
     {
         Set<ColumnFamilyStore> valid = new HashSet<>();
 

src/java/org/apache/cassandra/db/ReadCommand.java

@@ -981,13 +981,13 @@ public abstract class ReadCommand extends AbstractReadQuery
             TimeUUID pendingRepair = sstable.getPendingRepair();
             if (pendingRepair != ActiveRepairService.NO_PENDING_REPAIR)
             {
-                if (ActiveRepairService.instance.consistent.local.isSessionFinalized(pendingRepair))
+                if (ActiveRepairService.instance().consistent.local.isSessionFinalized(pendingRepair))
                     return true;
 
                 // In the edge case where compaction is backed up long enough for the session to
                 // timeout and be purged by LocalSessions::cleanup, consider the sstable unrepaired
                 // as it will be marked unrepaired when compaction catches up
-                if (!ActiveRepairService.instance.consistent.local.sessionExists(pendingRepair))
+                if (!ActiveRepairService.instance().consistent.local.sessionExists(pendingRepair))
                     return false;
 
                 repairedDataInfo.markInconclusive();

src/java/org/apache/cassandra/db/compaction/CompactionManager.java

@@ -135,7 +135,7 @@ import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID;
  * a set via Tracker. New scheduling attempts will ignore currently compacting
  * sstables.
  */
-public class CompactionManager implements CompactionManagerMBean
+public class CompactionManager implements CompactionManagerMBean, ICompactionManager
 {
     public static final String MBEAN_OBJECT_NAME = "org.apache.cassandra.db:type=CompactionManager";
     private static final Logger logger = LoggerFactory.getLogger(CompactionManager.class);
@@ -893,7 +893,7 @@ public class CompactionManager implements CompactionManagerMBean
             ActiveRepairService.ParentRepairSession prs;
             try
             {
-                prs = ActiveRepairService.instance.getParentRepairSession(sessionID);
+                prs = ActiveRepairService.instance().getParentRepairSession(sessionID);
             }
             catch (NoSuchRepairSessionException e)
             {
@@ -2193,6 +2193,7 @@ public class CompactionManager implements CompactionManagerMBean
         return metrics.totalCompactionsCompleted.getCount();
     }
 
+    @Override
     public int getPendingTasks()
     {
         return metrics.pendingTasks.getValue();

src/java/org/apache/cassandra/db/compaction/ICompactionManager.java

@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction;
+
+public interface ICompactionManager
+{
+    int getPendingTasks();
+}

src/java/org/apache/cassandra/db/compaction/PendingRepairManager.java

@@ -272,7 +272,7 @@ class PendingRepairManager
         if (compactionStrategy == null)
             return null;
         Set<SSTableReader> sstables = compactionStrategy.getSSTables();
-        long repairedAt = ActiveRepairService.instance.consistent.local.getFinalSessionRepairedAt(sessionID);
+        long repairedAt = ActiveRepairService.instance().consistent.local.getFinalSessionRepairedAt(sessionID);
         LifecycleTransaction txn = cfs.getTracker().tryModify(sstables, OperationType.COMPACTION);
         return txn == null ? null : new RepairFinishedCompactionTask(cfs, txn, sessionID, repairedAt);
     }
@@ -426,7 +426,7 @@ class PendingRepairManager
 
     boolean canCleanup(TimeUUID sessionID)
     {
-        return !ActiveRepairService.instance.consistent.local.isSessionInProgress(sessionID);
+        return !ActiveRepairService.instance().consistent.local.isSessionInProgress(sessionID);
     }
 
     @SuppressWarnings("resource")

src/java/org/apache/cassandra/db/repair/CassandraTableRepairManager.java

@@ -32,6 +32,7 @@ import org.apache.cassandra.dht.Range;
 import org.apache.cassandra.dht.Token;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
 import org.apache.cassandra.metrics.TopPartitionTracker;
+import org.apache.cassandra.repair.SharedContext;
 import org.apache.cassandra.repair.TableRepairManager;
 import org.apache.cassandra.repair.ValidationPartitionIterator;
 import org.apache.cassandra.repair.NoSuchRepairSessionException;
@@ -41,16 +42,23 @@ import org.apache.cassandra.service.ActiveRepairService;
 public class CassandraTableRepairManager implements TableRepairManager
 {
     private final ColumnFamilyStore cfs;
+    private final SharedContext ctx;
 
     public CassandraTableRepairManager(ColumnFamilyStore cfs)
+    {
+        this(cfs, SharedContext.Global.instance);
+    }
+
+    public CassandraTableRepairManager(ColumnFamilyStore cfs, SharedContext ctx)
     {
         this.cfs = cfs;
+        this.ctx = ctx;
     }
 
     @Override
     public ValidationPartitionIterator getValidationIterator(Collection<Range<Token>> ranges, TimeUUID parentId, TimeUUID sessionID, boolean isIncremental, long nowInSec, TopPartitionTracker.Collector topPartitionCollector) throws IOException, NoSuchRepairSessionException
     {
-        return new CassandraValidationIterator(cfs, ranges, parentId, sessionID, isIncremental, nowInSec, topPartitionCollector);
+        return new CassandraValidationIterator(cfs, ctx, ranges, parentId, sessionID, isIncremental, nowInSec, topPartitionCollector);
     }
 
     @Override
@@ -70,7 +78,7 @@ public class CassandraTableRepairManager implements TableRepairManager
     {
         try
         {
-            ActiveRepairService.instance.snapshotExecutor.submit(() -> {
+            ActiveRepairService.instance().snapshotExecutor.submit(() -> {
                 if (force || !cfs.snapshotExists(name))
                 {
                     cfs.snapshot(name, new Predicate<SSTableReader>()

src/java/org/apache/cassandra/db/repair/CassandraValidationIterator.java

@@ -52,6 +52,7 @@ import org.apache.cassandra.dht.Token;
 import org.apache.cassandra.io.sstable.ISSTableScanner;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
 import org.apache.cassandra.metrics.TopPartitionTracker;
+import org.apache.cassandra.repair.SharedContext;
 import org.apache.cassandra.repair.ValidationPartitionIterator;
 import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.service.ActiveRepairService;
@@ -112,11 +113,11 @@ public class CassandraValidationIterator extends ValidationPartitionIterator
     }
 
     @VisibleForTesting
-    public static synchronized Refs<SSTableReader> getSSTablesToValidate(ColumnFamilyStore cfs, Collection<Range<Token>> ranges, TimeUUID parentId, boolean isIncremental) throws NoSuchRepairSessionException
+    public static synchronized Refs<SSTableReader> getSSTablesToValidate(ColumnFamilyStore cfs, SharedContext ctx, Collection<Range<Token>> ranges, TimeUUID parentId, boolean isIncremental) throws NoSuchRepairSessionException
     {
         Refs<SSTableReader> sstables;
 
-        ActiveRepairService.ParentRepairSession prs = ActiveRepairService.instance.getParentRepairSession(parentId);
+        ActiveRepairService.ParentRepairSession prs = ctx.repair().getParentRepairSession(parentId);
 
         Set<SSTableReader> sstablesToValidate = new HashSet<>();
 
@@ -158,6 +159,7 @@ public class CassandraValidationIterator extends ValidationPartitionIterator
     }
 
     private final ColumnFamilyStore cfs;
+    private final SharedContext ctx;
     private final Refs<SSTableReader> sstables;
     private final String snapshotName;
     private final boolean isGlobalSnapshotValidation;
@@ -172,9 +174,10 @@ public class CassandraValidationIterator extends ValidationPartitionIterator
     private final long estimatedPartitions;
     private final Map<Range<Token>, Long> rangePartitionCounts;
 
-    public CassandraValidationIterator(ColumnFamilyStore cfs, Collection<Range<Token>> ranges, TimeUUID parentId, TimeUUID sessionID, boolean isIncremental, long nowInSec, TopPartitionTracker.Collector topPartitionCollector) throws IOException, NoSuchRepairSessionException
+    public CassandraValidationIterator(ColumnFamilyStore cfs, SharedContext ctx, Collection<Range<Token>> ranges, TimeUUID parentId, TimeUUID sessionID, boolean isIncremental, long nowInSec, TopPartitionTracker.Collector topPartitionCollector) throws IOException, NoSuchRepairSessionException
     {
         this.cfs = cfs;
+        this.ctx = ctx;
 
         isGlobalSnapshotValidation = cfs.snapshotExists(parentId.toString());
         if (isGlobalSnapshotValidation)
@@ -198,7 +201,7 @@ public class CassandraValidationIterator extends ValidationPartitionIterator
                 cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.VALIDATION);
                 // Note: we also flush for incremental repair during the anti-compaction process.
             }
-            sstables = getSSTablesToValidate(cfs, ranges, parentId, isIncremental);
+            sstables = getSSTablesToValidate(cfs, ctx, ranges, parentId, isIncremental);
         }
 
         // Persistent memtables will not flush or snapshot to sstables, make an sstable with their data.
@@ -208,7 +211,7 @@ public class CassandraValidationIterator extends ValidationPartitionIterator
 
         Preconditions.checkArgument(sstables != null);
 
-        ActiveRepairService.ParentRepairSession prs = ActiveRepairService.instance.getParentRepairSession(parentId);
+        ActiveRepairService.ParentRepairSession prs = ctx.repair().getParentRepairSession(parentId);
         logger.info("{}, parentSessionId={}: Performing validation compaction on {} sstables in {}.{}",
                     prs.previewKind.logPrefix(sessionID),
                     parentId,

src/java/org/apache/cassandra/db/repair/PendingAntiCompaction.java

@@ -137,7 +137,7 @@ public class PendingAntiCompaction
             // non-finalized sessions for a later error message
             if (metadata.pendingRepair != NO_PENDING_REPAIR)
             {
-                if (!ActiveRepairService.instance.consistent.local.isSessionFinalized(metadata.pendingRepair))
+                if (!ActiveRepairService.instance().consistent.local.isSessionFinalized(metadata.pendingRepair))
                 {
                     String message = String.format("Prepare phase for incremental repair session %s has failed because it encountered " +
                                                    "intersecting sstables belonging to another incremental repair session (%s). This is " +

src/java/org/apache/cassandra/db/virtual/LocalRepairTables.java

@@ -27,6 +27,7 @@ import java.util.UUID;
 import java.util.stream.Collectors;
 
 import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableSet;
 
 import org.apache.cassandra.cql3.statements.schema.CreateTableStatement;
 import org.apache.cassandra.db.DecoratedKey;
@@ -115,7 +116,7 @@ public class LocalRepairTables
         public DataSet data()
         {
             SimpleDataSet result = new SimpleDataSet(metadata());
-            ActiveRepairService.instance.coordinators().forEach(s -> updateDataset(result, s));
+            ActiveRepairService.instance().coordinators().forEach(s -> updateDataset(result, s));
             return result;
         }
 
@@ -123,7 +124,7 @@ public class LocalRepairTables
         {
             TimeUUID id = TimeUUIDType.instance.compose(partitionKey.getKey());
             SimpleDataSet result = new SimpleDataSet(metadata());
-            CoordinatorState state = ActiveRepairService.instance.coordinator(id);
+            CoordinatorState state = ActiveRepairService.instance().coordinator(id);
             if (state != null)
                 updateDataset(result, state);
             return result;
@@ -212,7 +213,7 @@ public class LocalRepairTables
         public DataSet data()
         {
             SimpleDataSet result = new SimpleDataSet(metadata());
-            ActiveRepairService.instance.coordinators().stream()
+            ActiveRepairService.instance().coordinators().stream()
                                                 .flatMap(s -> s.getSessions().stream())
                                                 .forEach(s -> updateDataset(result, s));
             return result;
@@ -251,7 +252,7 @@ public class LocalRepairTables
         public DataSet data()
         {
             SimpleDataSet result = new SimpleDataSet(metadata());
-            ActiveRepairService.instance.coordinators().stream()
+            ActiveRepairService.instance().coordinators().stream()
                                                 .flatMap(s -> s.getSessions().stream())
                                                 .flatMap(s -> s.getJobs().stream())
                                                 .forEach(s -> updateDataset(result, s));
@@ -291,7 +292,7 @@ public class LocalRepairTables
         public DataSet data()
         {
             SimpleDataSet result = new SimpleDataSet(metadata());
-            ActiveRepairService.instance.participates().stream()
+            ActiveRepairService.instance().participates().stream()
                                         .forEach(s -> updateDataset(result, s));
             return result;
         }
@@ -301,7 +302,7 @@ public class LocalRepairTables
         {
             TimeUUID id = TimeUUIDType.instance.compose(partitionKey.getKey());
             SimpleDataSet result = new SimpleDataSet(metadata());
-            ParticipateState state = ActiveRepairService.instance.participate(id);
+            ParticipateState state = ActiveRepairService.instance().participate(id);
             if (state != null)
                 updateDataset(result, state);
             return result;
@@ -322,7 +323,7 @@ public class LocalRepairTables
             result.column("preview_kind", state.previewKind.name());
             if (state.repairedAt != 0)
                 result.column("repaired_at", new Date(state.repairedAt));
-            result.column("validations", state.validationIds());
+            result.column("validations", ImmutableSet.copyOf(state.validationIds()));
             result.column("ranges", toStringList(state.ranges));
         }
     }
@@ -352,7 +353,7 @@ public class LocalRepairTables
         public DataSet data()
         {
             SimpleDataSet result = new SimpleDataSet(metadata());
-            ActiveRepairService.instance.validations().stream()
+            ActiveRepairService.instance().validations().stream()
                                                      .forEach(s -> updateDataset(result, s));
             return result;
         }
@@ -362,7 +363,7 @@ public class LocalRepairTables
         {
             UUID id = UUIDType.instance.compose(partitionKey.getKey());
             SimpleDataSet result = new SimpleDataSet(metadata());
-            ValidationState state = ActiveRepairService.instance.validation(id);
+            ValidationState state = ActiveRepairService.instance().validation(id);
             if (state != null)
                 updateDataset(result, state);
             return result;

src/java/org/apache/cassandra/exceptions/RepairException.java

@@ -17,6 +17,8 @@
  */
 package org.apache.cassandra.exceptions;
 
+import javax.annotation.Nullable;
+
 import org.apache.cassandra.repair.RepairJobDesc;
 import org.apache.cassandra.streaming.PreviewKind;
 
@@ -27,9 +29,9 @@ public class RepairException extends Exception
 {
     private final boolean shouldLogWarn;
 
-    private RepairException(RepairJobDesc desc, PreviewKind previewKind, String message, boolean shouldLogWarn)
+    private RepairException(@Nullable RepairJobDesc desc, PreviewKind previewKind, String message, boolean shouldLogWarn)
     {
-        this(desc.toString(previewKind != null ? previewKind : PreviewKind.NONE) + ' ' + message, shouldLogWarn);
+        this((desc == null ? "" : desc.toString(previewKind != null ? previewKind : PreviewKind.NONE)) + ' ' + message, shouldLogWarn);
     }
 
     private RepairException(String msg, boolean shouldLogWarn)
@@ -38,12 +40,12 @@ public class RepairException extends Exception
         this.shouldLogWarn = shouldLogWarn;
     }
 
-    public static RepairException error(RepairJobDesc desc, PreviewKind previewKind, String message)
+    public static RepairException error(@Nullable RepairJobDesc desc, PreviewKind previewKind, String message)
     {
         return new RepairException(desc, previewKind, message, false);
     }
 
-    public static RepairException warn(RepairJobDesc desc, PreviewKind previewKind, String message)
+    public static RepairException warn(@Nullable RepairJobDesc desc, PreviewKind previewKind, String message)
     {
         return new RepairException(desc, previewKind, message, true);
     }

src/java/org/apache/cassandra/gms/Gossiper.java

@@ -116,7 +116,7 @@ import static org.apache.cassandra.utils.FBUtilities.getBroadcastAddressAndPort;
  * This class is not threadsafe and any state changes should happen in the gossip stage.
  */
 
-public class Gossiper implements IFailureDetectionEventListener, GossiperMBean
+public class Gossiper implements IFailureDetectionEventListener, GossiperMBean, IGossiper
 {
     public static final String MBEAN_NAME = "org.apache.cassandra.net:type=Gossiper";
 
@@ -448,6 +448,7 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean
      *
      * @param subscriber module which implements the IEndpointStateChangeSubscriber
      */
+    @Override
     public void register(IEndpointStateChangeSubscriber subscriber)
     {
         subscribers.add(subscriber);
@@ -458,6 +459,7 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean
      *
      * @param subscriber module which implements the IEndpointStateChangeSubscriber
      */
+    @Override
     public void unregister(IEndpointStateChangeSubscriber subscriber)
     {
         subscribers.remove(subscriber);
@@ -1147,6 +1149,7 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean
         return storedTime == null ? computeExpireTime() : storedTime;
     }
 
+    @Override
     public EndpointState getEndpointStateForEndpoint(InetAddressAndPort ep)
     {
         return endpointStateMap.get(ep);
@@ -2319,13 +2322,6 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean
         return currentTimeMillis() + aVeryLongTime;
     }
 
-    @Nullable
-    public CassandraVersion getReleaseVersion(InetAddressAndPort ep)
-    {
-        EndpointState state = getEndpointStateForEndpoint(ep);
-        return state != null ? state.getReleaseVersion() : null;
-    }
-
     public Map<String, List<String>> getReleaseVersionsWithPort()
     {
         Map<String, List<String>> results = new HashMap<>();

src/java/org/apache/cassandra/gms/IGossiper.java

@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.gms;
+
+import javax.annotation.Nullable;
+
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.utils.CassandraVersion;
+
+public interface IGossiper
+{
+    void register(IEndpointStateChangeSubscriber subscriber);
+    void unregister(IEndpointStateChangeSubscriber subscriber);
+
+    @Nullable
+    EndpointState getEndpointStateForEndpoint(InetAddressAndPort ep);
+    @Nullable
+    default CassandraVersion getReleaseVersion(InetAddressAndPort ep)
+    {
+        EndpointState state = getEndpointStateForEndpoint(ep);
+        return state != null ? state.getReleaseVersion() : null;
+    }
+}

src/java/org/apache/cassandra/net/Message.java

@@ -102,7 +102,7 @@ public class Message<T>
         return header.verb;
     }
 
-    boolean isFailureResponse()
+    public boolean isFailureResponse()
     {
         return verb() == Verb.FAILURE_RSP;
     }
@@ -301,6 +301,11 @@ public class Message<T>
         return new Message<>(header.withParam(ParamType.FORWARD_TO, peers), payload);
     }
 
+    public Message<T> withFrom(InetAddressAndPort from)
+    {
+        return new Message<>(header.withFrom(from), payload);
+    }
+
     public Message<T> withFlag(MessageFlag flag)
     {
         return new Message<>(header.withFlag(flag), payload);
@@ -434,6 +439,11 @@ public class Message<T>
             this.params = params;
         }
 
+        Header withFrom(InetAddressAndPort from)
+        {
+            return new Header(id, verb, from, createdAtNanos, expiresAtNanos, flags, params);
+        }
+
         Header withFlag(MessageFlag flag)
         {
             return new Header(id, verb, from, createdAtNanos, expiresAtNanos, flag.addTo(flags), params);

src/java/org/apache/cassandra/net/MessageDelivery.java

@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.net;
+
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.utils.concurrent.Future;
+
+public interface MessageDelivery
+{
+    public <REQ> void send(Message<REQ> message, InetAddressAndPort to);
+    public <REQ, RSP> void sendWithCallback(Message<REQ> message, InetAddressAndPort to, RequestCallback<RSP> cb);
+    public <REQ, RSP> void sendWithCallback(Message<REQ> message, InetAddressAndPort to, RequestCallback<RSP> cb, ConnectionType specifyConnection);
+    public <REQ, RSP> Future<Message<RSP>> sendWithResult(Message<REQ> message, InetAddressAndPort to);
+    public <V> void respond(V response, Message<?> message);
+}

src/java/org/apache/cassandra/net/MessagingService.java

@@ -205,11 +205,43 @@ import static org.apache.cassandra.utils.Throwables.maybeFail;
  * implemented in {@link org.apache.cassandra.db.virtual.InternodeInboundTable} and
  * {@link org.apache.cassandra.db.virtual.InternodeOutboundTable} respectively.
  */
-public class MessagingService extends MessagingServiceMBeanImpl
+public class MessagingService extends MessagingServiceMBeanImpl implements MessageDelivery
 {
     private static final Logger logger = LoggerFactory.getLogger(MessagingService.class);
 
     // 8 bits version, so don't waste versions
+    public enum Version
+    {
+        @Deprecated
+        VERSION_30(10),
+        @Deprecated
+        VERSION_3014(11),
+        VERSION_40(12),
+        // c14227 TTL overflow, 'uint' timestamps
+        VERSION_50(13);
+
+        public static final Version CURRENT = DatabaseDescriptor.getStorageCompatibilityMode().isBefore(5) ? VERSION_40 : VERSION_50;
+
+        public final int value;
+
+        Version(int value)
+        {
+            this.value = value;
+        }
+
+        public static List<Version> supportedVersions()
+        {
+            List<Version> versions = Lists.newArrayList();
+            for (Version version : values())
+                if (minimum_version <= version.value)
+                    versions.add(version);
+
+            return Collections.unmodifiableList(versions);
+        }
+    }
+    // Maintance Note:
+    // Try to keep Version enum in-sync for testing.  By having the versions in the enum tests can get access without forcing this class
+    // to load, which adds a lot of costs to each test
     @Deprecated
     public static final int VERSION_30 = 10;
     @Deprecated
@@ -217,7 +249,7 @@ public class MessagingService extends MessagingServiceMBeanImpl
     public static final int VERSION_40 = 12;
     public static final int VERSION_50 = 13; // c14227 TTL overflow, 'uint' timestamps
     public static final int minimum_version = VERSION_40;
-    public static final int current_version = DatabaseDescriptor.getStorageCompatibilityMode().isBefore(5) ? VERSION_40 : VERSION_50;
+    public static final int current_version = Version.CURRENT.value;
     static AcceptVersions accept_messaging = new AcceptVersions(minimum_version, current_version);
     static AcceptVersions accept_streaming = new AcceptVersions(current_version, current_version);
     static Map<Integer, Integer> versionOrdinalMap = Arrays.stream(Version.values()).collect(Collectors.toMap(v -> v.value, v -> v.ordinal()));
@@ -238,33 +270,6 @@ public class MessagingService extends MessagingServiceMBeanImpl
         return ordinal;
     }
 
-    public enum Version
-    {
-        @Deprecated
-        VERSION_30(10),
-        @Deprecated
-        VERSION_3014(11),
-        VERSION_40(12),
-        VERSION_50(13);
-
-        public final int value;
-
-        Version(int value)
-        {
-            this.value = value;
-        }
-
-        public static List<Version> supportedVersions()
-        {
-            List<Version> versions = Lists.newArrayList();
-            for (Version version : values())
-                if (minimum_version <= version.value)
-                    versions.add(version);
-
-            return Collections.unmodifiableList(versions);
-        }
-    }
-
     private static class MSHandle
     {
         public static final MessagingService instance = new MessagingService(false);
@@ -312,13 +317,14 @@ public class MessagingService extends MessagingServiceMBeanImpl
         OutboundConnections.scheduleUnusedConnectionMonitoring(this, ScheduledExecutors.scheduledTasks, 1L, TimeUnit.HOURS);
     }
 
-    public <T> org.apache.cassandra.utils.concurrent.Future<Message<T>> sendWithResult(Message message, InetAddressAndPort to)
+    @Override
+    public <REQ, RSP> org.apache.cassandra.utils.concurrent.Future<Message<RSP>> sendWithResult(Message<REQ> message, InetAddressAndPort to)
     {
-        AsyncPromise<Message<T>> promise = new AsyncPromise<>();
-        MessagingService.instance().sendWithCallback(message, to, new RequestCallback<T>()
+        AsyncPromise<Message<RSP>> promise = new AsyncPromise<>();
+        sendWithCallback(message, to, new RequestCallback<RSP>()
         {
             @Override
-            public void onResponse(Message<T> msg)
+            public void onResponse(Message<RSP> msg)
             {
                 promise.trySuccess(msg);
             }

src/java/org/apache/cassandra/net/Verb.java

@@ -159,22 +159,23 @@ public enum Verb
     SCHEMA_VERSION_REQ     (20,  P1, rpcTimeout,      MIGRATION,         () -> NoPayload.serializer,                 () -> SchemaVersionVerbHandler.instance,   SCHEMA_VERSION_RSP  ),
 
     // repair; mostly doesn't use callbacks and sends responses as their own request messages, with matching sessions by uuid; should eventually harmonize and make idiomatic
+    // for the repair messages that implement retry logic, use rpcTimeout so the single request fails faster, then retries can be used to recover
     REPAIR_RSP             (100, P1, repairTimeout,   REQUEST_RESPONSE,  () -> NoPayload.serializer,                 () -> ResponseVerbHandler.instance                             ),
-    VALIDATION_RSP         (102, P1, longTimeout,     ANTI_ENTROPY,      () -> ValidationResponse.serializer,        () -> RepairMessageVerbHandler.instance,   REPAIR_RSP          ),
-    VALIDATION_REQ         (101, P1, repairTimeout,   ANTI_ENTROPY,      () -> ValidationRequest.serializer,         () -> RepairMessageVerbHandler.instance,   REPAIR_RSP          ),
-    SYNC_RSP               (104, P1, repairTimeout,   ANTI_ENTROPY,      () -> SyncResponse.serializer,              () -> RepairMessageVerbHandler.instance,   REPAIR_RSP          ),
-    SYNC_REQ               (103, P1, repairTimeout,   ANTI_ENTROPY,      () -> SyncRequest.serializer,               () -> RepairMessageVerbHandler.instance,   REPAIR_RSP          ),
-    PREPARE_MSG            (105, P1, repairTimeout,   ANTI_ENTROPY,      () -> PrepareMessage.serializer,            () -> RepairMessageVerbHandler.instance,   REPAIR_RSP          ),
-    SNAPSHOT_MSG           (106, P1, repairTimeout,   ANTI_ENTROPY,      () -> SnapshotMessage.serializer,           () -> RepairMessageVerbHandler.instance,   REPAIR_RSP          ),
-    CLEANUP_MSG            (107, P1, repairTimeout,   ANTI_ENTROPY,      () -> CleanupMessage.serializer,            () -> RepairMessageVerbHandler.instance,   REPAIR_RSP          ),
-    PREPARE_CONSISTENT_RSP (109, P1, repairTimeout,   ANTI_ENTROPY,      () -> PrepareConsistentResponse.serializer, () -> RepairMessageVerbHandler.instance,   REPAIR_RSP          ),
-    PREPARE_CONSISTENT_REQ (108, P1, repairTimeout,   ANTI_ENTROPY,      () -> PrepareConsistentRequest.serializer,  () -> RepairMessageVerbHandler.instance,   REPAIR_RSP          ),
-    FINALIZE_PROPOSE_MSG   (110, P1, repairTimeout,   ANTI_ENTROPY,      () -> FinalizePropose.serializer,           () -> RepairMessageVerbHandler.instance,   REPAIR_RSP          ),
-    FINALIZE_PROMISE_MSG   (111, P1, repairTimeout,   ANTI_ENTROPY,      () -> FinalizePromise.serializer,           () -> RepairMessageVerbHandler.instance,   REPAIR_RSP          ),
-    FINALIZE_COMMIT_MSG    (112, P1, repairTimeout,   ANTI_ENTROPY,      () -> FinalizeCommit.serializer,            () -> RepairMessageVerbHandler.instance,   REPAIR_RSP          ),
-    FAILED_SESSION_MSG     (113, P1, repairTimeout,   ANTI_ENTROPY,      () -> FailSession.serializer,               () -> RepairMessageVerbHandler.instance,   REPAIR_RSP          ),
-    STATUS_RSP             (115, P1, repairTimeout,   ANTI_ENTROPY,      () -> StatusResponse.serializer,            () -> RepairMessageVerbHandler.instance,   REPAIR_RSP          ),
-    STATUS_REQ             (114, P1, repairTimeout,   ANTI_ENTROPY,      () -> StatusRequest.serializer,             () -> RepairMessageVerbHandler.instance,   REPAIR_RSP          ),
+    VALIDATION_RSP         (102, P1, repairValidationRspTimeout,    ANTI_ENTROPY,      () -> ValidationResponse.serializer,        () -> RepairMessageVerbHandler.instance(),   REPAIR_RSP          ),
+    VALIDATION_REQ         (101, P1, repairWithBackoffTimeout,      ANTI_ENTROPY,      () -> ValidationRequest.serializer,         () -> RepairMessageVerbHandler.instance(),   REPAIR_RSP          ),
+    SYNC_RSP               (104, P1, repairWithBackoffTimeout,      ANTI_ENTROPY,      () -> SyncResponse.serializer,              () -> RepairMessageVerbHandler.instance(),   REPAIR_RSP          ),
+    SYNC_REQ               (103, P1, repairWithBackoffTimeout,      ANTI_ENTROPY,      () -> SyncRequest.serializer,               () -> RepairMessageVerbHandler.instance(),   REPAIR_RSP          ),
+    PREPARE_MSG            (105, P1, repairWithBackoffTimeout,      ANTI_ENTROPY,      () -> PrepareMessage.serializer,            () -> RepairMessageVerbHandler.instance(),   REPAIR_RSP          ),
+    SNAPSHOT_MSG           (106, P1, repairWithBackoffTimeout,      ANTI_ENTROPY,      () -> SnapshotMessage.serializer,           () -> RepairMessageVerbHandler.instance(),   REPAIR_RSP          ),
+    CLEANUP_MSG            (107, P1, repairWithBackoffTimeout,      ANTI_ENTROPY,      () -> CleanupMessage.serializer,            () -> RepairMessageVerbHandler.instance(),   REPAIR_RSP          ),
+    PREPARE_CONSISTENT_RSP (109, P1, repairTimeout,   ANTI_ENTROPY,      () -> PrepareConsistentResponse.serializer, () -> RepairMessageVerbHandler.instance(),   REPAIR_RSP          ),
+    PREPARE_CONSISTENT_REQ (108, P1, repairTimeout,   ANTI_ENTROPY,      () -> PrepareConsistentRequest.serializer,  () -> RepairMessageVerbHandler.instance(),   REPAIR_RSP          ),
+    FINALIZE_PROPOSE_MSG   (110, P1, repairTimeout,   ANTI_ENTROPY,      () -> FinalizePropose.serializer,           () -> RepairMessageVerbHandler.instance(),   REPAIR_RSP          ),
+    FINALIZE_PROMISE_MSG   (111, P1, repairTimeout,   ANTI_ENTROPY,      () -> FinalizePromise.serializer,           () -> RepairMessageVerbHandler.instance(),   REPAIR_RSP          ),
+    FINALIZE_COMMIT_MSG    (112, P1, repairTimeout,   ANTI_ENTROPY,      () -> FinalizeCommit.serializer,            () -> RepairMessageVerbHandler.instance(),   REPAIR_RSP          ),
+    FAILED_SESSION_MSG     (113, P1, repairTimeout,   ANTI_ENTROPY,      () -> FailSession.serializer,               () -> RepairMessageVerbHandler.instance(),   REPAIR_RSP          ),
+    STATUS_RSP             (115, P1, repairTimeout,   ANTI_ENTROPY,      () -> StatusResponse.serializer,            () -> RepairMessageVerbHandler.instance(),   REPAIR_RSP          ),
+    STATUS_REQ             (114, P1, repairTimeout,   ANTI_ENTROPY,      () -> StatusRequest.serializer,             () -> RepairMessageVerbHandler.instance(),   REPAIR_RSP          ),
 
     REPLICATION_DONE_RSP   (82,  P0, rpcTimeout,      MISC,              () -> NoPayload.serializer,                 () -> ResponseVerbHandler.instance                             ),
     REPLICATION_DONE_REQ   (22,  P0, rpcTimeout,      MISC,              () -> NoPayload.serializer,                 () -> ReplicationDoneVerbHandler.instance, REPLICATION_DONE_RSP),
@@ -475,4 +476,16 @@ class VerbTimeouts
     static final ToLongFunction<TimeUnit> pingTimeout     = DatabaseDescriptor::getPingTimeout;
     static final ToLongFunction<TimeUnit> longTimeout     = units -> Math.max(DatabaseDescriptor.getRpcTimeout(units), units.convert(5L, TimeUnit.MINUTES));
     static final ToLongFunction<TimeUnit> noTimeout       = units -> {  throw new IllegalStateException(); };
+
+    // repair verbs need to have different timeouts based off if retries are enabled or not
+    static final ToLongFunction<TimeUnit> repairWithBackoffTimeout      = units -> {
+        if (!DatabaseDescriptor.getRepairRetrySpec().isEnabled())
+            return repairTimeout.applyAsLong(units);
+        return rpcTimeout.applyAsLong(units);
+    };
+    static final ToLongFunction<TimeUnit> repairValidationRspTimeout    = units -> {
+        if (!DatabaseDescriptor.getRepairRetrySpec().isMerkleTreeRetriesEnabled())
+            return longTimeout.applyAsLong(units);
+        return rpcTimeout.applyAsLong(units);
+    };
 }

src/java/org/apache/cassandra/repair/AbstractRepairTask.java

@@ -31,8 +31,8 @@ import org.slf4j.LoggerFactory;
 import org.apache.cassandra.concurrent.ExecutorPlus;
 import org.apache.cassandra.dht.Range;
 import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.repair.messages.RepairOption;
-import org.apache.cassandra.service.ActiveRepairService;
 import org.apache.cassandra.utils.TimeUUID;
 import org.apache.cassandra.utils.concurrent.Future;
 import org.apache.cassandra.utils.concurrent.FutureCombiner;
@@ -41,15 +41,17 @@ public abstract class AbstractRepairTask implements RepairTask
 {
     protected static final Logger logger = LoggerFactory.getLogger(AbstractRepairTask.class);
 
+    protected final RepairCoordinator coordinator;
+    protected final InetAddressAndPort broadcastAddressAndPort;
     protected final RepairOption options;
     protected final String keyspace;
-    protected final RepairNotifier notifier;
 
-    protected AbstractRepairTask(RepairOption options, String keyspace, RepairNotifier notifier)
+    protected AbstractRepairTask(RepairCoordinator coordinator)
     {
-        this.options = Objects.requireNonNull(options);
-        this.keyspace = Objects.requireNonNull(keyspace);
-        this.notifier = Objects.requireNonNull(notifier);
+        this.coordinator = Objects.requireNonNull(coordinator);
+        this.broadcastAddressAndPort = coordinator.ctx.broadcastAddressAndPort();
+        this.options = Objects.requireNonNull(coordinator.state.options);
+        this.keyspace = Objects.requireNonNull(coordinator.state.keyspace);
     }
 
     private List<RepairSession> submitRepairSessions(TimeUUID parentSession,
@@ -63,18 +65,18 @@ public abstract class AbstractRepairTask implements RepairTask
         for (CommonRange commonRange : commonRanges)
         {
             logger.info("Starting RepairSession for {}", commonRange);
-            RepairSession session = ActiveRepairService.instance.submitRepairSession(parentSession,
-                                                                                     commonRange,
-                                                                                     keyspace,
-                                                                                     options.getParallelism(),
-                                                                                     isIncremental,
-                                                                                     options.isPullRepair(),
-                                                                                     options.getPreviewKind(),
-                                                                                     options.optimiseStreams(),
-                                                                                     options.repairPaxos(),
-                                                                                     options.paxosOnly(),
-                                                                                     executor,
-                                                                                     cfnames);
+            RepairSession session = coordinator.ctx.repair().submitRepairSession(parentSession,
+                                                                                 commonRange,
+                                                                                 keyspace,
+                                                                                 options.getParallelism(),
+                                                                                 isIncremental,
+                                                                                 options.isPullRepair(),
+                                                                                 options.getPreviewKind(),
+                                                                                 options.optimiseStreams(),
+                                                                                 options.repairPaxos(),
+                                                                                 options.paxosOnly(),
+                                                                                 executor,
+                                                                                 cfnames);
             if (session == null)
                 continue;
             session.addCallback(new RepairSessionCallback(session));
@@ -107,18 +109,20 @@ public abstract class AbstractRepairTask implements RepairTask
             this.session = session;
         }
 
+        @Override
         public void onSuccess(RepairSessionResult result)
         {
             String message = String.format("Repair session %s for range %s finished", session.getId(),
                                            session.ranges().toString());
-            notifier.notifyProgress(message);
+            coordinator.notifyProgress(message);
         }
 
+        @Override
         public void onFailure(Throwable t)
         {
             String message = String.format("Repair session %s for range %s failed with error %s",
                                            session.getId(), session.ranges().toString(), t.getMessage());
-            notifier.notifyError(new RuntimeException(message, t));
+            coordinator.notifyError(new RuntimeException(message, t));
         }
     }
 }

src/java/org/apache/cassandra/repair/AsymmetricRemoteSyncTask.java

@@ -29,7 +29,6 @@ import org.apache.cassandra.repair.messages.SyncRequest;
 import org.apache.cassandra.streaming.PreviewKind;
 import org.apache.cassandra.streaming.SessionSummary;
 import org.apache.cassandra.tracing.Tracing;
-import org.apache.cassandra.utils.FBUtilities;
 
 /**
  * AsymmetricRemoteSyncTask sends {@link SyncRequest} to target node to repair(stream)
@@ -39,14 +38,14 @@ import org.apache.cassandra.utils.FBUtilities;
  */
 public class AsymmetricRemoteSyncTask extends SyncTask implements CompletableRemoteSyncTask
 {
-    public AsymmetricRemoteSyncTask(RepairJobDesc desc, InetAddressAndPort to, InetAddressAndPort from, List<Range<Token>> differences, PreviewKind previewKind)
+    public AsymmetricRemoteSyncTask(SharedContext ctx, RepairJobDesc desc, InetAddressAndPort to, InetAddressAndPort from, List<Range<Token>> differences, PreviewKind previewKind)
     {
-        super(desc, to, from, differences, previewKind);
+        super(ctx, desc, to, from, differences, previewKind);
     }
 
     public void startSync()
     {
-        InetAddressAndPort local = FBUtilities.getBroadcastAddressAndPort();
+        InetAddressAndPort local = ctx.broadcastAddressAndPort();
         SyncRequest request = new SyncRequest(desc, local, nodePair.coordinator, nodePair.peer, rangesToSync, previewKind, true);
         String message = String.format("Forwarding streaming repair of %d ranges to %s (to be streamed with %s)", request.ranges.size(), request.src, request.dst);
         Tracing.traceRepair(message);

src/java/org/apache/cassandra/repair/IValidationManager.java

@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.repair;
+
+import java.util.concurrent.Future;
+
+import org.apache.cassandra.db.ColumnFamilyStore;
+
+public interface IValidationManager
+{
+    Future<?> submitValidation(ColumnFamilyStore cfs, Validator validator);
+}

src/java/org/apache/cassandra/repair/IncrementalRepairTask.java

@@ -25,26 +25,21 @@ import com.google.common.collect.ImmutableSet;
 import org.apache.cassandra.concurrent.ExecutorPlus;
 import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.repair.consistent.CoordinatorSession;
-import org.apache.cassandra.repair.messages.RepairOption;
-import org.apache.cassandra.service.ActiveRepairService;
-import org.apache.cassandra.utils.FBUtilities;
 import org.apache.cassandra.utils.TimeUUID;
 import org.apache.cassandra.utils.concurrent.Future;
 
 public class IncrementalRepairTask extends AbstractRepairTask
 {
     private final TimeUUID parentSession;
-    private final RepairRunnable.NeighborsAndRanges neighborsAndRanges;
+    private final RepairCoordinator.NeighborsAndRanges neighborsAndRanges;
     private final String[] cfnames;
 
-    protected IncrementalRepairTask(RepairOption options,
-                                    String keyspace,
-                                    RepairNotifier notifier,
+    protected IncrementalRepairTask(RepairCoordinator coordinator,
                                     TimeUUID parentSession,
-                                    RepairRunnable.NeighborsAndRanges neighborsAndRanges,
+                                    RepairCoordinator.NeighborsAndRanges neighborsAndRanges,
                                     String[] cfnames)
     {
-        super(options, keyspace, notifier);
+        super(coordinator);
         this.parentSession = parentSession;
         this.neighborsAndRanges = neighborsAndRanges;
         this.cfnames = cfnames;
@@ -62,12 +57,12 @@ public class IncrementalRepairTask extends AbstractRepairTask
         // the local node also needs to be included in the set of participants, since coordinator sessions aren't persisted
         Set<InetAddressAndPort> allParticipants = ImmutableSet.<InetAddressAndPort>builder()
                                                   .addAll(neighborsAndRanges.participants)
-                                                  .add(FBUtilities.getBroadcastAddressAndPort())
+                                                  .add(broadcastAddressAndPort)
                                                   .build();
         // Not necessary to include self for filtering. The common ranges only contains neighbhor node endpoints.
         List<CommonRange> allRanges = neighborsAndRanges.filterCommonRanges(keyspace, cfnames);
 
-        CoordinatorSession coordinatorSession = ActiveRepairService.instance.consistent.coordinated.registerSession(parentSession, allParticipants, neighborsAndRanges.shouldExcludeDeadParticipants);
+        CoordinatorSession coordinatorSession = coordinator.ctx.repair().consistent.coordinated.registerSession(parentSession, allParticipants, neighborsAndRanges.shouldExcludeDeadParticipants);
 
         return coordinatorSession.execute(() -> runRepair(parentSession, true, executor, allRanges, cfnames));
 

src/java/org/apache/cassandra/repair/LocalSyncTask.java

@@ -64,13 +64,13 @@ public class LocalSyncTask extends SyncTask implements StreamEventHandler
     private final AtomicBoolean active = new AtomicBoolean(true);
     private final Promise<StreamPlan> planPromise = new AsyncPromise<>();
 
-    public LocalSyncTask(RepairJobDesc desc, InetAddressAndPort local, InetAddressAndPort remote,
+    public LocalSyncTask(SharedContext ctx, RepairJobDesc desc, InetAddressAndPort local, InetAddressAndPort remote,
                          List<Range<Token>> diff, TimeUUID pendingRepair,
                          boolean requestRanges, boolean transferRanges, PreviewKind previewKind)
     {
-        super(desc, local, remote, diff, previewKind);
+        super(ctx, desc, local, remote, diff, previewKind);
         Preconditions.checkArgument(requestRanges || transferRanges, "Nothing to do in a sync job");
-        Preconditions.checkArgument(local.equals(FBUtilities.getBroadcastAddressAndPort()));
+        Preconditions.checkArgument(local.equals(ctx.broadcastAddressAndPort()));
 
         this.pendingRepair = pendingRepair;
         this.requestRanges = requestRanges;
@@ -119,7 +119,7 @@ public class LocalSyncTask extends SyncTask implements StreamEventHandler
             Tracing.traceRepair(message);
 
             StreamPlan plan = createStreamPlan();
-            plan.execute();
+            ctx.streamExecutor().execute(plan);
             planPromise.setSuccess(plan);
         }
     }
@@ -130,6 +130,7 @@ public class LocalSyncTask extends SyncTask implements StreamEventHandler
         return true;
     }
 
+    @Override
     public void handleStreamEvent(StreamEvent event)
     {
         if (state == null)
@@ -193,8 +194,9 @@ public class LocalSyncTask extends SyncTask implements StreamEventHandler
     }
 
     @Override
-    public void abort()
+    public void abort(Throwable reason)
     {
+        super.abort(reason);
         planPromise.addCallback((plan, cause) ->
         {
             assert plan != null : "StreamPlan future should never be completed exceptionally";

src/java/org/apache/cassandra/repair/NormalRepairTask.java

@@ -20,7 +20,6 @@ package org.apache.cassandra.repair;
 import java.util.List;
 
 import org.apache.cassandra.concurrent.ExecutorPlus;
-import org.apache.cassandra.repair.messages.RepairOption;
 import org.apache.cassandra.utils.TimeUUID;
 import org.apache.cassandra.utils.concurrent.Future;
 
@@ -30,14 +29,12 @@ public class NormalRepairTask extends AbstractRepairTask
     private final List<CommonRange> commonRanges;
     private final String[] cfnames;
 
-    protected NormalRepairTask(RepairOption options,
-                               String keyspace,
-                               RepairNotifier notifier,
+    protected NormalRepairTask(RepairCoordinator coordinator,
                                TimeUUID parentSession,
                                List<CommonRange> commonRanges,
                                String[] cfnames)
     {
-        super(options, keyspace, notifier);
+        super(coordinator);
         this.parentSession = parentSession;
         this.commonRanges = commonRanges;
         this.cfnames = cfnames;

src/java/org/apache/cassandra/repair/PreviewRepairTask.java

@@ -32,7 +32,6 @@ import org.apache.cassandra.dht.Token;
 import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.metrics.RepairMetrics;
 import org.apache.cassandra.repair.consistent.SyncStatSummary;
-import org.apache.cassandra.repair.messages.RepairOption;
 import org.apache.cassandra.streaming.PreviewKind;
 import org.apache.cassandra.utils.DiagnosticSnapshotService;
 import org.apache.cassandra.utils.TimeUUID;
@@ -45,9 +44,9 @@ public class PreviewRepairTask extends AbstractRepairTask
     private final String[] cfnames;
     private volatile String successMessage = name() + " completed successfully";
 
-    protected PreviewRepairTask(RepairOption options, String keyspace, RepairNotifier notifier, TimeUUID parentSession, List<CommonRange> commonRanges, String[] cfnames)
+    protected PreviewRepairTask(RepairCoordinator coordinator, TimeUUID parentSession, List<CommonRange> commonRanges, String[] cfnames)
     {
-        super(options, keyspace, notifier);
+        super(coordinator);
         this.parentSession = parentSession;
         this.commonRanges = commonRanges;
         this.cfnames = cfnames;
@@ -91,7 +90,7 @@ public class PreviewRepairTask extends AbstractRepairTask
                     maybeSnapshotReplicas(parentSession, keyspace, result.results.get()); // we know its present as summary used it
             }
             successMessage += "; " + message;
-            notifier.notification(message);
+            coordinator.notification(message);
 
             return result;
         });

src/java/org/apache/cassandra/repair/RepairCoordinator.java

@@ -17,7 +17,6 @@
  */
 package org.apache.cassandra.repair;
 
-import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -29,6 +28,9 @@ import java.util.UUID;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.BiFunction;
+import java.util.function.Function;
+import java.util.function.Supplier;
 
 import com.google.common.base.Preconditions;
 import com.google.common.collect.ImmutableList;
@@ -38,7 +40,8 @@ import com.google.common.collect.Iterables;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Sets;
 
-import org.apache.cassandra.utils.TimeUUID;
+import org.apache.cassandra.locator.RangesAtEndpoint;
+import org.apache.cassandra.utils.*;
 import org.apache.cassandra.utils.concurrent.Future;
 import org.apache.commons.lang3.time.DurationFormatUtils;
 import org.slf4j.Logger;
@@ -57,7 +60,6 @@ import org.apache.cassandra.db.Keyspace;
 import org.apache.cassandra.dht.Range;
 import org.apache.cassandra.dht.Token;
 import org.apache.cassandra.exceptions.RepairException;
-import org.apache.cassandra.gms.FailureDetector;
 import org.apache.cassandra.locator.EndpointsForRange;
 import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.locator.Replica;
@@ -74,44 +76,51 @@ import org.apache.cassandra.tracing.TraceKeyspace;
 import org.apache.cassandra.tracing.TraceState;
 import org.apache.cassandra.tracing.Tracing;
 import org.apache.cassandra.transport.messages.ResultMessage;
-import org.apache.cassandra.utils.FBUtilities;
-import org.apache.cassandra.utils.Throwables;
-import org.apache.cassandra.utils.WrappedRunnable;
 import org.apache.cassandra.utils.progress.ProgressEvent;
 import org.apache.cassandra.utils.progress.ProgressEventNotifier;
 import org.apache.cassandra.utils.progress.ProgressEventType;
 import org.apache.cassandra.utils.progress.ProgressListener;
 
-import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory;
 import static org.apache.cassandra.repair.state.AbstractState.COMPLETE;
 import static org.apache.cassandra.repair.state.AbstractState.INIT;
 import static org.apache.cassandra.service.QueryState.forInternalCalls;
-import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis;
-import static org.apache.cassandra.utils.Clock.Global.nanoTime;
 
-public class RepairRunnable implements Runnable, ProgressEventNotifier, RepairNotifier
+public class RepairCoordinator implements Runnable, ProgressEventNotifier, RepairNotifier
 {
-    private static final Logger logger = LoggerFactory.getLogger(RepairRunnable.class);
+    private static final Logger logger = LoggerFactory.getLogger(RepairCoordinator.class);
 
     private static final AtomicInteger THREAD_COUNTER = new AtomicInteger(1);
 
     public final CoordinatorState state;
-    private final StorageService storageService;
-
     private final String tag;
+    private final BiFunction<String, String[], Iterable<ColumnFamilyStore>> validColumnFamilies;
+    private final Function<String, RangesAtEndpoint> getLocalReplicas;
 
     private final List<ProgressListener> listeners = new ArrayList<>();
     private final AtomicReference<Throwable> firstError = new AtomicReference<>(null);
+    final SharedContext ctx;
 
     private TraceState traceState;
 
-    public RepairRunnable(StorageService storageService, int cmd, RepairOption options, String keyspace)
+    public RepairCoordinator(StorageService storageService, int cmd, RepairOption options, String keyspace)
     {
-        this.state = new CoordinatorState(cmd, keyspace, options);
-        this.storageService = storageService;
+        this(SharedContext.Global.instance,
+                (ks, tables) -> storageService.getValidColumnFamilies(false, false, ks, tables),
+                storageService::getLocalReplicas,
+                cmd, options, keyspace);
+    }
 
+    RepairCoordinator(SharedContext ctx,
+                      BiFunction<String, String[], Iterable<ColumnFamilyStore>> validColumnFamilies,
+                      Function<String, RangesAtEndpoint> getLocalReplicas,
+                      int cmd, RepairOption options, String keyspace)
+    {
+        this.ctx = ctx;
+        this.state = new CoordinatorState(ctx.clock(), cmd, keyspace, options);
         this.tag = "repair:" + cmd;
-        ActiveRepairService.instance.register(state);
+        this.validColumnFamilies = validColumnFamilies;
+        this.getLocalReplicas = getLocalReplicas;
+        ctx.repair().register(state);
     }
 
     @Override
@@ -187,7 +196,7 @@ public class RepairRunnable implements Runnable, ProgressEventNotifier, RepairNo
     {
         state.phase.success(msg);
         fireProgressEvent(jmxEvent(ProgressEventType.SUCCESS, msg));
-        ActiveRepairService.instance.recordRepairStatus(state.cmd, ActiveRepairService.ParentRepairStatus.COMPLETED,
+        ctx.repair().recordRepairStatus(state.cmd, ActiveRepairService.ParentRepairStatus.COMPLETED,
                                                         ImmutableList.of(msg));
         complete(null);
     }
@@ -197,14 +206,14 @@ public class RepairRunnable implements Runnable, ProgressEventNotifier, RepairNo
         if (reason == null)
         {
             Throwable error = firstError.get();
-            reason = error != null ? error.getMessage() : "Some repair failed";
+            reason = error != null ? error.toString() : "Some repair failed";
         }
         state.phase.fail(reason);
         String completionMessage = String.format("Repair command #%d finished with error", state.cmd);
 
         // Note we rely on the first message being the reason for the failure
         // when inspecting this state from RepairRunner.queryForCompletedRepair
-        ActiveRepairService.instance.recordRepairStatus(state.cmd, ParentRepairStatus.FAILED,
+        ctx.repair().recordRepairStatus(state.cmd, ParentRepairStatus.FAILED,
                                                         ImmutableList.of(reason, completionMessage));
 
         complete(completionMessage);
@@ -222,7 +231,7 @@ public class RepairRunnable implements Runnable, ProgressEventNotifier, RepairNo
         fireProgressEvent(jmxEvent(ProgressEventType.COMPLETE, msg));
         logger.info(state.options.getPreviewKind().logPrefix(state.id) + msg);
 
-        ActiveRepairService.instance.removeParentRepairSession(state.id);
+        ctx.repair().removeParentRepairSession(state.id);
         TraceState localState = traceState;
         if (state.options.isTraced() && localState != null)
         {
@@ -251,17 +260,17 @@ public class RepairRunnable implements Runnable, ProgressEventNotifier, RepairNo
         {
             skip(e.getMessage());
         }
-        catch (Exception | Error e)
+        catch (Throwable e)
         {
             notifyError(e);
             fail(e.getMessage());
         }
     }
 
-    private void runMayThrow() throws Exception
+    private void runMayThrow() throws Throwable
     {
         state.phase.setup();
-        ActiveRepairService.instance.recordRepairStatus(state.cmd, ParentRepairStatus.IN_PROGRESS, ImmutableList.of());
+        ctx.repair().recordRepairStatus(state.cmd, ParentRepairStatus.IN_PROGRESS, ImmutableList.of());
 
         List<ColumnFamilyStore> columnFamilies = getColumnFamilies();
         String[] cfnames = columnFamilies.stream().map(cfs -> cfs.name).toArray(String[]::new);
@@ -276,15 +285,36 @@ public class RepairRunnable implements Runnable, ProgressEventNotifier, RepairNo
 
         maybeStoreParentRepairStart(cfnames);
 
-        prepare(columnFamilies, neighborsAndRanges.participants, neighborsAndRanges.shouldExcludeDeadParticipants);
-
-        repair(cfnames, neighborsAndRanges);
+        prepare(columnFamilies, neighborsAndRanges.participants, neighborsAndRanges.shouldExcludeDeadParticipants)
+        .flatMap(ignore -> repair(cfnames, neighborsAndRanges))
+        .addCallback((pair, failure) -> {
+            if (failure != null)
+            {
+                notifyError(failure);
+                fail(failure.getMessage());
+            }
+            else
+            {
+                state.phase.repairCompleted();
+                CoordinatedRepairResult result = pair.left;
+                maybeStoreParentRepairSuccess(result.successfulRanges);
+                if (result.hasFailed())
+                {
+                    fail(null);
+                }
+                else
+                {
+                    success(pair.right.get());
+                    ctx.repair().cleanUp(state.id, neighborsAndRanges.participants);
+                }
+            }
+        });
     }
 
-    private List<ColumnFamilyStore> getColumnFamilies() throws IOException
+    private List<ColumnFamilyStore> getColumnFamilies()
     {
         String[] columnFamilies = state.options.getColumnFamilies().toArray(new String[state.options.getColumnFamilies().size()]);
-        Iterable<ColumnFamilyStore> validColumnFamilies = storageService.getValidColumnFamilies(false, false, state.keyspace, columnFamilies);
+        Iterable<ColumnFamilyStore> validColumnFamilies = this.validColumnFamilies.apply(state.keyspace, columnFamilies);
 
         if (Iterables.isEmpty(validColumnFamilies))
             throw new SkipRepairException(String.format("%s Empty keyspace, skipping repair: %s", state.id, state.keyspace));
@@ -327,11 +357,11 @@ public class RepairRunnable implements Runnable, ProgressEventNotifier, RepairNo
 
         //pre-calculate output of getLocalReplicas and pass it to getNeighbors to increase performance and prevent
         //calculation multiple times
-        Iterable<Range<Token>> keyspaceLocalRanges = storageService.getLocalReplicas(state.keyspace).ranges();
+        Iterable<Range<Token>> keyspaceLocalRanges = getLocalReplicas.apply(state.keyspace).ranges();
 
         for (Range<Token> range : state.options.getRanges())
         {
-            EndpointsForRange neighbors = ActiveRepairService.getNeighbors(state.keyspace, keyspaceLocalRanges, range,
+            EndpointsForRange neighbors = ctx.repair().getNeighbors(state.keyspace, keyspaceLocalRanges, range,
                                                                            state.options.getDataCenters(),
                                                                            state.options.getHosts());
             if (neighbors.isEmpty())
@@ -361,7 +391,7 @@ public class RepairRunnable implements Runnable, ProgressEventNotifier, RepairNo
 
         if (shouldExcludeDeadParticipants)
         {
-            Set<InetAddressAndPort> actualNeighbors = Sets.newHashSet(Iterables.filter(allNeighbors, FailureDetector.instance::isAlive));
+            Set<InetAddressAndPort> actualNeighbors = Sets.newHashSet(Iterables.filter(allNeighbors, ctx.failureDetector()::isAlive));
             shouldExcludeDeadParticipants = !allNeighbors.equals(actualNeighbors);
             allNeighbors = actualNeighbors;
         }
@@ -392,68 +422,46 @@ public class RepairRunnable implements Runnable, ProgressEventNotifier, RepairNo
         }
     }
 
-    private void prepare(List<ColumnFamilyStore> columnFamilies, Set<InetAddressAndPort> allNeighbors, boolean force)
+    private Future<?> prepare(List<ColumnFamilyStore> columnFamilies, Set<InetAddressAndPort> allNeighbors, boolean force)
     {
         state.phase.prepareStart();
-        try (Timer.Context ignore = Keyspace.open(state.keyspace).metric.repairPrepareTime.time())
-        {
-            ActiveRepairService.instance.prepareForRepair(state.id, FBUtilities.getBroadcastAddressAndPort(), allNeighbors, state.options, force, columnFamilies);
-        }
-        state.phase.prepareComplete();
+        Timer timer = Keyspace.open(state.keyspace).metric.repairPrepareTime;
+        long startNanos = ctx.clock().nanoTime();
+        return ctx.repair().prepareForRepair(state.id, ctx.broadcastAddressAndPort(), allNeighbors, state.options, force, columnFamilies)
+                  .map(ignore -> {
+                      timer.update(ctx.clock().nanoTime() - startNanos, TimeUnit.NANOSECONDS);
+                      state.phase.prepareComplete();
+                      return null;
+                  });
     }
 
-    private void repair(String[] cfnames, NeighborsAndRanges neighborsAndRanges)
+    private Future<Pair<CoordinatedRepairResult, Supplier<String>>> repair(String[] cfnames, NeighborsAndRanges neighborsAndRanges)
     {
         RepairTask task;
         if (state.options.isPreview())
         {
-            task = new PreviewRepairTask(state.options, state.keyspace, this, state.id, neighborsAndRanges.filterCommonRanges(state.keyspace, cfnames), cfnames);
+            task = new PreviewRepairTask(this, state.id, neighborsAndRanges.filterCommonRanges(state.keyspace, cfnames), cfnames);
         }
         else if (state.options.isIncremental())
         {
-            task = new IncrementalRepairTask(state.options, state.keyspace, this, state.id, neighborsAndRanges, cfnames);
+            task = new IncrementalRepairTask(this, state.id, neighborsAndRanges, cfnames);
         }
         else
         {
-            task = new NormalRepairTask(state.options, state.keyspace, this, state.id, neighborsAndRanges.filterCommonRanges(state.keyspace, cfnames), cfnames);
+            task = new NormalRepairTask(this, state.id, neighborsAndRanges.filterCommonRanges(state.keyspace, cfnames), cfnames);
         }
 
         ExecutorPlus executor = createExecutor();
         state.phase.repairSubmitted();
-        Future<CoordinatedRepairResult> f = task.perform(executor);
-        f.addCallback((result, failure) -> {
-            state.phase.repairCompleted();
-            try
-            {
-                if (failure != null)
-                {
-                    notifyError(failure);
-                    fail(failure.getMessage());
-                }
-                else
-                {
-                    maybeStoreParentRepairSuccess(result.successfulRanges);
-                    if (result.hasFailed())
-                    {
-                        fail(null);
-                    }
-                    else
-                    {
-                        success(task.successMessage());
-                        ActiveRepairService.instance.cleanUp(state.id, neighborsAndRanges.participants);
-                    }
-                }
-            }
-            finally
-            {
-                executor.shutdown();
-            }
-        });
+        return task.perform(executor)
+                   // after adding the callback java could no longer infer the type...
+                   .<Pair<CoordinatedRepairResult, Supplier<String>>>map(r -> Pair.create(r, task::successMessage))
+                   .addCallback((s, f) -> executor.shutdown());
     }
 
     private ExecutorPlus createExecutor()
     {
-        return executorFactory()
+        return ctx.executorFactory()
                 .localAware()
                 .withJmxInternal()
                 .pooled("Repair#" + state.cmd, state.options.getJobThreads());
@@ -480,7 +488,7 @@ public class RepairRunnable implements Runnable, ProgressEventNotifier, RepairNo
 
     private Thread createQueryThread(final TimeUUID sessionId)
     {
-        return executorFactory().startThread("Repair-Runnable-" + THREAD_COUNTER.incrementAndGet(), new WrappedRunnable()
+        return ctx.executorFactory().startThread("Repair-Runnable-" + THREAD_COUNTER.incrementAndGet(), new WrappedRunnable()
         {
             // Query events within a time interval that overlaps the last by one second. Ignore duplicates. Ignore local traces.
             // Wake up upon local trace activity. Query when notified of trace activity with a timeout that doubles every two timeouts.
@@ -495,13 +503,13 @@ public class RepairRunnable implements Runnable, ProgressEventNotifier, RepairNo
                 SelectStatement statement = (SelectStatement) QueryProcessor.parseStatement(query).prepare(ClientState.forInternalCalls());
 
                 ByteBuffer sessionIdBytes = sessionId.toBytes();
-                InetAddressAndPort source = FBUtilities.getBroadcastAddressAndPort();
+                InetAddressAndPort source = ctx.broadcastAddressAndPort();
 
                 HashSet<UUID>[] seen = new HashSet[]{ new HashSet<>(), new HashSet<>() };
                 int si = 0;
                 UUID uuid;
 
-                long tlast = currentTimeMillis(), tcur;
+                long tlast = ctx.clock().currentTimeMillis(), tcur;
 
                 TraceState.Status status;
                 long minWaitMillis = 125;
@@ -522,11 +530,11 @@ public class RepairRunnable implements Runnable, ProgressEventNotifier, RepairNo
                         shouldDouble = false;
                     }
                     ByteBuffer tminBytes = TimeUUID.minAtUnixMillis(tlast - 1000).toBytes();
-                    ByteBuffer tmaxBytes = TimeUUID.maxAtUnixMillis(tcur = currentTimeMillis()).toBytes();
+                    ByteBuffer tmaxBytes = TimeUUID.maxAtUnixMillis(tcur = ctx.clock().currentTimeMillis()).toBytes();
                     QueryOptions options = QueryOptions.forInternalCalls(ConsistencyLevel.ONE, Lists.newArrayList(sessionIdBytes,
                                                                                                                   tminBytes,
                                                                                                                   tmaxBytes));
-                    ResultMessage.Rows rows = statement.execute(forInternalCalls(), options, nanoTime());
+                    ResultMessage.Rows rows = statement.execute(forInternalCalls(), options, ctx.clock().nanoTime());
                     UntypedResultSet result = UntypedResultSet.create(rows.result);
 
                     for (UntypedResultSet.Row r : result)

src/java/org/apache/cassandra/repair/RepairJob.java

@@ -19,16 +19,19 @@ package org.apache.cassandra.repair;
 
 import java.util.*;
 import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.Executor;
 import java.util.function.Predicate;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 
 
+import javax.annotation.Nullable;
+
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.util.concurrent.*;
-import org.apache.cassandra.concurrent.ExecutorPlus;
+
 import org.apache.cassandra.schema.Schema;
 import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.repair.state.JobState;
@@ -45,7 +48,6 @@ import org.apache.cassandra.repair.asymmetric.PreferedNodeFilter;
 import org.apache.cassandra.repair.asymmetric.ReduceHelper;
 import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.schema.SystemDistributedKeyspace;
-import org.apache.cassandra.service.ActiveRepairService;
 import org.apache.cassandra.streaming.PreviewKind;
 import org.apache.cassandra.dht.Range;
 import org.apache.cassandra.dht.Token;
@@ -60,7 +62,6 @@ import org.apache.cassandra.utils.concurrent.ImmediateFuture;
 
 import static org.apache.cassandra.config.DatabaseDescriptor.paxosRepairEnabled;
 import static org.apache.cassandra.service.paxos.Paxos.useV2;
-import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis;
 
 /**
  * RepairJob runs repair on given ColumnFamily.
@@ -69,11 +70,12 @@ public class RepairJob extends AsyncFuture<RepairResult> implements Runnable
 {
     private static final Logger logger = LoggerFactory.getLogger(RepairJob.class);
 
+    private final SharedContext ctx;
     public final JobState state;
     private final RepairJobDesc desc;
     private final RepairSession session;
     private final RepairParallelism parallelismDegree;
-    private final ExecutorPlus taskExecutor;
+    private final Executor taskExecutor;
 
     @VisibleForTesting
     final List<ValidationTask> validationTasks = new CopyOnWriteArrayList<>();
@@ -88,16 +90,17 @@ public class RepairJob extends AsyncFuture<RepairResult> implements Runnable
      */
     public RepairJob(RepairSession session, String columnFamily)
     {
+        this.ctx = session.ctx;
         this.session = session;
         this.taskExecutor = session.taskExecutor;
         this.parallelismDegree = session.parallelismDegree;
         this.desc = new RepairJobDesc(session.state.parentRepairSession, session.getId(), session.state.keyspace, columnFamily, session.state.commonRange.ranges);
-        this.state = new JobState(desc, session.state.commonRange.endpoints);
+        this.state = new JobState(ctx.clock(), desc, session.state.commonRange.endpoints);
     }
 
     public long getNowInSeconds()
     {
-        long nowInSeconds = FBUtilities.nowInSeconds();
+        long nowInSeconds = ctx.clock().nowInSeconds();
         if (session.previewKind == PreviewKind.REPAIRED)
         {
             return nowInSeconds + DatabaseDescriptor.getValidationPreviewPurgeHeadStartInSec();
@@ -121,7 +124,7 @@ public class RepairJob extends AsyncFuture<RepairResult> implements Runnable
         ColumnFamilyStore cfs = ks.getColumnFamilyStore(desc.columnFamily);
         cfs.metric.repairsStarted.inc();
         List<InetAddressAndPort> allEndpoints = new ArrayList<>(session.state.commonRange.endpoints);
-        allEndpoints.add(FBUtilities.getBroadcastAddressAndPort());
+        allEndpoints.add(ctx.broadcastAddressAndPort());
 
         Future<List<TreeResponse>> treeResponses;
         Future<Void> paxosRepair;
@@ -176,7 +179,7 @@ public class RepairJob extends AsyncFuture<RepairResult> implements Runnable
                     state.phase.snapshotsSubmitted();
                     for (InetAddressAndPort endpoint : allEndpoints)
                     {
-                        SnapshotTask snapshotTask = new SnapshotTask(desc, endpoint);
+                        SnapshotTask snapshotTask = new SnapshotTask(ctx, desc, endpoint);
                         snapshotTasks.add(snapshotTask);
                         taskExecutor.execute(snapshotTask);
                     }
@@ -231,9 +234,7 @@ public class RepairJob extends AsyncFuture<RepairResult> implements Runnable
             public void onFailure(Throwable t)
             {
                 state.phase.fail(t);
-                // Make sure all validation tasks have cleaned up the off-heap Merkle trees they might contain.
-                validationTasks.forEach(ValidationTask::abort);
-                syncTasks.forEach(SyncTask::abort);
+                abort(t);
 
                 if (!session.previewKind.isPreview())
                 {
@@ -248,6 +249,17 @@ public class RepairJob extends AsyncFuture<RepairResult> implements Runnable
         }, taskExecutor);
     }
 
+    public synchronized void abort(@Nullable Throwable reason)
+    {
+        if (reason == null)
+            reason = new RuntimeException("Abort");
+        // Make sure all validation tasks have cleaned up the off-heap Merkle trees they might contain.
+        for (ValidationTask v : validationTasks)
+            v.abort(reason);
+        for (SyncTask s : syncTasks)
+            s.abort(reason);
+    }
+
     private boolean isTransient(InetAddressAndPort ep)
     {
         return session.state.commonRange.transEndpoints.contains(ep);
@@ -255,10 +267,9 @@ public class RepairJob extends AsyncFuture<RepairResult> implements Runnable
 
     private Future<List<SyncStat>> standardSyncing(List<TreeResponse> trees)
     {
-        state.phase.streamSubmitted();
-        List<SyncTask> syncTasks = createStandardSyncTasks(desc,
+        List<SyncTask> syncTasks = createStandardSyncTasks(ctx, desc,
                                                            trees,
-                                                           FBUtilities.getLocalAddressAndPort(),
+                                                           ctx.broadcastAddressAndPort(),
                                                            this::isTransient,
                                                            session.isIncremental,
                                                            session.pullRepair,
@@ -266,7 +277,8 @@ public class RepairJob extends AsyncFuture<RepairResult> implements Runnable
         return executeTasks(syncTasks);
     }
 
-    static List<SyncTask> createStandardSyncTasks(RepairJobDesc desc,
+    static List<SyncTask> createStandardSyncTasks(SharedContext ctx,
+                                                  RepairJobDesc desc,
                                                   List<TreeResponse> trees,
                                                   InetAddressAndPort local,
                                                   Predicate<InetAddressAndPort> isTransient,
@@ -274,7 +286,7 @@ public class RepairJob extends AsyncFuture<RepairResult> implements Runnable
                                                   boolean pullRepair,
                                                   PreviewKind previewKind)
     {
-        long startedAt = currentTimeMillis();
+        long startedAt = ctx.clock().currentTimeMillis();
         List<SyncTask> syncTasks = new ArrayList<>();
         // We need to difference all trees one against another
         for (int i = 0; i < trees.size() - 1; ++i)
@@ -309,7 +321,7 @@ public class RepairJob extends AsyncFuture<RepairResult> implements Runnable
                     if (!requestRanges && !transferRanges)
                         continue;
 
-                    task = new LocalSyncTask(desc, self.endpoint, remote.endpoint, differences, isIncremental ? desc.parentSessionId : null,
+                    task = new LocalSyncTask(ctx, desc, self.endpoint, remote.endpoint, differences, isIncremental ? desc.parentSessionId : null,
                                              requestRanges, transferRanges, previewKind);
                 }
                 else if (isTransient.test(r1.endpoint) || isTransient.test(r2.endpoint))
@@ -317,11 +329,11 @@ public class RepairJob extends AsyncFuture<RepairResult> implements Runnable
                     // Stream only from transient replica
                     TreeResponse streamFrom = isTransient.test(r1.endpoint) ? r1 : r2;
                     TreeResponse streamTo = isTransient.test(r1.endpoint) ? r2 : r1;
-                    task = new AsymmetricRemoteSyncTask(desc, streamTo.endpoint, streamFrom.endpoint, differences, previewKind);
+                    task = new AsymmetricRemoteSyncTask(ctx, desc, streamTo.endpoint, streamFrom.endpoint, differences, previewKind);
                 }
                 else
                 {
-                    task = new SymmetricRemoteSyncTask(desc, r1.endpoint, r2.endpoint, differences, previewKind);
+                    task = new SymmetricRemoteSyncTask(ctx, desc, r1.endpoint, r2.endpoint, differences, previewKind);
                 }
                 syncTasks.add(task);
             }
@@ -329,14 +341,14 @@ public class RepairJob extends AsyncFuture<RepairResult> implements Runnable
         }
         trees.get(trees.size() - 1).trees.release();
         logger.info("Created {} sync tasks based on {} merkle tree responses for {} (took: {}ms)",
-                    syncTasks.size(), trees.size(), desc.parentSessionId, currentTimeMillis() - startedAt);
+                    syncTasks.size(), trees.size(), desc.parentSessionId, ctx.clock().currentTimeMillis() - startedAt);
         return syncTasks;
     }
 
     private Future<List<SyncStat>> optimisedSyncing(List<TreeResponse> trees)
     {
-        state.phase.streamSubmitted();
-        List<SyncTask> syncTasks = createOptimisedSyncingSyncTasks(desc,
+        List<SyncTask> syncTasks = createOptimisedSyncingSyncTasks(ctx,
+                                                                   desc,
                                                                    trees,
                                                                    FBUtilities.getLocalAddressAndPort(),
                                                                    this::isTransient,
@@ -352,9 +364,12 @@ public class RepairJob extends AsyncFuture<RepairResult> implements Runnable
     {
         try
         {
-            ActiveRepairService.instance.getParentRepairSession(desc.parentSessionId);
+            ctx.repair().getParentRepairSession(desc.parentSessionId);
             syncTasks.addAll(tasks);
 
+            if (!tasks.isEmpty())
+                state.phase.streamSubmitted();
+
             for (SyncTask task : tasks)
             {
                 if (!task.isLocal())
@@ -385,7 +400,8 @@ public class RepairJob extends AsyncFuture<RepairResult> implements Runnable
         }
     }
 
-    static List<SyncTask> createOptimisedSyncingSyncTasks(RepairJobDesc desc,
+    static List<SyncTask> createOptimisedSyncingSyncTasks(SharedContext ctx,
+                                                          RepairJobDesc desc,
                                                           List<TreeResponse> trees,
                                                           InetAddressAndPort local,
                                                           Predicate<InetAddressAndPort> isTransient,
@@ -393,7 +409,7 @@ public class RepairJob extends AsyncFuture<RepairResult> implements Runnable
                                                           boolean isIncremental,
                                                           PreviewKind previewKind)
     {
-        long startedAt = currentTimeMillis();
+        long startedAt = ctx.clock().currentTimeMillis();
         List<SyncTask> syncTasks = new ArrayList<>();
         // We need to difference all trees one against another
         DifferenceHolder diffHolder = new DifferenceHolder(trees);
@@ -427,12 +443,12 @@ public class RepairJob extends AsyncFuture<RepairResult> implements Runnable
                     SyncTask task;
                     if (address.equals(local))
                     {
-                        task = new LocalSyncTask(desc, address, fetchFrom, toFetch, isIncremental ? desc.parentSessionId : null,
+                        task = new LocalSyncTask(ctx, desc, address, fetchFrom, toFetch, isIncremental ? desc.parentSessionId : null,
                                                  true, false, previewKind);
                     }
                     else
                     {
-                        task = new AsymmetricRemoteSyncTask(desc, address, fetchFrom, toFetch, previewKind);
+                        task = new AsymmetricRemoteSyncTask(ctx, desc, address, fetchFrom, toFetch, previewKind);
                     }
                     syncTasks.add(task);
 
@@ -444,14 +460,14 @@ public class RepairJob extends AsyncFuture<RepairResult> implements Runnable
             }
         }
         logger.info("Created {} optimised sync tasks based on {} merkle tree responses for {} (took: {}ms)",
-                    syncTasks.size(), trees.size(), desc.parentSessionId, currentTimeMillis() - startedAt);
+                    syncTasks.size(), trees.size(), desc.parentSessionId, ctx.clock().currentTimeMillis() - startedAt);
         logger.trace("Optimised sync tasks for {}: {}", desc.parentSessionId, syncTasks);
         return syncTasks;
     }
 
     private String getDC(InetAddressAndPort address)
     {
-        return DatabaseDescriptor.getEndpointSnitch().getDatacenter(address);
+        return ctx.snitch().getDatacenter(address);
     }
 
     /**
@@ -582,7 +598,7 @@ public class RepairJob extends AsyncFuture<RepairResult> implements Runnable
 
     private ValidationTask newValidationTask(InetAddressAndPort endpoint, long nowInSec)
     {
-        ValidationTask task = new ValidationTask(desc, endpoint, nowInSec, session.previewKind);
+        ValidationTask task = new ValidationTask(session.ctx, desc, endpoint, nowInSec, session.previewKind);
         validationTasks.add(task);
         return task;
     }

src/java/org/apache/cassandra/repair/RepairJobDesc.java

@@ -21,10 +21,9 @@ import java.io.IOException;
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.Objects;
 import java.util.UUID;
 
-import com.google.common.base.Objects;
-
 import org.apache.commons.lang3.ArrayUtils;
 
 import org.apache.cassandra.db.TypeSizes;
@@ -96,11 +95,11 @@ public class RepairJobDesc
 
         RepairJobDesc that = (RepairJobDesc) o;
 
-        if (!columnFamily.equals(that.columnFamily)) return false;
-        if (!keyspace.equals(that.keyspace)) return false;
-        if (ranges != null ? that.ranges == null || (ranges.size() != that.ranges.size()) || (ranges.size() == that.ranges.size() && !ranges.containsAll(that.ranges)) : that.ranges != null) return false;
+        if (!Objects.equals(parentSessionId, that.parentSessionId)) return false;
         if (!sessionId.equals(that.sessionId)) return false;
-        if (parentSessionId != null ? !parentSessionId.equals(that.parentSessionId) : that.parentSessionId != null) return false;
+        if (!keyspace.equals(that.keyspace)) return false;
+        if (!columnFamily.equals(that.columnFamily)) return false;
+        if (ranges != null ? that.ranges == null || (ranges.size() != that.ranges.size()) || (ranges.size() == that.ranges.size() && !ranges.containsAll(that.ranges)) : that.ranges != null) return false;
 
         return true;
     }
@@ -108,7 +107,7 @@ public class RepairJobDesc
     @Override
     public int hashCode()
     {
-        return Objects.hashCode(sessionId, keyspace, columnFamily, ranges);
+        return Objects.hash(parentSessionId, sessionId, keyspace, columnFamily, ranges);
     }
 
     private static class RepairJobDescSerializer implements IVersionedSerializer<RepairJobDesc>

src/java/org/apache/cassandra/repair/RepairMessageVerbHandler.java

@@ -18,6 +18,8 @@
 package org.apache.cassandra.repair;
 
 import java.util.*;
+import java.util.function.BiFunction;
+import java.util.function.Function;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -26,17 +28,19 @@ import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.exceptions.RequestFailureReason;
 import org.apache.cassandra.net.IVerbHandler;
 import org.apache.cassandra.net.Message;
-import org.apache.cassandra.net.MessagingService;
 import org.apache.cassandra.repair.messages.*;
+import org.apache.cassandra.repair.state.AbstractCompletable;
+import org.apache.cassandra.repair.state.AbstractState;
+import org.apache.cassandra.repair.state.Completable;
 import org.apache.cassandra.repair.state.ParticipateState;
+import org.apache.cassandra.repair.state.SyncState;
 import org.apache.cassandra.repair.state.ValidationState;
 import org.apache.cassandra.schema.TableId;
 import org.apache.cassandra.service.ActiveRepairService;
 import org.apache.cassandra.streaming.PreviewKind;
+import org.apache.cassandra.utils.JVMStabilityInspector;
 import org.apache.cassandra.utils.TimeUUID;
 
-import static org.apache.cassandra.net.Verb.VALIDATION_RSP;
-
 /**
  * Handles all repair related message.
  *
@@ -44,18 +48,38 @@ import static org.apache.cassandra.net.Verb.VALIDATION_RSP;
  */
 public class RepairMessageVerbHandler implements IVerbHandler<RepairMessage>
 {
-    public static RepairMessageVerbHandler instance = new RepairMessageVerbHandler();
+    private static class Holder
+    {
+        private static final RepairMessageVerbHandler instance = new RepairMessageVerbHandler();
+    }
+
+    public static RepairMessageVerbHandler instance()
+    {
+        return Holder.instance;
+    }
+
+    private final SharedContext ctx;
+
+    private RepairMessageVerbHandler()
+    {
+        this(SharedContext.Global.instance);
+    }
+
+    public RepairMessageVerbHandler(SharedContext ctx)
+    {
+        this.ctx = ctx;
+    }
 
     private static final Logger logger = LoggerFactory.getLogger(RepairMessageVerbHandler.class);
 
     private boolean isIncremental(TimeUUID sessionID)
     {
-        return ActiveRepairService.instance.consistent.local.isSessionInProgress(sessionID);
+        return ctx.repair().consistent.local.isSessionInProgress(sessionID);
     }
 
     private PreviewKind previewKind(TimeUUID sessionID) throws NoSuchRepairSessionException
     {
-        ActiveRepairService.ParentRepairSession prs = ActiveRepairService.instance.getParentRepairSession(sessionID);
+        ActiveRepairService.ParentRepairSession prs = ctx.repair().getParentRepairSession(sessionID);
         return prs != null ? prs.previewKind : PreviewKind.NONE;
     }
 
@@ -71,16 +95,17 @@ public class RepairMessageVerbHandler implements IVerbHandler<RepairMessage>
                 {
                     PrepareMessage prepareMessage = (PrepareMessage) message.payload;
                     logger.debug("Preparing, {}", prepareMessage);
-                    ParticipateState state = new ParticipateState(message.from(), prepareMessage);
-                    if (!ActiveRepairService.instance.register(state))
+                    ParticipateState state = new ParticipateState(ctx.clock(), message.from(), prepareMessage);
+                    if (!ctx.repair().register(state))
                     {
-                        logger.debug("Duplicate prepare message found for {}", state.id);
+                        replyDedup(ctx.repair().participate(state.id), message);
                         return;
                     }
-                    if (!ActiveRepairService.verifyCompactionsPendingThreshold(prepareMessage.parentRepairSession, prepareMessage.previewKind))
+                    if (!ctx.repair().verifyCompactionsPendingThreshold(prepareMessage.parentRepairSession, prepareMessage.previewKind))
                     {
                         // error is logged in verifyCompactionsPendingThreshold
                         state.phase.fail("Too many pending compactions");
+
                         sendFailureResponse(message);
                         return;
                     }
@@ -99,22 +124,23 @@ public class RepairMessageVerbHandler implements IVerbHandler<RepairMessage>
                         }
                         columnFamilyStores.add(columnFamilyStore);
                     }
-                    ActiveRepairService.instance.registerParentRepairSession(prepareMessage.parentRepairSession,
-                                                                             message.from(),
-                                                                             columnFamilyStores,
-                                                                             prepareMessage.ranges,
-                                                                             prepareMessage.isIncremental,
-                                                                             prepareMessage.repairedAt,
-                                                                             prepareMessage.isGlobal,
-                                                                             prepareMessage.previewKind);
-                    MessagingService.instance().send(message.emptyResponse(), message.from());
+                    state.phase.accept();
+                    ctx.repair().registerParentRepairSession(prepareMessage.parentRepairSession,
+                                                                    message.from(),
+                                                                    columnFamilyStores,
+                                                                    prepareMessage.ranges,
+                                                                    prepareMessage.isIncremental,
+                                                                    prepareMessage.repairedAt,
+                                                                    prepareMessage.isGlobal,
+                                                                    prepareMessage.previewKind);
+                    sendAck(message);
                 }
                     break;
 
                 case SNAPSHOT_MSG:
                 {
                     logger.debug("Snapshotting {}", desc);
-                    ParticipateState state = ActiveRepairService.instance.participate(desc.parentSessionId);
+                    ParticipateState state = ctx.repair().participate(desc.parentSessionId);
                     if (state == null)
                     {
                         logErrorAndSendFailureResponse("Unknown repair " + desc.parentSessionId, message);
@@ -130,56 +156,65 @@ public class RepairMessageVerbHandler implements IVerbHandler<RepairMessage>
                         return;
                     }
 
-                    ActiveRepairService.ParentRepairSession prs = ActiveRepairService.instance.getParentRepairSession(desc.parentSessionId);
-                    prs.setHasSnapshots();
-                    TableRepairManager repairManager = cfs.getRepairManager();
-                    if (prs.isGlobal)
+                    ActiveRepairService.ParentRepairSession prs = ctx.repair().getParentRepairSession(desc.parentSessionId);
+                    if (prs.setHasSnapshots())
                     {
-                        repairManager.snapshot(desc.parentSessionId.toString(), prs.getRanges(), false);
+                        state.getOrCreateJob(desc).snapshot();
+                        TableRepairManager repairManager = cfs.getRepairManager();
+                        if (prs.isGlobal)
+                        {
+                            repairManager.snapshot(desc.parentSessionId.toString(), prs.getRanges(), false);
+                        }
+                        else
+                        {
+                            repairManager.snapshot(desc.parentSessionId.toString(), desc.ranges, true);
+                        }
+                        logger.debug("Enqueuing response to snapshot request {} to {}", desc.sessionId, message.from());
                     }
-                    else
-                    {
-                        repairManager.snapshot(desc.parentSessionId.toString(), desc.ranges, true);
-                    }
-                    logger.debug("Enqueuing response to snapshot request {} to {}", desc.sessionId, message.from());
-                    MessagingService.instance().send(message.emptyResponse(), message.from());
+                    sendAck(message);
                 }
                     break;
 
                 case VALIDATION_REQ:
                 {
-                    // notify initiator that the message has been received, allowing this method to take as long as it needs to
-                    MessagingService.instance().send(message.emptyResponse(), message.from());
                     ValidationRequest validationRequest = (ValidationRequest) message.payload;
                     logger.debug("Validating {}", validationRequest);
 
-                    ParticipateState participate = ActiveRepairService.instance.participate(desc.parentSessionId);
+                    ParticipateState participate = ctx.repair().participate(desc.parentSessionId);
                     if (participate == null)
                     {
                         logErrorAndSendFailureResponse("Unknown repair " + desc.parentSessionId, message);
                         return;
                     }
 
-                    ValidationState vState = new ValidationState(desc, message.from());
-                    if (!participate.register(vState))
-                    {
-                        logger.debug("Duplicate validation message found for parent={}, validation={}", participate.id, vState.id);
+                    ValidationState vState = new ValidationState(ctx.clock(), desc, message.from());
+                    if (!register(message, participate, vState,
+                                  participate::register,
+                                  (d, i) -> participate.validation(d)))
                         return;
-                    }
                     try
                     {
                         // trigger read-only compaction
                         ColumnFamilyStore store = ColumnFamilyStore.getIfExists(desc.keyspace, desc.columnFamily);
                         if (store == null)
                         {
-                            logger.error("Table {}.{} was dropped during validation phase of repair {}",
-                                         desc.keyspace, desc.columnFamily, desc.parentSessionId);
-                            vState.phase.fail(String.format("Table %s.%s was dropped", desc.keyspace, desc.columnFamily));
-                            MessagingService.instance().send(Message.out(VALIDATION_RSP, new ValidationResponse(desc)), message.from());
+                            String msg = String.format("Table %s.%s was dropped during validation phase of repair %s", desc.keyspace, desc.columnFamily, desc.parentSessionId);
+                            vState.phase.fail(msg);
+                            logErrorAndSendFailureResponse(msg, message);
                             return;
                         }
 
-                        ActiveRepairService.instance.consistent.local.maybeSetRepairing(desc.parentSessionId);
+                        try
+                        {
+                            ctx.repair().consistent.local.maybeSetRepairing(desc.parentSessionId);
+                        }
+                        catch (Throwable t)
+                        {
+                            JVMStabilityInspector.inspectThrowable(t);
+                            vState.phase.fail(t.toString());
+                            logErrorAndSendFailureResponse(t.toString(), message);
+                            return;
+                        }
                         PreviewKind previewKind;
                         try
                         {
@@ -189,13 +224,15 @@ public class RepairMessageVerbHandler implements IVerbHandler<RepairMessage>
                         {
                             logger.warn("Parent repair session {} has been removed, failing repair", desc.parentSessionId);
                             vState.phase.fail(e);
-                            MessagingService.instance().send(Message.out(VALIDATION_RSP, new ValidationResponse(desc)), message.from());
+                            sendFailureResponse(message);
                             return;
                         }
+                        vState.phase.accept();
+                        sendAck(message);
 
-                        Validator validator = new Validator(vState, validationRequest.nowInSec,
+                        Validator validator = new Validator(ctx, vState, validationRequest.nowInSec,
                                                             isIncremental(desc.parentSessionId), previewKind);
-                        ValidationManager.instance.submitValidation(store, validator);
+                        ctx.validationManager().submitValidation(store, validator);
                     }
                     catch (Throwable t)
                     {
@@ -207,12 +244,23 @@ public class RepairMessageVerbHandler implements IVerbHandler<RepairMessage>
 
                 case SYNC_REQ:
                 {
-                    // notify initiator that the message has been received, allowing this method to take as long as it needs to
-                    MessagingService.instance().send(message.emptyResponse(), message.from());
                     // forwarded sync request
                     SyncRequest request = (SyncRequest) message.payload;
                     logger.debug("Syncing {}", request);
-                    StreamingRepairTask task = new StreamingRepairTask(desc,
+
+                    ParticipateState participate = ctx.repair().participate(desc.parentSessionId);
+                    if (participate == null)
+                    {
+                        logErrorAndSendFailureResponse("Unknown repair " + desc.parentSessionId, message);
+                        return;
+                    }
+                    SyncState state = new SyncState(ctx.clock(), desc, request.initiator, request.src, request.dst);
+                    if (!register(message, participate, state,
+                                  participate::register,
+                                  participate::sync))
+                        return;
+                    state.phase.accept();
+                    StreamingRepairTask task = new StreamingRepairTask(ctx, state, desc,
                                                                        request.initiator,
                                                                        request.src,
                                                                        request.dst,
@@ -221,6 +269,7 @@ public class RepairMessageVerbHandler implements IVerbHandler<RepairMessage>
                                                                        request.previewKind,
                                                                        request.asymmetric);
                     task.run();
+                    sendAck(message);
                 }
                     break;
 
@@ -228,50 +277,50 @@ public class RepairMessageVerbHandler implements IVerbHandler<RepairMessage>
                 {
                     logger.debug("cleaning up repair");
                     CleanupMessage cleanup = (CleanupMessage) message.payload;
-                    ParticipateState state = ActiveRepairService.instance.participate(cleanup.parentRepairSession);
+                    ParticipateState state = ctx.repair().participate(cleanup.parentRepairSession);
                     if (state != null)
                         state.phase.success("Cleanup message recieved");
-                    ActiveRepairService.instance.removeParentRepairSession(cleanup.parentRepairSession);
-                    MessagingService.instance().send(message.emptyResponse(), message.from());
+                    ctx.repair().removeParentRepairSession(cleanup.parentRepairSession);
+                    sendAck(message);
                 }
                     break;
 
                 case PREPARE_CONSISTENT_REQ:
-                    ActiveRepairService.instance.consistent.local.handlePrepareMessage(message.from(), (PrepareConsistentRequest) message.payload);
+                    ctx.repair().consistent.local.handlePrepareMessage(message.from(), (PrepareConsistentRequest) message.payload);
                     break;
 
                 case PREPARE_CONSISTENT_RSP:
-                    ActiveRepairService.instance.consistent.coordinated.handlePrepareResponse((PrepareConsistentResponse) message.payload);
+                    ctx.repair().consistent.coordinated.handlePrepareResponse((PrepareConsistentResponse) message.payload);
                     break;
 
                 case FINALIZE_PROPOSE_MSG:
-                    ActiveRepairService.instance.consistent.local.handleFinalizeProposeMessage(message.from(), (FinalizePropose) message.payload);
+                    ctx.repair().consistent.local.handleFinalizeProposeMessage(message.from(), (FinalizePropose) message.payload);
                     break;
 
                 case FINALIZE_PROMISE_MSG:
-                    ActiveRepairService.instance.consistent.coordinated.handleFinalizePromiseMessage((FinalizePromise) message.payload);
+                    ctx.repair().consistent.coordinated.handleFinalizePromiseMessage((FinalizePromise) message.payload);
                     break;
 
                 case FINALIZE_COMMIT_MSG:
-                    ActiveRepairService.instance.consistent.local.handleFinalizeCommitMessage(message.from(), (FinalizeCommit) message.payload);
+                    ctx.repair().consistent.local.handleFinalizeCommitMessage(message.from(), (FinalizeCommit) message.payload);
                     break;
 
                 case FAILED_SESSION_MSG:
                     FailSession failure = (FailSession) message.payload;
-                    ActiveRepairService.instance.consistent.coordinated.handleFailSessionMessage(failure);
-                    ActiveRepairService.instance.consistent.local.handleFailSessionMessage(message.from(), failure);
+                    ctx.repair().consistent.coordinated.handleFailSessionMessage(failure);
+                    ctx.repair().consistent.local.handleFailSessionMessage(message.from(), failure);
                     break;
 
                 case STATUS_REQ:
-                    ActiveRepairService.instance.consistent.local.handleStatusRequest(message.from(), (StatusRequest) message.payload);
+                    ctx.repair().consistent.local.handleStatusRequest(message.from(), (StatusRequest) message.payload);
                     break;
 
                 case STATUS_RSP:
-                    ActiveRepairService.instance.consistent.local.handleStatusResponse(message.from(), (StatusResponse) message.payload);
+                    ctx.repair().consistent.local.handleStatusResponse(message.from(), (StatusResponse) message.payload);
                     break;
 
                 default:
-                    ActiveRepairService.instance.handleMessage(message);
+                    ctx.repair().handleMessage(message);
                     break;
             }
         }
@@ -280,15 +329,79 @@ public class RepairMessageVerbHandler implements IVerbHandler<RepairMessage>
             logger.error("Got error, removing parent repair session");
             if (desc != null && desc.parentSessionId != null)
             {
-                ParticipateState parcipate = ActiveRepairService.instance.participate(desc.parentSessionId);
+                ParticipateState parcipate = ctx.repair().participate(desc.parentSessionId);
                 if (parcipate != null)
                     parcipate.phase.fail(e);
-                ActiveRepairService.instance.removeParentRepairSession(desc.parentSessionId);
+                ctx.repair().removeParentRepairSession(desc.parentSessionId);
             }
             throw new RuntimeException(e);
         }
     }
 
+    private <I, T extends AbstractState<?, I>> boolean register(Message<RepairMessage> message,
+                                                                ParticipateState participate,
+                                                                T vState,
+                                                                Function<T, ParticipateState.RegisterStatus> register,
+                                                                BiFunction<RepairJobDesc, I, T> getter)
+    {
+        ParticipateState.RegisterStatus registerStatus = register.apply(vState);
+        switch (registerStatus)
+        {
+            case ACCEPTED:
+                return true;
+            case EXISTS:
+                logger.debug("Duplicate validation message found for parent={}, validation={}", participate.id, vState.id);
+                replyDedup(getter.apply(message.payload.desc, vState.id), message);
+                return false;
+            case ALREADY_COMPLETED:
+            case STATUS_REJECTED:
+                // the repair is complete (most likely failed as we don't know success always), or is at a later phase such as sync
+                // so send a nack saying that the validation could not be accepted
+                sendFailureResponse(message);
+                return false;
+            default:
+                throw new IllegalStateException("Unexpected status: " + registerStatus);
+        }
+    }
+
+    private enum DedupResult { UNKNOWN, ACCEPT, REJECT }
+
+    private static DedupResult dedupResult(AbstractCompletable<?> state)
+    {
+        AbstractCompletable.Status status = state.getCompletionStatus();
+        switch (status)
+        {
+            case INIT:
+                return DedupResult.UNKNOWN;
+            case ACCEPTED:
+                return DedupResult.ACCEPT;
+            case COMPLETED:
+                return state.getResult().kind == Completable.Result.Kind.FAILURE ? DedupResult.REJECT: DedupResult.ACCEPT;
+            default:
+                throw new IllegalStateException("Unknown status: " + state);
+        }
+    }
+
+    private void replyDedup(AbstractCompletable<?> state, Message<RepairMessage> message)
+    {
+        if (state == null)
+            throw new IllegalStateException("State is null");
+        DedupResult result = dedupResult(state);
+        switch (result)
+        {
+            case ACCEPT:
+                sendAck(message);
+                break;
+            case REJECT:
+                sendFailureResponse(message);
+                break;
+            case UNKNOWN:
+                break;
+            default:
+                throw new IllegalStateException("Unknown result: " + result);
+        }
+    }
+
     private void logErrorAndSendFailureResponse(String errorMessage, Message<?> respondTo)
     {
         logger.error(errorMessage);
@@ -298,6 +411,11 @@ public class RepairMessageVerbHandler implements IVerbHandler<RepairMessage>
     private void sendFailureResponse(Message<?> respondTo)
     {
         Message<?> reply = respondTo.failureResponse(RequestFailureReason.UNKNOWN);
-        MessagingService.instance().send(reply, respondTo.from());
+        ctx.messaging().send(reply, respondTo.from());
+    }
+
+    private void sendAck(Message<RepairMessage> message)
+    {
+        ctx.messaging().send(message.emptyResponse(), message.from());
     }
 }

src/java/org/apache/cassandra/repair/RepairSession.java

@@ -27,20 +27,21 @@ import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.TimeUnit;
+import java.util.concurrent.Executor;
+import java.util.concurrent.RejectedExecutionException;
 import java.util.concurrent.atomic.AtomicBoolean;
 
+import javax.annotation.Nullable;
+
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.Lists;
 import com.google.common.util.concurrent.*;
-import org.apache.cassandra.concurrent.ExecutorFactory;
-import org.apache.cassandra.concurrent.ExecutorPlus;
-import org.apache.cassandra.utils.TimeUUID;
-import org.apache.cassandra.repair.state.SessionState;
-import org.apache.cassandra.utils.concurrent.AsyncFuture;
+
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import org.apache.cassandra.concurrent.ExecutorPlus;
+import org.apache.cassandra.concurrent.Stage;
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.Keyspace;
@@ -49,19 +50,23 @@ import org.apache.cassandra.dht.Token;
 import org.apache.cassandra.exceptions.RepairException;
 import org.apache.cassandra.gms.*;
 import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.net.Message;
 import org.apache.cassandra.repair.consistent.ConsistentSession;
 import org.apache.cassandra.repair.consistent.LocalSession;
 import org.apache.cassandra.repair.consistent.LocalSessions;
+import org.apache.cassandra.repair.messages.SyncResponse;
+import org.apache.cassandra.repair.messages.ValidationResponse;
+import org.apache.cassandra.repair.state.SessionState;
 import org.apache.cassandra.schema.SystemDistributedKeyspace;
 import org.apache.cassandra.schema.TableId;
 import org.apache.cassandra.streaming.PreviewKind;
-import org.apache.cassandra.streaming.SessionSummary;
 import org.apache.cassandra.tracing.Tracing;
 import org.apache.cassandra.utils.FBUtilities;
 import org.apache.cassandra.utils.MerkleTrees;
 import org.apache.cassandra.utils.Pair;
 import org.apache.cassandra.utils.Throwables;
-import org.apache.cassandra.utils.concurrent.Future;
+import org.apache.cassandra.utils.TimeUUID;
+import org.apache.cassandra.utils.concurrent.AsyncFuture;
 
 /**
  * Coordinates the (active) repair of a list of non overlapping token ranges.
@@ -125,8 +130,10 @@ public class RepairSession extends AsyncFuture<RepairSessionResult> implements I
     private final ConcurrentMap<Pair<RepairJobDesc, SyncNodePair>, CompletableRemoteSyncTask> syncingTasks = new ConcurrentHashMap<>();
 
     // Tasks(snapshot, validate request, differencing, ...) are run on taskExecutor
-    public final ExecutorPlus taskExecutor;
+    public final SafeExecutor taskExecutor;
     public final boolean optimiseStreams;
+    public final SharedContext ctx;
+    private volatile List<RepairJob> jobs = Collections.emptyList();
 
     private volatile boolean terminated = false;
 
@@ -141,7 +148,8 @@ public class RepairSession extends AsyncFuture<RepairSessionResult> implements I
      * @param paxosOnly true if we should only complete paxos operations, not run a normal repair
      * @param cfnames names of columnfamilies
      */
-    public RepairSession(TimeUUID parentRepairSession,
+    public RepairSession(SharedContext ctx,
+                         TimeUUID parentRepairSession,
                          CommonRange commonRange,
                          String keyspace,
                          RepairParallelism parallelismDegree,
@@ -153,21 +161,23 @@ public class RepairSession extends AsyncFuture<RepairSessionResult> implements I
                          boolean paxosOnly,
                          String... cfnames)
     {
+        this.ctx = ctx;
         this.repairPaxos = repairPaxos;
         this.paxosOnly = paxosOnly;
         assert cfnames.length > 0 : "Repairing no column families seems pointless, doesn't it";
-        this.state = new SessionState(parentRepairSession, keyspace, cfnames, commonRange);
+        this.state = new SessionState(ctx.clock(), parentRepairSession, keyspace, cfnames, commonRange);
         this.parallelismDegree = parallelismDegree;
         this.isIncremental = isIncremental;
         this.previewKind = previewKind;
         this.pullRepair = pullRepair;
         this.optimiseStreams = optimiseStreams;
-        this.taskExecutor = createExecutor();
+        this.taskExecutor = new SafeExecutor(createExecutor(ctx));
     }
 
-    protected ExecutorPlus createExecutor()
+    @VisibleForTesting
+    protected ExecutorPlus createExecutor(SharedContext ctx)
     {
-        return ExecutorFactory.Global.executorFactory().pooled("RepairJobTask", Integer.MAX_VALUE);
+        return ctx.executorFactory().pooled("RepairJobTask", Integer.MAX_VALUE);
     }
 
     public TimeUUID getId()
@@ -185,13 +195,20 @@ public class RepairSession extends AsyncFuture<RepairSessionResult> implements I
         return state.commonRange.endpoints;
     }
 
-    public void trackValidationCompletion(Pair<RepairJobDesc, InetAddressAndPort> key, ValidationTask task)
+    public synchronized void trackValidationCompletion(Pair<RepairJobDesc, InetAddressAndPort> key, ValidationTask task)
     {
+        if (terminated)
+        {
+            task.abort(new RuntimeException("Session terminated"));
+            return;
+        }
         validating.put(key, task);
     }
 
-    public void trackSyncCompletion(Pair<RepairJobDesc, SyncNodePair> key, CompletableRemoteSyncTask task)
+    public synchronized void trackSyncCompletion(Pair<RepairJobDesc, SyncNodePair> key, CompletableRemoteSyncTask task)
     {
+        if (terminated)
+            return;
         syncingTasks.put(key, task);
     }
 
@@ -199,26 +216,28 @@ public class RepairSession extends AsyncFuture<RepairSessionResult> implements I
      * Receive merkle tree response or failed response from {@code endpoint} for current repair job.
      *
      * @param desc repair job description
-     * @param endpoint endpoint that sent merkle tree
-     * @param trees calculated merkle trees, or null if validation failed
+     * @param message containing the merkle trees or an error
      */
-    public void validationComplete(RepairJobDesc desc, InetAddressAndPort endpoint, MerkleTrees trees)
+    public void validationComplete(RepairJobDesc desc, Message<ValidationResponse> message)
     {
+        InetAddressAndPort endpoint = message.from();
+        MerkleTrees trees = message.payload.trees;
         ValidationTask task = validating.remove(Pair.create(desc, endpoint));
+        // replies without a callback get dropped, so if in mixed mode this should be ignored
+        ctx.messaging().send(message.emptyResponse(), message.from());
         if (task == null)
         {
-            assert terminated : "The repair session should be terminated if the validation we're completing no longer exists.";
-            
             // The trees may be off-heap, and will therefore need to be released.
             if (trees != null)
                 trees.release();
-            
+
+            // either the session completed so the validation is no longer needed, or this is a retry; in both cases there is nothing to do
             return;
         }
 
-        String message = String.format("Received merkle tree for %s from %s", desc.columnFamily, endpoint);
-        logger.info("{} {}", previewKind.logPrefix(getId()), message);
-        Tracing.traceRepair(message);
+        String msg = String.format("Received merkle tree for %s from %s", desc.columnFamily, endpoint);
+        logger.info("{} {}", previewKind.logPrefix(getId()), msg);
+        Tracing.traceRepair(msg);
         task.treesReceived(trees);
     }
 
@@ -226,21 +245,20 @@ public class RepairSession extends AsyncFuture<RepairSessionResult> implements I
      * Notify this session that sync completed/failed with given {@code SyncNodePair}.
      *
      * @param desc synced repair job
-     * @param nodes nodes that completed sync
-     * @param success true if sync succeeded
+     * @param message nodes that completed sync and if they were successful
      */
-    public void syncComplete(RepairJobDesc desc, SyncNodePair nodes, boolean success, List<SessionSummary> summaries)
+    public void syncComplete(RepairJobDesc desc, Message<SyncResponse> message)
     {
+        SyncNodePair nodes = message.payload.nodes;
         CompletableRemoteSyncTask task = syncingTasks.remove(Pair.create(desc, nodes));
+        // replies without a callback get dropped, so if in mixed mode this should be ignored
+        ctx.messaging().send(message.emptyResponse(), message.from());
         if (task == null)
-        {
-            assert terminated : "The repair session should be terminated if the sync task we're completing no longer exists.";
             return;
-        }
 
         if (logger.isDebugEnabled())
             logger.debug("{} Repair completed between {} and {} on {}", previewKind.logPrefix(getId()), nodes.coordinator, nodes.peer, desc.columnFamily);
-        task.syncComplete(success, summaries);
+        task.syncComplete(message.payload.success, message.payload.summaries);
     }
 
     @VisibleForTesting
@@ -297,7 +315,7 @@ public class RepairSession extends AsyncFuture<RepairSessionResult> implements I
         // Checking all nodes are live
         for (InetAddressAndPort endpoint : state.commonRange.endpoints)
         {
-            if (!FailureDetector.instance.isAlive(endpoint) && !state.commonRange.hasSkippedReplicas)
+            if (!ctx.failureDetector().isAlive(endpoint) && !state.commonRange.hasSkippedReplicas)
             {
                 message = String.format("Cannot proceed on repair because a neighbor (%s) is dead: session failed", endpoint);
                 state.phase.fail(message);
@@ -314,7 +332,7 @@ public class RepairSession extends AsyncFuture<RepairSessionResult> implements I
 
         // Create and submit RepairJob for each ColumnFamily
         state.phase.jobsSubmitted();
-        List<Future<RepairResult>> jobs = new ArrayList<>(state.cfnames.length);
+        List<RepairJob> jobs = new ArrayList<>(state.cfnames.length);
         for (String cfname : state.cfnames)
         {
             RepairJob job = new RepairJob(this, cfname);
@@ -322,6 +340,7 @@ public class RepairSession extends AsyncFuture<RepairSessionResult> implements I
             executor.execute(job);
             jobs.add(job);
         }
+        this.jobs = jobs;
 
         // When all RepairJobs are done without error, cleanup and set the final result
         FBUtilities.allOf(jobs).addCallback(new FutureCallback<List<RepairResult>>()
@@ -334,10 +353,9 @@ public class RepairSession extends AsyncFuture<RepairSessionResult> implements I
                 Tracing.traceRepair("Completed sync of range {}", state.commonRange);
                 trySuccess(new RepairSessionResult(state.id, state.keyspace, state.commonRange.ranges, results, state.commonRange.hasSkippedReplicas));
 
-                taskExecutor.shutdown();
                 // mark this session as terminated
-                terminate();
-                awaitTaskExecutorTermination();
+                terminate(null);
+                taskExecutor.shutdown();
             }
 
             public void onFailure(Throwable t)
@@ -351,12 +369,19 @@ public class RepairSession extends AsyncFuture<RepairSessionResult> implements I
                 Tracing.traceRepair("Session completed with the following error: {}", t);
                 forceShutdown(t);
             }
-        }, executor);
+        }, taskExecutor);
     }
 
-    public void terminate()
+    public synchronized void terminate(@Nullable Throwable reason)
     {
         terminated = true;
+        List<RepairJob> jobs = this.jobs;
+        if (jobs != null)
+        {
+            for (RepairJob job : jobs)
+                job.abort(reason);
+        }
+        this.jobs = null;
         validating.clear();
         syncingTasks.clear();
     }
@@ -369,26 +394,9 @@ public class RepairSession extends AsyncFuture<RepairSessionResult> implements I
     public void forceShutdown(Throwable reason)
     {
         tryFailure(reason);
+        terminate(reason);
         taskExecutor.shutdown();
-        terminate();
-        awaitTaskExecutorTermination();
     }
-
-    private void awaitTaskExecutorTermination()
-    {
-        try
-        {
-            if (taskExecutor.awaitTermination(30, TimeUnit.SECONDS))
-                logger.debug("{} session task executor shut down gracefully", previewKind.logPrefix(getId()));
-            else
-                logger.warn("{} session task executor unable to shut down gracefully", previewKind.logPrefix(getId()));
-        }
-        catch (InterruptedException e)
-        {
-            Thread.currentThread().interrupt();
-        }
-    }
-
     public void onRemove(InetAddressAndPort endpoint)
     {
         convict(endpoint, Double.MAX_VALUE);
@@ -452,4 +460,33 @@ public class RepairSession extends AsyncFuture<RepairSessionResult> implements I
         }
         return false;
     }
+
+    private static class SafeExecutor implements Executor
+    {
+        private final ExecutorPlus delegate;
+
+        private SafeExecutor(ExecutorPlus delegate)
+        {
+            this.delegate = delegate;
+        }
+
+        @Override
+        public void execute(Runnable command)
+        {
+            try
+            {
+                delegate.execute(command);
+            }
+            catch (RejectedExecutionException e)
+            {
+                // task executor was shutdown, so fall back to a known good executor to finish callbacks
+                Stage.INTERNAL_RESPONSE.execute(command);
+            }
+        }
+
+        public void shutdown()
+        {
+            delegate.shutdown();
+        }
+    }
 }

src/java/org/apache/cassandra/repair/SharedContext.java

@@ -0,0 +1,165 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.repair;
+
+import java.util.Random;
+import java.util.function.Supplier;
+
+import org.apache.cassandra.concurrent.ExecutorFactory;
+import org.apache.cassandra.concurrent.ScheduledExecutorPlus;
+import org.apache.cassandra.concurrent.ScheduledExecutors;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.compaction.CompactionManager;
+import org.apache.cassandra.db.compaction.ICompactionManager;
+import org.apache.cassandra.gms.FailureDetector;
+import org.apache.cassandra.gms.Gossiper;
+import org.apache.cassandra.gms.IFailureDetector;
+import org.apache.cassandra.gms.IGossiper;
+import org.apache.cassandra.locator.IEndpointSnitch;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.net.MessageDelivery;
+import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.service.ActiveRepairService;
+import org.apache.cassandra.streaming.StreamPlan;
+import org.apache.cassandra.utils.Clock;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.MBeanWrapper;
+
+/**
+ * Access methods to shared resources and services.
+ * <p>
+ * In many parts of the code base we reach into the global space to pull out singletons, but this makes testing much harder; the main goals for this type is to make users easier to test.
+ *
+ * @see {@link Global#instance} for the main production path
+ */
+public interface SharedContext
+{
+    InetAddressAndPort broadcastAddressAndPort();
+    Supplier<Random> random();
+    Clock clock();
+    ExecutorFactory executorFactory();
+    MBeanWrapper mbean();
+    ScheduledExecutorPlus optionalTasks();
+
+    MessageDelivery messaging();
+    IFailureDetector failureDetector();
+    IEndpointSnitch snitch();
+    IGossiper gossiper();
+    ICompactionManager compactionManager();
+    ActiveRepairService repair();
+    IValidationManager validationManager();
+    TableRepairManager repairManager(ColumnFamilyStore store);
+    StreamExecutor streamExecutor();
+
+    class Global implements SharedContext
+    {
+        public static final Global instance = new Global();
+
+        @Override
+        public InetAddressAndPort broadcastAddressAndPort()
+        {
+            return FBUtilities.getBroadcastAddressAndPort();
+        }
+
+        @Override
+        public Supplier<Random> random()
+        {
+            return Random::new;
+        }
+
+        @Override
+        public Clock clock()
+        {
+            return Clock.Global.clock();
+        }
+
+        @Override
+        public ExecutorFactory executorFactory()
+        {
+            return ExecutorFactory.Global.executorFactory();
+        }
+
+        @Override
+        public MBeanWrapper mbean()
+        {
+            return MBeanWrapper.instance;
+        }
+
+        @Override
+        public ScheduledExecutorPlus optionalTasks()
+        {
+            return ScheduledExecutors.optionalTasks;
+        }
+
+        @Override
+        public MessageDelivery messaging()
+        {
+            return MessagingService.instance();
+        }
+
+        @Override
+        public IFailureDetector failureDetector()
+        {
+            return FailureDetector.instance;
+        }
+
+        @Override
+        public IEndpointSnitch snitch()
+        {
+            return DatabaseDescriptor.getEndpointSnitch();
+        }
+
+        @Override
+        public IGossiper gossiper()
+        {
+            return Gossiper.instance;
+        }
+
+        @Override
+        public ICompactionManager compactionManager()
+        {
+            return CompactionManager.instance;
+        }
+
+        @Override
+        public ActiveRepairService repair()
+        {
+            return ActiveRepairService.instance();
+        }
+
+        @Override
+        public IValidationManager validationManager()
+        {
+            return ValidationManager.instance;
+        }
+
+        @Override
+        public TableRepairManager repairManager(ColumnFamilyStore store)
+        {
+            return store.getRepairManager();
+        }
+
+        @Override
+        public StreamExecutor streamExecutor()
+        {
+            return StreamPlan::execute;
+        }
+    }
+}

src/java/org/apache/cassandra/repair/SnapshotTask.java

@@ -23,11 +23,12 @@ import org.apache.cassandra.exceptions.RequestFailureReason;
 import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.net.RequestCallback;
 import org.apache.cassandra.net.Message;
-import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.repair.messages.RepairMessage;
 import org.apache.cassandra.repair.messages.SnapshotMessage;
 import org.apache.cassandra.utils.concurrent.AsyncFuture;
 
 import static org.apache.cassandra.net.Verb.SNAPSHOT_MSG;
+import static org.apache.cassandra.repair.messages.RepairMessage.notDone;
 
 /**
  * SnapshotTask is a task that sends snapshot request.
@@ -36,18 +37,18 @@ public class SnapshotTask extends AsyncFuture<InetAddressAndPort> implements Run
 {
     private final RepairJobDesc desc;
     private final InetAddressAndPort endpoint;
+    private final SharedContext ctx;
 
-    SnapshotTask(RepairJobDesc desc, InetAddressAndPort endpoint)
+    SnapshotTask(SharedContext ctx, RepairJobDesc desc, InetAddressAndPort endpoint)
     {
+        this.ctx = ctx;
         this.desc = desc;
         this.endpoint = endpoint;
     }
 
     public void run()
     {
-        MessagingService.instance().sendWithCallback(Message.out(SNAPSHOT_MSG, new SnapshotMessage(desc)),
-                                                     endpoint,
-                                                     new SnapshotCallback(this));
+        RepairMessage.sendMessageWithRetries(ctx, notDone(this), new SnapshotMessage(desc), SNAPSHOT_MSG, endpoint, new SnapshotCallback(this));
     }
 
     /**

src/java/org/apache/cassandra/repair/StreamExecutor.java

@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.repair;
+
+import org.apache.cassandra.streaming.StreamPlan;
+import org.apache.cassandra.streaming.StreamResultFuture;
+
+public interface StreamExecutor
+{
+    StreamResultFuture execute(StreamPlan plan);
+}

src/java/org/apache/cassandra/repair/StreamingRepairTask.java

@@ -21,6 +21,7 @@ import java.util.Collections;
 import java.util.Collection;
 
 import com.google.common.annotations.VisibleForTesting;
+
 import org.apache.cassandra.locator.RangesAtEndpoint;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -28,9 +29,9 @@ import org.slf4j.LoggerFactory;
 import org.apache.cassandra.dht.Range;
 import org.apache.cassandra.dht.Token;
 import org.apache.cassandra.locator.InetAddressAndPort;
-import org.apache.cassandra.net.Message;
-import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.repair.messages.RepairMessage;
 import org.apache.cassandra.repair.messages.SyncResponse;
+import org.apache.cassandra.repair.state.SyncState;
 import org.apache.cassandra.streaming.PreviewKind;
 import org.apache.cassandra.streaming.StreamEvent;
 import org.apache.cassandra.streaming.StreamEventHandler;
@@ -53,6 +54,8 @@ public class StreamingRepairTask implements Runnable, StreamEventHandler
 {
     private static final Logger logger = LoggerFactory.getLogger(StreamingRepairTask.class);
 
+    private final SharedContext ctx;
+    private final SyncState state;
     private final RepairJobDesc desc;
     private final boolean asymmetric;
     private final InetAddressAndPort initiator;
@@ -62,8 +65,10 @@ public class StreamingRepairTask implements Runnable, StreamEventHandler
     private final TimeUUID pendingRepair;
     private final PreviewKind previewKind;
 
-    public StreamingRepairTask(RepairJobDesc desc, InetAddressAndPort initiator, InetAddressAndPort src, InetAddressAndPort dst, Collection<Range<Token>> ranges, TimeUUID pendingRepair, PreviewKind previewKind, boolean asymmetric)
+    public StreamingRepairTask(SharedContext ctx, SyncState state, RepairJobDesc desc, InetAddressAndPort initiator, InetAddressAndPort src, InetAddressAndPort dst, Collection<Range<Token>> ranges, TimeUUID pendingRepair, PreviewKind previewKind, boolean asymmetric)
     {
+        this.ctx = ctx;
+        this.state = state;
         this.desc = desc;
         this.initiator = initiator;
         this.src = src;
@@ -80,12 +85,14 @@ public class StreamingRepairTask implements Runnable, StreamEventHandler
         long start = approxTime.now();
         StreamPlan streamPlan = createStreamPlan(dst);
         logger.info("[streaming task #{}] Stream plan created in {}ms", desc.sessionId, MILLISECONDS.convert(approxTime.now() - start, NANOSECONDS));
-        streamPlan.execute();
+        state.phase.start();
+        ctx.streamExecutor().execute(streamPlan);
     }
 
     @VisibleForTesting
     StreamPlan createStreamPlan(InetAddressAndPort dest)
     {
+        state.phase.planning();
         StreamPlan sp = new StreamPlan(StreamOperation.REPAIR, 1, false, pendingRepair, previewKind)
                .listeners(this)
                .flushBeforeTransfer(pendingRepair == null) // sstables are isolated at the beginning of an incremental repair session, so flushing isn't neccessary
@@ -98,6 +105,7 @@ public class StreamingRepairTask implements Runnable, StreamEventHandler
         return sp;
     }
 
+    @Override
     public void handleStreamEvent(StreamEvent event)
     {
         // Nothing to do here, all we care about is the final success or failure and that's handled by
@@ -107,17 +115,21 @@ public class StreamingRepairTask implements Runnable, StreamEventHandler
     /**
      * If we succeeded on both stream in and out, respond back to coordinator
      */
+    @Override
     public void onSuccess(StreamState state)
     {
         logger.info("[repair #{}] streaming task succeed, returning response to {}", desc.sessionId, initiator);
-        MessagingService.instance().send(Message.out(SYNC_RSP, new SyncResponse(desc, src, dst, true, state.createSummaries())), initiator);
+        this.state.phase.success();
+        RepairMessage.sendMessageWithRetries(ctx, new SyncResponse(desc, src, dst, true, state.createSummaries()), SYNC_RSP, initiator);
     }
 
     /**
      * If we failed on either stream in or out, respond fail to coordinator
      */
+    @Override
     public void onFailure(Throwable t)
     {
-        MessagingService.instance().send(Message.out(SYNC_RSP, new SyncResponse(desc, src, dst, false, Collections.emptyList())), initiator);
+        this.state.phase.fail(t);
+        RepairMessage.sendMessageWithRetries(ctx, new SyncResponse(desc, src, dst, false, Collections.emptyList()), SYNC_RSP, initiator);
     }
 }

src/java/org/apache/cassandra/repair/SymmetricRemoteSyncTask.java

@@ -31,7 +31,6 @@ import org.apache.cassandra.repair.messages.SyncRequest;
 import org.apache.cassandra.streaming.PreviewKind;
 import org.apache.cassandra.streaming.SessionSummary;
 import org.apache.cassandra.tracing.Tracing;
-import org.apache.cassandra.utils.FBUtilities;
 
 /**
  * SymmetricRemoteSyncTask sends {@link SyncRequest} to remote(non-coordinator) node
@@ -43,15 +42,15 @@ public class SymmetricRemoteSyncTask extends SyncTask implements CompletableRemo
 {
     private static final Logger logger = LoggerFactory.getLogger(SymmetricRemoteSyncTask.class);
 
-    public SymmetricRemoteSyncTask(RepairJobDesc desc, InetAddressAndPort r1, InetAddressAndPort r2, List<Range<Token>> differences, PreviewKind previewKind)
+    public SymmetricRemoteSyncTask(SharedContext ctx, RepairJobDesc desc, InetAddressAndPort r1, InetAddressAndPort r2, List<Range<Token>> differences, PreviewKind previewKind)
     {
-        super(desc, r1, r2, differences, previewKind);
+        super(ctx, desc, r1, r2, differences, previewKind);
     }
 
     @Override
     protected void startSync()
     {
-        InetAddressAndPort local = FBUtilities.getBroadcastAddressAndPort();
+        InetAddressAndPort local = ctx.broadcastAddressAndPort();
         SyncRequest request = new SyncRequest(desc, local, nodePair.coordinator, nodePair.peer, rangesToSync, previewKind, false);
         Preconditions.checkArgument(nodePair.coordinator.equals(request.src));
         String message = String.format("Forwarding streaming repair of %d ranges to %s (to be streamed with %s)", request.ranges.size(), request.src, request.dst);

src/java/org/apache/cassandra/repair/SyncTask.java

@@ -37,13 +37,14 @@ import org.apache.cassandra.repair.messages.SyncRequest;
 import org.apache.cassandra.streaming.PreviewKind;
 import org.apache.cassandra.tracing.Tracing;
 
-import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis;
 import static org.apache.cassandra.net.Verb.SYNC_REQ;
+import static org.apache.cassandra.repair.messages.RepairMessage.notDone;
 
 public abstract class SyncTask extends AsyncFuture<SyncStat> implements Runnable
 {
     private static final Logger logger = LoggerFactory.getLogger(SyncTask.class);
 
+    protected final SharedContext ctx;
     protected final RepairJobDesc desc;
     @VisibleForTesting
     public final List<Range<Token>> rangesToSync;
@@ -53,9 +54,10 @@ public abstract class SyncTask extends AsyncFuture<SyncStat> implements Runnable
     protected volatile long startTime = Long.MIN_VALUE;
     protected final SyncStat stat;
 
-    protected SyncTask(RepairJobDesc desc, InetAddressAndPort primaryEndpoint, InetAddressAndPort peer, List<Range<Token>> rangesToSync, PreviewKind previewKind)
+    protected SyncTask(SharedContext ctx, RepairJobDesc desc, InetAddressAndPort primaryEndpoint, InetAddressAndPort peer, List<Range<Token>> rangesToSync, PreviewKind previewKind)
     {
         Preconditions.checkArgument(!peer.equals(primaryEndpoint), "Sending and receiving node are the same: %s", peer);
+        this.ctx = ctx;
         this.desc = desc;
         this.rangesToSync = rangesToSync;
         this.nodePair = new SyncNodePair(primaryEndpoint, peer);
@@ -75,8 +77,7 @@ public abstract class SyncTask extends AsyncFuture<SyncStat> implements Runnable
      */
     public final void run()
     {
-        startTime = currentTimeMillis();
-
+        startTime = ctx.clock().currentTimeMillis();
 
         // choose a repair method based on the significance of the difference
         String format = String.format("%s Endpoints %s and %s %%s for %s", previewKind.logPrefix(desc.sessionId), nodePair.coordinator, nodePair.peer, desc.columnFamily);
@@ -102,14 +103,17 @@ public abstract class SyncTask extends AsyncFuture<SyncStat> implements Runnable
     protected void finished()
     {
         if (startTime != Long.MIN_VALUE)
-            Keyspace.open(desc.keyspace).getColumnFamilyStore(desc.columnFamily).metric.repairSyncTime.update(currentTimeMillis() - startTime, TimeUnit.MILLISECONDS);
+            Keyspace.open(desc.keyspace).getColumnFamilyStore(desc.columnFamily).metric.repairSyncTime.update(ctx.clock().currentTimeMillis() - startTime, TimeUnit.MILLISECONDS);
     }
 
-    public void abort() {}
+    public void abort(Throwable reason)
+    {
+        tryFailure(reason);
+    }
 
     void sendRequest(SyncRequest request, InetAddressAndPort to)
     {
-        RepairMessage.sendMessageWithFailureCB(request,
+        RepairMessage.sendMessageWithFailureCB(ctx, notDone(this), request,
                                                SYNC_REQ,
                                                to,
                                                this::tryFailure);

src/java/org/apache/cassandra/repair/ValidationManager.java

@@ -37,13 +37,12 @@ import org.apache.cassandra.dht.Token;
 import org.apache.cassandra.metrics.TableMetrics;
 import org.apache.cassandra.metrics.TopPartitionTracker;
 import org.apache.cassandra.repair.state.ValidationState;
+import org.apache.cassandra.utils.Clock;
 import org.apache.cassandra.utils.FBUtilities;
 import org.apache.cassandra.utils.MerkleTree;
 import org.apache.cassandra.utils.MerkleTrees;
 
-import static org.apache.cassandra.utils.Clock.Global.nanoTime;
-
-public class ValidationManager
+public class ValidationManager implements IValidationManager
 {
     private static final Logger logger = LoggerFactory.getLogger(ValidationManager.class);
 
@@ -99,8 +98,10 @@ public class ValidationManager
      * but without writing the merge result
      */
     @SuppressWarnings("resource")
-    private void doValidation(ColumnFamilyStore cfs, Validator validator) throws IOException, NoSuchRepairSessionException
+    public static void doValidation(ColumnFamilyStore cfs, Validator validator) throws IOException, NoSuchRepairSessionException
     {
+        SharedContext ctx = validator.ctx;
+        Clock clock = ctx.clock();
         // this isn't meant to be race-proof, because it's not -- it won't cause bugs for a CFS to be dropped
         // mid-validation, or to attempt to validate a droped CFS.  this is just a best effort to avoid useless work,
         // particularly in the scenario where a validation is submitted before the drop, and there are compactions
@@ -119,8 +120,8 @@ public class ValidationManager
 
         // Create Merkle trees suitable to hold estimated partitions for the given ranges.
         // We blindly assume that a partition is evenly distributed on all sstables for now.
-        long start = nanoTime();
-        try (ValidationPartitionIterator vi = getValidationIterator(cfs.getRepairManager(), validator, topPartitionCollector))
+        long start = clock.nanoTime();
+        try (ValidationPartitionIterator vi = getValidationIterator(ctx.repairManager(cfs), validator, topPartitionCollector))
         {
             state.phase.start(vi.estimatedPartitions(), vi.getEstimatedBytes());
             MerkleTrees trees = createMerkleTrees(vi, validator.desc.ranges, cfs);
@@ -148,7 +149,7 @@ public class ValidationManager
         }
         if (logger.isDebugEnabled())
         {
-            long duration = TimeUnit.NANOSECONDS.toMillis(nanoTime() - start);
+            long duration = TimeUnit.NANOSECONDS.toMillis(clock.nanoTime() - start);
             logger.debug("Validation of {} partitions (~{}) finished in {} msec, for {}",
                          state.partitionsProcessed,
                          FBUtilities.prettyPrintMemory(state.estimatedTotalBytes),
@@ -177,6 +178,7 @@ public class ValidationManager
     /**
      * Does not mutate data, so is not scheduled.
      */
+    @Override
     public Future<?> submitValidation(ColumnFamilyStore cfs, Validator validator)
     {
         Callable<Object> validation = new Callable<Object>()

src/java/org/apache/cassandra/repair/ValidationTask.java

@@ -28,6 +28,7 @@ import org.apache.cassandra.utils.MerkleTrees;
 import org.apache.cassandra.utils.concurrent.AsyncFuture;
 
 import static org.apache.cassandra.net.Verb.VALIDATION_REQ;
+import static org.apache.cassandra.repair.messages.RepairMessage.notDone;
 
 /**
  * ValidationTask sends {@link ValidationRequest} to a replica.
@@ -39,11 +40,11 @@ public class ValidationTask extends AsyncFuture<TreeResponse> implements Runnabl
     private final InetAddressAndPort endpoint;
     private final long nowInSec;
     private final PreviewKind previewKind;
-    
-    private boolean active = true;
+    private final SharedContext ctx;
 
-    public ValidationTask(RepairJobDesc desc, InetAddressAndPort endpoint, long nowInSec, PreviewKind previewKind)
+    public ValidationTask(SharedContext ctx, RepairJobDesc desc, InetAddressAndPort endpoint, long nowInSec, PreviewKind previewKind)
     {
+        this.ctx = ctx;
         this.desc = desc;
         this.endpoint = endpoint;
         this.nowInSec = nowInSec;
@@ -55,7 +56,8 @@ public class ValidationTask extends AsyncFuture<TreeResponse> implements Runnabl
      */
     public void run()
     {
-        RepairMessage.sendMessageWithFailureCB(new ValidationRequest(desc, nowInSec),
+        RepairMessage.sendMessageWithFailureCB(ctx, notDone(this),
+                                               new ValidationRequest(desc, nowInSec),
                                                VALIDATION_REQ,
                                                endpoint,
                                                this::tryFailure);
@@ -70,18 +72,12 @@ public class ValidationTask extends AsyncFuture<TreeResponse> implements Runnabl
     {
         if (trees == null)
         {
-            active = false;
             tryFailure(RepairException.warn(desc, previewKind, "Validation failed in " + endpoint));
         }
-        else if (active)
+        else if (!trySuccess(new TreeResponse(endpoint, trees)))
         {
-            trySuccess(new TreeResponse(endpoint, trees));
-        }
-        else
-        {
-            // If the task has already been aborted, just release the possibly off-heap trees and move along.
+            // If the task is done, just release the possibly off-heap trees and move along.
             trees.release();
-            trySuccess(null);
         }
     }
 
@@ -89,37 +85,32 @@ public class ValidationTask extends AsyncFuture<TreeResponse> implements Runnabl
      * Release any trees already received by this task, and place it a state where any trees 
      * received subsequently will be properly discarded.
      */
-    public synchronized void abort()
+    public synchronized void abort(Throwable reason)
     {
-        if (active) 
+        if (!tryFailure(reason) && isSuccess())
         {
-            if (isDone())
+            try
             {
-                try 
-                {
-                    // If we're done, this should return immediately.
-                    TreeResponse response = get();
-                    
-                    if (response.trees != null)
-                        response.trees.release();
-                } 
-                catch (InterruptedException e) 
-                {
-                    // Restore the interrupt.
-                    Thread.currentThread().interrupt();
-                } 
-                catch (ExecutionException e) 
-                {
-                    // Do nothing here. If an exception was set, there were no trees to release.
-                }
+                // If we're done, this should return immediately.
+                TreeResponse response = get();
+
+                if (response.trees != null)
+                    response.trees.release();
+            }
+            catch (InterruptedException e)
+            {
+                // Restore the interrupt.
+                Thread.currentThread().interrupt();
+            }
+            catch (ExecutionException e)
+            {
+                // Do nothing here. If an exception was set, there were no trees to release.
             }
-            
-            active = false;
         }
     }
     
     public synchronized boolean isActive()
     {
-        return active;
+        return !isDone();
     }
 }

src/java/org/apache/cassandra/repair/Validator.java

@@ -22,6 +22,7 @@ import java.util.ArrayList;
 import java.util.List;
 import java.util.Random;
 
+import com.google.common.annotations.VisibleForTesting;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -37,10 +38,10 @@ import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.metrics.TopPartitionTracker;
 import org.apache.cassandra.net.Message;
 import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.repair.messages.RepairMessage;
 import org.apache.cassandra.repair.messages.ValidationResponse;
 import org.apache.cassandra.repair.state.ValidationState;
 import org.apache.cassandra.streaming.PreviewKind;
-import org.apache.cassandra.service.ActiveRepairService;
 import org.apache.cassandra.tracing.Tracing;
 import org.apache.cassandra.utils.FBUtilities;
 import org.apache.cassandra.utils.MerkleTree;
@@ -66,6 +67,7 @@ public class Validator implements Runnable
     public final long nowInSec;
     private final boolean evenTreeDistribution;
     public final boolean isIncremental;
+    public final SharedContext ctx;
 
     // null when all rows with the min token have been consumed
     private long validated;
@@ -83,16 +85,22 @@ public class Validator implements Runnable
 
     public Validator(ValidationState state, long nowInSec, PreviewKind previewKind)
     {
-        this(state, nowInSec, false, false, previewKind);
+        this(SharedContext.Global.instance, state, nowInSec, false, false, previewKind);
+    }
+
+    public Validator(SharedContext ctx, ValidationState state, long nowInSec, boolean isIncremental, PreviewKind previewKind)
+    {
+        this(ctx, state, nowInSec, false, isIncremental, previewKind);
     }
 
     public Validator(ValidationState state, long nowInSec, boolean isIncremental, PreviewKind previewKind)
     {
-        this(state, nowInSec, false, isIncremental, previewKind);
+        this(SharedContext.Global.instance, state, nowInSec, false, isIncremental, previewKind);
     }
 
-    public Validator(ValidationState state, long nowInSec, boolean evenTreeDistribution, boolean isIncremental, PreviewKind previewKind)
+    public Validator(SharedContext ctx, ValidationState state, long nowInSec, boolean evenTreeDistribution, boolean isIncremental, PreviewKind previewKind)
     {
+        this.ctx = ctx;
         this.state = state;
         this.desc = state.desc;
         this.initiator = state.initiator;
@@ -118,7 +126,7 @@ public class Validator implements Runnable
         else
         {
             List<DecoratedKey> keys = new ArrayList<>();
-            Random random = new Random();
+            Random random = ctx.random().get();
 
             for (Range<Token> range : trees.ranges())
             {
@@ -269,11 +277,12 @@ public class Validator implements Runnable
         return !FBUtilities.getBroadcastAddressAndPort().equals(initiator);
     }
 
-    private void respond(ValidationResponse response)
+    @VisibleForTesting
+    void respond(ValidationResponse response)
     {
         if (initiatorIsRemote())
         {
-            MessagingService.instance().send(Message.out(VALIDATION_RSP, response), initiator);
+            RepairMessage.sendMessageWithRetries(ctx, response, VALIDATION_RSP, initiator);
             return;
         }
 
@@ -294,7 +303,7 @@ public class Validator implements Runnable
             {
                 logger.error("Failed to move local merkle tree for {} off heap", desc, e);
             }
-            ActiveRepairService.instance.handleMessage(Message.out(VALIDATION_RSP, movedResponse));
+            ctx.repair().handleMessage(Message.out(VALIDATION_RSP, movedResponse));
         });
     }
 }

src/java/org/apache/cassandra/repair/consistent/ConsistentSession.java

@@ -44,6 +44,7 @@ import org.apache.cassandra.repair.messages.RepairOption;
 import org.apache.cassandra.repair.messages.StatusRequest;
 import org.apache.cassandra.repair.messages.StatusResponse;
 import org.apache.cassandra.repair.messages.ValidationRequest;
+import org.apache.cassandra.repair.SharedContext;
 import org.apache.cassandra.schema.TableId;
 import org.apache.cassandra.service.ActiveRepairService;
 import org.apache.cassandra.tools.nodetool.RepairAdmin;
@@ -187,6 +188,7 @@ public abstract class ConsistentSession
     }
 
     private volatile State state;
+    public final SharedContext ctx;
     public final TimeUUID sessionID;
     public final InetAddressAndPort coordinator;
     public final ImmutableSet<TableId> tableIds;
@@ -197,6 +199,7 @@ public abstract class ConsistentSession
     ConsistentSession(AbstractBuilder builder)
     {
         builder.validate();
+        this.ctx = builder.ctx;
         this.state = builder.state;
         this.sessionID = builder.sessionID;
         this.coordinator = builder.coordinator;
@@ -264,6 +267,7 @@ public abstract class ConsistentSession
 
     abstract static class AbstractBuilder
     {
+        private final SharedContext ctx;
         private State state;
         private TimeUUID sessionID;
         private InetAddressAndPort coordinator;
@@ -272,6 +276,11 @@ public abstract class ConsistentSession
         private Collection<Range<Token>> ranges;
         private Set<InetAddressAndPort> participants;
 
+        protected AbstractBuilder(SharedContext ctx)
+        {
+            this.ctx = ctx;
+        }
+
         void withState(State state)
         {
             this.state = state;

src/java/org/apache/cassandra/repair/consistent/CoordinatorSession.java

@@ -29,6 +29,7 @@ import com.google.common.base.Preconditions;
 import com.google.common.collect.Iterables;
 
 import org.apache.cassandra.concurrent.ImmediateExecutor;
+import org.apache.cassandra.repair.SharedContext;
 import org.apache.cassandra.repair.CoordinatedRepairResult;
 import org.apache.cassandra.utils.concurrent.AsyncPromise;
 import org.apache.cassandra.utils.concurrent.Future;
@@ -40,7 +41,6 @@ import org.slf4j.LoggerFactory;
 import org.apache.cassandra.exceptions.RepairException;
 import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.net.Message;
-import org.apache.cassandra.net.MessagingService;
 import org.apache.cassandra.net.Verb;
 import org.apache.cassandra.repair.SomeRepairFailedException;
 import org.apache.cassandra.repair.messages.FailSession;
@@ -51,8 +51,6 @@ import org.apache.cassandra.repair.messages.RepairMessage;
 import org.apache.cassandra.service.ActiveRepairService;
 import org.apache.cassandra.utils.concurrent.ImmediateFuture;
 
-import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis;
-
 /**
  * Coordinator side logic and state of a consistent repair session. Like {@link ActiveRepairService.ParentRepairSession},
  * there is only one {@code CoordinatorSession} per user repair command, regardless of the number of tables and token
@@ -62,6 +60,7 @@ public class CoordinatorSession extends ConsistentSession
 {
     private static final Logger logger = LoggerFactory.getLogger(CoordinatorSession.class);
 
+    private final SharedContext ctx;
     private final Map<InetAddressAndPort, State> participantStates = new HashMap<>();
     private final AsyncPromise<Void> prepareFuture = AsyncPromise.uncancellable();
     private final AsyncPromise<Void> finalizeProposeFuture = AsyncPromise.uncancellable();
@@ -73,6 +72,7 @@ public class CoordinatorSession extends ConsistentSession
     public CoordinatorSession(Builder builder)
     {
         super(builder);
+        ctx = builder.ctx == null ? SharedContext.Global.instance : builder.ctx;
         for (InetAddressAndPort participant : participants)
         {
             participantStates.put(participant, State.PREPARING);
@@ -81,6 +81,19 @@ public class CoordinatorSession extends ConsistentSession
 
     public static class Builder extends AbstractBuilder
     {
+        private SharedContext ctx;
+
+        public Builder(SharedContext ctx)
+        {
+            super(ctx);
+        }
+
+        public Builder withContext(SharedContext ctx)
+        {
+            this.ctx = ctx;
+            return this;
+        }
+
         public CoordinatorSession build()
         {
             validate();
@@ -88,9 +101,9 @@ public class CoordinatorSession extends ConsistentSession
         }
     }
 
-    public static Builder builder()
+    public static Builder builder(SharedContext ctx)
     {
-        return new Builder();
+        return new Builder(ctx);
     }
 
     public void setState(State state)
@@ -144,7 +157,8 @@ public class CoordinatorSession extends ConsistentSession
     protected void sendMessage(InetAddressAndPort destination, Message<RepairMessage> message)
     {
         logger.trace("Sending {} to {}", message.payload, destination);
-        MessagingService.instance().send(message, destination);
+
+        ctx.messaging().send(message, destination);
     }
 
     public Future<Void> prepare()
@@ -297,12 +311,12 @@ public class CoordinatorSession extends ConsistentSession
     {
         logger.info("Beginning coordination of incremental repair session {}", sessionID);
 
-        sessionStart = currentTimeMillis();
+        sessionStart = ctx.clock().currentTimeMillis();
         Future<Void> prepareResult = prepare();
 
         // run repair sessions normally
         Future<CoordinatedRepairResult> repairSessionResults = prepareResult.flatMap(ignore -> {
-            repairStart = currentTimeMillis();
+            repairStart = ctx.clock().currentTimeMillis();
             if (logger.isDebugEnabled())
                 logger.debug("Incremental repair {} prepare phase completed in {}", sessionID, formatDuration(sessionStart, repairStart));
             setRepairing();
@@ -311,7 +325,7 @@ public class CoordinatorSession extends ConsistentSession
 
         // if any session failed, then fail the future
         Future<CoordinatedRepairResult> onlySuccessSessionResults = repairSessionResults.flatMap(result -> {
-            finalizeStart = currentTimeMillis();
+            finalizeStart = ctx.clock().currentTimeMillis();
             if (result.hasFailed())
             {
                 if (logger.isDebugEnabled())
@@ -324,10 +338,10 @@ public class CoordinatorSession extends ConsistentSession
         // mark propose finalization and commit
         Future<CoordinatedRepairResult> proposeFuture = onlySuccessSessionResults.flatMap(results -> finalizePropose().map(ignore -> {
             if (logger.isDebugEnabled())
-                logger.debug("Incremental repair {} finalization phase completed in {}", sessionID, formatDuration(finalizeStart, currentTimeMillis()));
+                logger.debug("Incremental repair {} finalization phase completed in {}", sessionID, formatDuration(finalizeStart, ctx.clock().currentTimeMillis()));
             finalizeCommit();
             if (logger.isDebugEnabled())
-                logger.debug("Incremental repair {} phase completed in {}", sessionID, formatDuration(sessionStart, currentTimeMillis()));
+                logger.debug("Incremental repair {} phase completed in {}", sessionID, formatDuration(sessionStart, ctx.clock().currentTimeMillis()));
             return results;
         }));
 
@@ -335,7 +349,7 @@ public class CoordinatorSession extends ConsistentSession
             if (failure != null)
             {
                 if (logger.isDebugEnabled())
-                    logger.debug("Incremental repair {} phase failed in {}", sessionID, formatDuration(sessionStart, currentTimeMillis()));
+                    logger.debug("Incremental repair {} phase failed in {}", sessionID, formatDuration(sessionStart, ctx.clock().currentTimeMillis()));
                 fail();
             }
         }, ImmediateExecutor.INSTANCE);

src/java/org/apache/cassandra/repair/consistent/CoordinatorSessions.java

@@ -24,6 +24,7 @@ import java.util.Set;
 
 import com.google.common.base.Preconditions;
 
+import org.apache.cassandra.repair.SharedContext;
 import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.repair.messages.FailSession;
 import org.apache.cassandra.repair.messages.FinalizePromise;
@@ -37,8 +38,15 @@ import org.apache.cassandra.utils.TimeUUID;
  */
 public class CoordinatorSessions
 {
+    private final SharedContext ctx;
+    
     private final Map<TimeUUID, CoordinatorSession> sessions = new HashMap<>();
 
+    public CoordinatorSessions(SharedContext ctx)
+    {
+        this.ctx = ctx;
+    }
+
     protected CoordinatorSession buildSession(CoordinatorSession.Builder builder)
     {
         return new CoordinatorSession(builder);
@@ -46,14 +54,14 @@ public class CoordinatorSessions
 
     public synchronized CoordinatorSession registerSession(TimeUUID sessionId, Set<InetAddressAndPort> participants, boolean isForced) throws NoSuchRepairSessionException
     {
-        ActiveRepairService.ParentRepairSession prs = ActiveRepairService.instance.getParentRepairSession(sessionId);
+        ActiveRepairService.ParentRepairSession prs = ctx.repair().getParentRepairSession(sessionId);
 
         Preconditions.checkArgument(!sessions.containsKey(sessionId),
                                     "A coordinator already exists for session %s", sessionId);
         Preconditions.checkArgument(!isForced || prs.repairedAt == ActiveRepairService.UNREPAIRED_SSTABLE,
                                     "cannot promote data for forced incremental repairs");
 
-        CoordinatorSession.Builder builder = CoordinatorSession.builder();
+        CoordinatorSession.Builder builder = CoordinatorSession.builder(ctx);
         builder.withState(ConsistentSession.State.PREPARING);
         builder.withSessionID(sessionId);
         builder.withCoordinator(prs.coordinator);
@@ -62,6 +70,7 @@ public class CoordinatorSessions
         builder.withRepairedAt(prs.repairedAt);
         builder.withRanges(prs.getRanges());
         builder.withParticipants(participants);
+        builder.withContext(ctx);
         CoordinatorSession session = buildSession(builder);
         sessions.put(session.sessionID, session);
         return session;

src/java/org/apache/cassandra/repair/consistent/LocalSession.java

@@ -22,7 +22,7 @@ import java.util.Objects;
 
 import com.google.common.base.Preconditions;
 
-import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.repair.SharedContext;
 
 /**
  * Basically just a record of a local session. All of the local session logic is implemented in {@link LocalSessions}
@@ -57,7 +57,7 @@ public class LocalSession extends ConsistentSession
 
     public void setLastUpdate()
     {
-        lastUpdate = FBUtilities.nowInSeconds();
+        lastUpdate = ctx.clock().nowInSeconds();
     }
 
     public boolean equals(Object o)
@@ -97,6 +97,11 @@ public class LocalSession extends ConsistentSession
         private long startedAt;
         private long lastUpdate;
 
+        public Builder(SharedContext ctx)
+        {
+            super(ctx);
+        }
+
         public Builder withStartedAt(long startedAt)
         {
             this.startedAt = startedAt;
@@ -123,8 +128,8 @@ public class LocalSession extends ConsistentSession
         }
     }
 
-    public static Builder builder()
+    public static Builder builder(SharedContext ctx)
     {
-        return new Builder();
+        return new Builder(ctx);
     }
 }

src/java/org/apache/cassandra/repair/consistent/LocalSessions.java

@@ -74,11 +74,9 @@ import org.apache.cassandra.db.marshal.UUIDType;
 import org.apache.cassandra.dht.IPartitioner;
 import org.apache.cassandra.dht.Range;
 import org.apache.cassandra.dht.Token;
-import org.apache.cassandra.gms.FailureDetector;
 import org.apache.cassandra.io.util.DataInputBuffer;
 import org.apache.cassandra.io.util.DataOutputBuffer;
 import org.apache.cassandra.net.Message;
-import org.apache.cassandra.net.MessagingService;
 import org.apache.cassandra.repair.messages.FailSession;
 import org.apache.cassandra.repair.messages.FinalizeCommit;
 import org.apache.cassandra.repair.messages.FinalizePromise;
@@ -88,16 +86,15 @@ import org.apache.cassandra.repair.messages.PrepareConsistentResponse;
 import org.apache.cassandra.repair.messages.RepairMessage;
 import org.apache.cassandra.repair.messages.StatusRequest;
 import org.apache.cassandra.repair.messages.StatusResponse;
+import org.apache.cassandra.repair.SharedContext;
 import org.apache.cassandra.schema.TableId;
 import org.apache.cassandra.service.ActiveRepairService;
 import org.apache.cassandra.repair.NoSuchRepairSessionException;
 import org.apache.cassandra.service.StorageService;
-import org.apache.cassandra.utils.FBUtilities;
 import org.apache.cassandra.utils.Throwables;
 import org.apache.cassandra.utils.TimeUUID;
 import org.apache.cassandra.utils.concurrent.Future;
 
-import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory;
 import static org.apache.cassandra.config.CassandraRelevantProperties.REPAIR_CLEANUP_INTERVAL_SECONDS;
 import static org.apache.cassandra.config.CassandraRelevantProperties.REPAIR_DELETE_TIMEOUT_SECONDS;
 import static org.apache.cassandra.config.CassandraRelevantProperties.REPAIR_FAIL_TIMEOUT_SECONDS;
@@ -154,10 +151,16 @@ public class LocalSessions
 
     private final String keyspace = SchemaConstants.SYSTEM_KEYSPACE_NAME;
     private final String table = SystemKeyspace.REPAIRS;
+    private final SharedContext ctx;
     private boolean started = false;
     private volatile ImmutableMap<TimeUUID, LocalSession> sessions = ImmutableMap.of();
     private volatile ImmutableMap<TableId, RepairedState> repairedStates = ImmutableMap.of();
 
+    public LocalSessions(SharedContext ctx)
+    {
+        this.ctx = ctx;
+    }
+
     @VisibleForTesting
     int getNumSessions()
     {
@@ -167,13 +170,13 @@ public class LocalSessions
     @VisibleForTesting
     protected InetAddressAndPort getBroadcastAddressAndPort()
     {
-        return FBUtilities.getBroadcastAddressAndPort();
+        return ctx.broadcastAddressAndPort();
     }
 
     @VisibleForTesting
     protected boolean isAlive(InetAddressAndPort address)
     {
-        return FailureDetector.instance.isAlive(address);
+        return ctx.failureDetector().isAlive(address);
     }
 
     @VisibleForTesting
@@ -443,7 +446,7 @@ public class LocalSessions
         {
             synchronized (session)
             {
-                long now = FBUtilities.nowInSeconds();
+                long now = ctx.clock().nowInSeconds();
                 if (shouldFail(session, now))
                 {
                     logger.warn("Auto failing timed out repair session {}", session);
@@ -564,7 +567,7 @@ public class LocalSessions
 
     private LocalSession load(UntypedResultSet.Row row)
     {
-        LocalSession.Builder builder = LocalSession.builder();
+        LocalSession.Builder builder = LocalSession.builder(ctx);
         builder.withState(ConsistentSession.State.valueOf(row.getInt("state")));
         builder.withSessionID(row.getTimeUUID("parent_id"));
         InetAddressAndPort coordinator = InetAddressAndPort.getByAddressOverrideDefaults(
@@ -662,7 +665,7 @@ public class LocalSessions
     @VisibleForTesting
     LocalSession createSessionUnsafe(TimeUUID sessionId, ActiveRepairService.ParentRepairSession prs, Set<InetAddressAndPort> peers)
     {
-        LocalSession.Builder builder = LocalSession.builder();
+        LocalSession.Builder builder = LocalSession.builder(ctx);
         builder.withState(ConsistentSession.State.PREPARING);
         builder.withSessionID(sessionId);
         builder.withCoordinator(prs.coordinator);
@@ -672,7 +675,7 @@ public class LocalSessions
         builder.withRanges(prs.getRanges());
         builder.withParticipants(peers);
 
-        long now = FBUtilities.nowInSeconds();
+        long now = ctx.clock().nowInSeconds();
         builder.withStartedAt(now);
         builder.withLastUpdate(now);
 
@@ -681,13 +684,14 @@ public class LocalSessions
 
     protected ActiveRepairService.ParentRepairSession getParentRepairSession(TimeUUID sessionID) throws NoSuchRepairSessionException
     {
-        return ActiveRepairService.instance.getParentRepairSession(sessionID);
+
+        return ctx.repair().getParentRepairSession(sessionID);
     }
 
     protected void sendMessage(InetAddressAndPort destination, Message<? extends RepairMessage> message)
     {
         logger.trace("sending {} to {}", message.payload, destination);
-        MessagingService.instance().send(message, destination);
+        ctx.messaging().send(message, destination);
     }
 
     @VisibleForTesting
@@ -821,7 +825,7 @@ public class LocalSessions
         putSessionUnsafe(session);
         logger.info("Beginning local incremental repair session {}", session);
 
-        ExecutorService executor = executorFactory().pooled("Repair-" + sessionID, parentSession.getColumnFamilyStores().size());
+        ExecutorService executor = ctx.executorFactory().pooled("Repair-" + sessionID, parentSession.getColumnFamilyStores().size());
 
         KeyspaceRepairManager repairManager = parentSession.getKeyspace().getRepairManager();
         RangesAtEndpoint tokenRanges = filterLocalRanges(parentSession.getKeyspace().getName(), parentSession.getRanges());
@@ -1099,6 +1103,12 @@ public class LocalSessions
         listeners.remove(listener);
     }
 
+    @VisibleForTesting
+    public static void unsafeClearListeners()
+    {
+        listeners.clear();
+    }
+
     public interface Listener
     {
         void onIRStateChange(LocalSession session);

src/java/org/apache/cassandra/repair/messages/CleanupMessage.java

@@ -40,6 +40,12 @@ public class CleanupMessage extends RepairMessage
         this.parentRepairSession = parentRepairSession;
     }
 
+    @Override
+    public TimeUUID parentRepairSession()
+    {
+        return parentRepairSession;
+    }
+
     @Override
     public boolean equals(Object o)
     {

src/java/org/apache/cassandra/repair/messages/PrepareMessage.java

@@ -61,6 +61,12 @@ public class PrepareMessage extends RepairMessage
         this.previewKind = previewKind;
     }
 
+    @Override
+    public TimeUUID parentRepairSession()
+    {
+        return parentRepairSession;
+    }
+
     @Override
     public boolean equals(Object o)
     {

src/java/org/apache/cassandra/repair/messages/RepairMessage.java

@@ -17,21 +17,34 @@
  */
 package org.apache.cassandra.repair.messages;
 
+import java.util.Collections;
+import java.util.EnumMap;
+import java.util.EnumSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.Supplier;
+
+import javax.annotation.Nullable;
+
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.config.RepairRetrySpec;
+import org.apache.cassandra.config.RetrySpec;
+import org.apache.cassandra.repair.SharedContext;
 import org.apache.cassandra.exceptions.RepairException;
 import org.apache.cassandra.exceptions.RequestFailureReason;
-import org.apache.cassandra.gms.Gossiper;
 import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.net.Message;
-import org.apache.cassandra.net.MessagingService;
 import org.apache.cassandra.net.RequestCallback;
 import org.apache.cassandra.net.Verb;
 import org.apache.cassandra.repair.RepairJobDesc;
 import org.apache.cassandra.streaming.PreviewKind;
+import org.apache.cassandra.utils.Backoff;
 import org.apache.cassandra.utils.CassandraVersion;
 import org.apache.cassandra.utils.TimeUUID;
+import org.apache.cassandra.utils.concurrent.Future;
 
 import static org.apache.cassandra.net.MessageFlag.CALL_BACK_ON_FAILURE;
 
@@ -42,29 +55,121 @@ import static org.apache.cassandra.net.MessageFlag.CALL_BACK_ON_FAILURE;
  */
 public abstract class RepairMessage
 {
-    private static final CassandraVersion SUPPORTS_TIMEOUTS = new CassandraVersion("4.0.7-SNAPSHOT");
+    private enum ErrorHandling { NONE, TIMEOUT, RETRY }
+    private static final CassandraVersion SUPPORTS_RETRY = new CassandraVersion("5.0.0-alpha2.SNAPSHOT");
+    private static final Map<Verb, CassandraVersion> VERB_TIMEOUT_VERSIONS;
+
+    static
+    {
+        CassandraVersion timeoutVersion = new CassandraVersion("4.0.7-SNAPSHOT");
+        EnumMap<Verb, CassandraVersion> map = new EnumMap<>(Verb.class);
+        map.put(Verb.VALIDATION_REQ, timeoutVersion);
+        map.put(Verb.SYNC_REQ, timeoutVersion);
+        map.put(Verb.VALIDATION_RSP, SUPPORTS_RETRY);
+        map.put(Verb.SYNC_RSP, SUPPORTS_RETRY);
+        VERB_TIMEOUT_VERSIONS = Collections.unmodifiableMap(map);
+    }
+    private static final Set<Verb> SUPPORTS_RETRY_WITHOUT_VERSION_CHECK = Collections.unmodifiableSet(EnumSet.of(Verb.CLEANUP_MSG));
+
     private static final Logger logger = LoggerFactory.getLogger(RepairMessage.class);
+    @Nullable
     public final RepairJobDesc desc;
 
-    protected RepairMessage(RepairJobDesc desc)
+    protected RepairMessage(@Nullable RepairJobDesc desc)
     {
         this.desc = desc;
     }
 
+    public TimeUUID parentRepairSession()
+    {
+        return desc.parentSessionId;
+    }
+
     public interface RepairFailureCallback
     {
         void onFailure(Exception e);
     }
 
-    public static void sendMessageWithFailureCB(RepairMessage request, Verb verb, InetAddressAndPort endpoint, RepairFailureCallback failureCallback)
+    private static Backoff backoff(SharedContext ctx, Verb verb)
     {
-        RequestCallback<?> callback = new RequestCallback<Object>()
+        RepairRetrySpec retrySpec = DatabaseDescriptor.getRepairRetrySpec();
+        RetrySpec spec = verb == Verb.VALIDATION_RSP ? retrySpec.getMerkleTreeResponseSpec() : retrySpec;
+        if (!spec.isEnabled())
+            return Backoff.None.INSTANCE;
+        return new Backoff.ExponentialBackoff(spec.maxAttempts.value, spec.baseSleepTime.toMilliseconds(), spec.maxSleepTime.toMilliseconds(), ctx.random().get()::nextDouble);
+    }
+
+    public static Supplier<Boolean> notDone(Future<?> f)
+    {
+        return () -> !f.isDone();
+    }
+
+    private static Supplier<Boolean> always()
+    {
+        return () -> true;
+    }
+
+    public static <T> void sendMessageWithRetries(SharedContext ctx, Supplier<Boolean> allowRetry, RepairMessage request, Verb verb, InetAddressAndPort endpoint, RequestCallback<T> finalCallback)
+    {
+        sendMessageWithRetries(ctx, backoff(ctx, verb), allowRetry, request, verb, endpoint, finalCallback, 0);
+    }
+
+    public static <T> void sendMessageWithRetries(SharedContext ctx, RepairMessage request, Verb verb, InetAddressAndPort endpoint, RequestCallback<T> finalCallback)
+    {
+        sendMessageWithRetries(ctx, backoff(ctx, verb), always(), request, verb, endpoint, finalCallback, 0);
+    }
+
+    public static void sendMessageWithRetries(SharedContext ctx, RepairMessage request, Verb verb, InetAddressAndPort endpoint)
+    {
+        sendMessageWithRetries(ctx, backoff(ctx, verb), always(), request, verb, endpoint, new RequestCallback<>()
         {
             @Override
             public void onResponse(Message<Object> msg)
             {
-                logger.info("[#{}] {} received by {}", request.desc.parentSessionId, verb, endpoint);
-                // todo: at some point we should make repair messages follow the normal path, actually using this
+            }
+
+            @Override
+            public void onFailure(InetAddressAndPort from, RequestFailureReason failureReason)
+            {
+            }
+        }, 0);
+    }
+
+    private static <T> void sendMessageWithRetries(SharedContext ctx, Backoff backoff, Supplier<Boolean> allowRetry, RepairMessage request, Verb verb, InetAddressAndPort endpoint, RequestCallback<T> finalCallback, int attempt)
+    {
+        RequestCallback<T> callback = new RequestCallback<>()
+        {
+            @Override
+            public void onResponse(Message<T> msg)
+            {
+                finalCallback.onResponse(msg);
+            }
+
+            @Override
+            public void onFailure(InetAddressAndPort from, RequestFailureReason failureReason)
+            {
+                ErrorHandling allowed = errorHandlingSupported(ctx, endpoint, verb, request.parentRepairSession());
+                switch (allowed)
+                {
+                    case NONE:
+                        logger.error("[#{}] {} failed on {}: {}", request.parentRepairSession(), verb, from, failureReason);
+                        return;
+                    case TIMEOUT:
+                        finalCallback.onFailure(from, failureReason);
+                        return;
+                    case RETRY:
+                        int maxAttempts = backoff.maxAttempts();
+                        if (failureReason == RequestFailureReason.TIMEOUT && attempt < maxAttempts && allowRetry.get())
+                        {
+                            ctx.optionalTasks().schedule(() -> sendMessageWithRetries(ctx, backoff, allowRetry, request, verb, endpoint, finalCallback, attempt + 1),
+                                                         backoff.computeWaitTime(attempt), backoff.unit());
+                            return;
+                        }
+                        finalCallback.onFailure(from, failureReason);
+                        return;
+                    default:
+                        throw new AssertionError("Unknown error handler: " + allowed);
+                }
             }
 
             @Override
@@ -72,27 +177,59 @@ public abstract class RepairMessage
             {
                 return true;
             }
+        };
+        ctx.messaging().sendWithCallback(Message.outWithFlag(verb, request, CALL_BACK_ON_FAILURE),
+                                         endpoint,
+                                         callback);
+    }
 
+    public static void sendMessageWithFailureCB(SharedContext ctx, Supplier<Boolean> allowRetry, RepairMessage request, Verb verb, InetAddressAndPort endpoint, RepairFailureCallback failureCallback)
+    {
+        RequestCallback<?> callback = new RequestCallback<>()
+        {
+            @Override
+            public void onResponse(Message<Object> msg)
+            {
+                logger.info("[#{}] {} received by {}", request.parentRepairSession(), verb, endpoint);
+                // todo: at some point we should make repair messages follow the normal path, actually using this
+            }
+
+            @Override
             public void onFailure(InetAddressAndPort from, RequestFailureReason failureReason)
             {
-                logger.error("[#{}] {} failed on {}: {}", request.desc.parentSessionId, verb, from, failureReason);
+                failureCallback.onFailure(RepairException.error(request.desc, PreviewKind.NONE, String.format("Got %s failure from %s: %s", verb, from, failureReason)));
+            }
 
-                if (supportsTimeouts(from, request.desc.parentSessionId))
-                    failureCallback.onFailure(RepairException.error(request.desc, PreviewKind.NONE, String.format("Got %s failure from %s: %s", verb, from, failureReason)));
+            @Override
+            public boolean invokeOnFailure()
+            {
+                return true;
             }
         };
-
-        MessagingService.instance().sendWithCallback(Message.outWithFlag(verb, request, CALL_BACK_ON_FAILURE),
-                                                     endpoint,
-                                                     callback);
+        sendMessageWithRetries(ctx, allowRetry, request, verb, endpoint, callback);
     }
 
-    private static boolean supportsTimeouts(InetAddressAndPort from, TimeUUID parentSessionId)
+    private static ErrorHandling errorHandlingSupported(SharedContext ctx, InetAddressAndPort from, Verb verb, TimeUUID parentSessionId)
     {
-        CassandraVersion remoteVersion = Gossiper.instance.getReleaseVersion(from);
-        if (remoteVersion != null && remoteVersion.compareTo(SUPPORTS_TIMEOUTS) >= 0)
-            return true;
-        logger.warn("[#{}] Not failing repair due to remote host {} not supporting repair message timeouts (version = {})", parentSessionId, from, remoteVersion);
-        return false;
+        if (SUPPORTS_RETRY_WITHOUT_VERSION_CHECK.contains(verb))
+            return ErrorHandling.RETRY;
+        // Repair in mixed mode isn't fully supported, but also not activally blocked... so in the common case all participants
+        // will be on the same version as this instance, so can avoid the lookup from gossip
+        CassandraVersion remoteVersion = ctx.gossiper().getReleaseVersion(from);
+        if (remoteVersion == null)
+        {
+            if (VERB_TIMEOUT_VERSIONS.containsKey(verb))
+            {
+                logger.warn("[#{}] Not failing repair due to remote host {} not supporting repair message timeouts (version is unknown)", parentSessionId, from);
+                return ErrorHandling.NONE;
+            }
+            return ErrorHandling.TIMEOUT;
+        }
+        if (remoteVersion.compareTo(SUPPORTS_RETRY) >= 0)
+            return ErrorHandling.RETRY;
+        CassandraVersion timeoutVersion = VERB_TIMEOUT_VERSIONS.get(verb);
+        if (timeoutVersion == null || remoteVersion.compareTo(timeoutVersion) >= 0)
+            return ErrorHandling.TIMEOUT;
+        return ErrorHandling.NONE;
     }
 }

src/java/org/apache/cassandra/repair/state/AbstractCompletable.java

@@ -24,19 +24,35 @@ import com.google.common.base.Throwables;
 
 import org.apache.cassandra.utils.Clock;
 
-public class AbstractCompletable<I> implements Completable<I>
+public abstract class AbstractCompletable<I> implements Completable<I>
 {
-    private final long creationTimeMillis = Clock.Global.currentTimeMillis(); // used to convert from nanos to millis
-    private final long creationTimeNanos = Clock.Global.nanoTime();
+    public enum Status { INIT, ACCEPTED, COMPLETED }
+
+    private final long creationTimeMillis; // used to convert from nanos to millis
+    private final long creationTimeNanos;
+    protected final Clock clock;
     private final AtomicReference<Result> result = new AtomicReference<>(null);
     public final I id;
     protected volatile long lastUpdatedAtNs;
 
-    public AbstractCompletable(I id)
+    public AbstractCompletable(Clock clock, I id)
     {
+        this.creationTimeMillis = clock.currentTimeMillis();
+        this.creationTimeNanos = clock.nanoTime();
+        this.clock = clock;
         this.id = id;
     }
 
+    public abstract boolean isAccepted();
+
+    public Status getCompletionStatus()
+    {
+        Result result = getResult();
+        if (result != null)
+            return Status.COMPLETED;
+        return isAccepted() ? Status.ACCEPTED : Status.INIT;
+    }
+
     @Override
     public I getId()
     {
@@ -75,7 +91,7 @@ public class AbstractCompletable<I> implements Completable<I>
 
     public void updated()
     {
-        lastUpdatedAtNs = Clock.Global.nanoTime();
+        lastUpdatedAtNs = clock.nanoTime();
     }
 
     protected boolean tryResult(Result result)
@@ -83,7 +99,7 @@ public class AbstractCompletable<I> implements Completable<I>
         if (!this.result.compareAndSet(null, result))
             return false;
         onComplete();
-        lastUpdatedAtNs = Clock.Global.nanoTime();
+        lastUpdatedAtNs = clock.nanoTime();
         return true;
     }
 

src/java/org/apache/cassandra/repair/state/AbstractState.java

@@ -23,6 +23,27 @@ import org.apache.cassandra.utils.Clock;
 
 public abstract class AbstractState<T extends Enum<T>, I> extends AbstractCompletable<I> implements State<T, I>
 {
+    protected enum UpdateType
+    {
+        NO_CHANGE, ACCEPTED,
+        LARGER_STATE_SEEN, ALREADY_COMPLETED;
+
+        protected boolean isRejected()
+        {
+            switch (this)
+            {
+                case NO_CHANGE:
+                case ACCEPTED:
+                    return false;
+                case LARGER_STATE_SEEN:
+                case ALREADY_COMPLETED:
+                    return true;
+                default:
+                    throw new IllegalStateException("Unknown type: " + this);
+            }
+        }
+    }
+
     public static final int INIT = -1;
     public static final int COMPLETE = -2;
 
@@ -30,13 +51,19 @@ public abstract class AbstractState<T extends Enum<T>, I> extends AbstractComple
     protected final long[] stateTimesNanos;
     protected int currentState = INIT;
 
-    public AbstractState(I id, Class<T> klass)
+    public AbstractState(Clock clock, I id, Class<T> klass)
     {
-        super(id);
+        super(clock, id);
         this.klass = klass;
         this.stateTimesNanos = new long[klass.getEnumConstants().length];
     }
 
+    @Override
+    public boolean isAccepted()
+    {
+        return currentState == INIT ? false : true;
+    }
+
     @Override
     public T getStatus()
     {
@@ -46,6 +73,27 @@ public abstract class AbstractState<T extends Enum<T>, I> extends AbstractComple
         return klass.getEnumConstants()[current];
     }
 
+    public String status()
+    {
+        T state = getStatus();
+        Result result = getResult();
+        if (result != null)
+            return result.kind.name();
+        if (state == null)
+            return "init";
+        return state.name();
+    }
+
+    @Override
+    public String toString()
+    {
+        return getClass().getSimpleName() + "{" +
+               "id=" + id +
+               ", status=" + status() +
+               ", lastUpdatedAtNs=" + lastUpdatedAtNs +
+               '}';
+    }
+
     public int getCurrentState()
     {
         return currentState;
@@ -85,11 +133,22 @@ public abstract class AbstractState<T extends Enum<T>, I> extends AbstractComple
 
     protected void updateState(T state)
     {
-        int currentState = this.currentState;
-        if (currentState >= state.ordinal())
+        if (maybeUpdateState(state).isRejected())
             throw new IllegalStateException("State went backwards; current=" + klass.getEnumConstants()[currentState] + ", desired=" + state);
-        long now = Clock.Global.nanoTime();
+    }
+
+    protected UpdateType maybeUpdateState(T state)
+    {
+        int currentState = this.currentState;
+        if (currentState == COMPLETE)
+            return UpdateType.ALREADY_COMPLETED;
+        if (currentState == state.ordinal())
+            return UpdateType.NO_CHANGE;
+        if (currentState > state.ordinal())
+            return UpdateType.LARGER_STATE_SEEN;
+        long now = clock.nanoTime();
         stateTimesNanos[this.currentState = state.ordinal()] = now;
         lastUpdatedAtNs = now;
+        return UpdateType.ACCEPTED;
     }
 }

src/java/org/apache/cassandra/repair/state/Completable.java

@@ -17,6 +17,7 @@
  */
 package org.apache.cassandra.repair.state;
 
+import java.util.Objects;
 import java.util.concurrent.TimeUnit;
 
 import org.apache.cassandra.utils.Clock;
@@ -78,24 +79,48 @@ public interface Completable<I>
             this.message = message;
         }
 
-        protected static Result success()
+        public static Result success()
         {
             return new Result(Result.Kind.SUCCESS, null);
         }
 
-        protected static Result success(String msg)
+        public static Result success(String msg)
         {
             return new Result(Result.Kind.SUCCESS, msg);
         }
 
-        protected static Result skip(String msg)
+        public static Result skip(String msg)
         {
             return new Result(Result.Kind.SKIPPED, msg);
         }
 
-        protected static Result fail(String msg)
+        public static Result fail(String msg)
         {
             return new Result(Result.Kind.FAILURE, msg);
         }
+
+        @Override
+        public boolean equals(Object o)
+        {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            Result result = (Result) o;
+            return kind == result.kind && Objects.equals(message, result.message);
+        }
+
+        @Override
+        public int hashCode()
+        {
+            return Objects.hash(kind, message);
+        }
+
+        @Override
+        public String toString()
+        {
+            return "Result{" +
+                   "kind=" + kind +
+                   ", message='" + message + '\'' +
+                   '}';
+        }
     }
 }

src/java/org/apache/cassandra/repair/state/CoordinatorState.java

@@ -17,18 +17,21 @@
  */
 package org.apache.cassandra.repair.state;
 
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.List;
 import java.util.Objects;
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
+import java.util.stream.Collectors;
 
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.repair.CommonRange;
-import org.apache.cassandra.repair.RepairRunnable;
+import org.apache.cassandra.repair.RepairCoordinator;
 import org.apache.cassandra.repair.messages.RepairOption;
+import org.apache.cassandra.utils.Clock;
 import org.apache.cassandra.utils.TimeUUID;
 
 import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID;
@@ -48,14 +51,14 @@ public class CoordinatorState extends AbstractState<CoordinatorState.State, Time
     private final ConcurrentMap<TimeUUID, SessionState> sessions = new ConcurrentHashMap<>();
 
     private List<ColumnFamilyStore> columnFamilies = null;
-    private RepairRunnable.NeighborsAndRanges neighborsAndRanges = null;
+    private RepairCoordinator.NeighborsAndRanges neighborsAndRanges = null;
 
     // API to split function calls for phase changes from getting the state
     public final Phase phase = new Phase();
 
-    public CoordinatorState(int cmd, String keyspace, RepairOption options)
+    public CoordinatorState(Clock clock, int cmd, String keyspace, RepairOption options)
     {
-        super(nextTimeUUID(), State.class);
+        super(clock, nextTimeUUID(), State.class);
         this.cmd = cmd;
         this.keyspace = Objects.requireNonNull(keyspace);
         this.options = Objects.requireNonNull(options);
@@ -88,7 +91,7 @@ public class CoordinatorState extends AbstractState<CoordinatorState.State, Time
         return columnFamilies.stream().map(ColumnFamilyStore::getTableName).toArray(String[]::new);
     }
 
-    public RepairRunnable.NeighborsAndRanges getNeighborsAndRanges()
+    public RepairCoordinator.NeighborsAndRanges getNeighborsAndRanges()
     {
         return neighborsAndRanges;
     }
@@ -114,6 +117,32 @@ public class CoordinatorState extends AbstractState<CoordinatorState.State, Time
         return neighborsAndRanges.filterCommonRanges(keyspace, getColumnFamilyNames());
     }
 
+    @Override
+    public String status()
+    {
+        State currentState = getStatus();
+        Result result = getResult();
+        if (result != null)
+            return result.kind.name();
+        else if (currentState == null)
+            return "init";
+        else if (currentState == State.REPAIR_START)
+            return currentState.name() + " " + sessions.entrySet().stream().map(e -> e.getKey() + " -> " + e.getValue().status()).collect(Collectors.toList());
+        else
+            return currentState.name();
+    }
+
+    @Override
+    public String toString()
+    {
+        return "CoordinatorState{" +
+               "id=" + id +
+               ", stateTimesNanos=" + Arrays.toString(stateTimesNanos) +
+               ", status=" + status() +
+               ", lastUpdatedAtNs=" + lastUpdatedAtNs +
+               '}';
+    }
+
     public final class Phase extends BaseSkipPhase
     {
         public void setup()
@@ -121,7 +150,7 @@ public class CoordinatorState extends AbstractState<CoordinatorState.State, Time
             updateState(State.SETUP);
         }
 
-        public void start(List<ColumnFamilyStore> columnFamilies, RepairRunnable.NeighborsAndRanges neighborsAndRanges)
+        public void start(List<ColumnFamilyStore> columnFamilies, RepairCoordinator.NeighborsAndRanges neighborsAndRanges)
         {
             CoordinatorState.this.columnFamilies = Objects.requireNonNull(columnFamilies);
             CoordinatorState.this.neighborsAndRanges = Objects.requireNonNull(neighborsAndRanges);

src/java/org/apache/cassandra/repair/state/JobState.java

@@ -24,6 +24,7 @@ import com.google.common.collect.ImmutableSet;
 
 import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.repair.RepairJobDesc;
+import org.apache.cassandra.utils.Clock;
 
 public class JobState extends AbstractState<JobState.State, UUID>
 {
@@ -40,9 +41,9 @@ public class JobState extends AbstractState<JobState.State, UUID>
 
     public final Phase phase = new Phase();
 
-    public JobState(RepairJobDesc desc, ImmutableSet<InetAddressAndPort> endpoints)
+    public JobState(Clock clock, RepairJobDesc desc, ImmutableSet<InetAddressAndPort> endpoints)
     {
-        super(desc.determanisticId(), State.class);
+        super(clock, desc.determanisticId(), State.class);
         this.desc = desc;
         this.endpoints = endpoints;
     }

src/java/org/apache/cassandra/repair/state/ParticipateState.java

@@ -22,17 +22,26 @@ import java.util.List;
 import java.util.UUID;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+import javax.annotation.Nullable;
 
 import org.apache.cassandra.dht.Range;
 import org.apache.cassandra.dht.Token;
 import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.repair.RepairJobDesc;
 import org.apache.cassandra.repair.messages.PrepareMessage;
 import org.apache.cassandra.schema.TableId;
 import org.apache.cassandra.streaming.PreviewKind;
+import org.apache.cassandra.utils.Clock;
 import org.apache.cassandra.utils.TimeUUID;
 
 public class ParticipateState extends AbstractCompletable<TimeUUID>
 {
+    public enum RegisterStatus
+    { ACCEPTED, EXISTS, STATUS_REJECTED, ALREADY_COMPLETED }
     public final InetAddressAndPort initiator;
     public final List<TableId> tableIds;
     public final Collection<Range<Token>> ranges;
@@ -40,14 +49,21 @@ public class ParticipateState extends AbstractCompletable<TimeUUID>
     public final long repairedAt;
     public final boolean global;
     public final PreviewKind previewKind;
-
-    private final ConcurrentMap<UUID, ValidationState> validations = new ConcurrentHashMap<>();
+    private volatile boolean accepted = false;
 
     public final Phase phase = new Phase();
 
-    public ParticipateState(InetAddressAndPort initiator, PrepareMessage msg)
+    @Override
+    public boolean isAccepted()
     {
-        super(msg.parentRepairSession);
+        return accepted;
+    }
+
+    public final ConcurrentMap<RepairJobDesc, Job> jobs = new ConcurrentHashMap<>();
+
+    public ParticipateState(Clock clock, InetAddressAndPort initiator, PrepareMessage msg)
+    {
+        super(clock, msg.parentRepairSession);
         this.initiator = initiator;
         this.tableIds = msg.tableIds;
         this.ranges = msg.ranges;
@@ -57,24 +73,150 @@ public class ParticipateState extends AbstractCompletable<TimeUUID>
         this.previewKind = msg.previewKind;
     }
 
-    public boolean register(ValidationState state)
+    @Nullable
+    public Job job(RepairJobDesc desc)
     {
-        ValidationState current = validations.putIfAbsent(state.id, state);
-        return current == null;
+        return jobs.get(desc);
+    }
+
+    public Job getOrCreateJob(RepairJobDesc desc)
+    {
+        return jobs.computeIfAbsent(desc, d -> new Job(clock, d));
+    }
+
+    @Nullable
+    public ValidationState validation(RepairJobDesc desc)
+    {
+        Job job = job(desc);
+        if (job == null)
+            return null;
+        return job.validation();
+    }
+
+    public RegisterStatus register(ValidationState state)
+    {
+        return getOrCreateJob(state.desc).register(state);
+    }
+
+    @Nullable
+    public SyncState sync(RepairJobDesc desc, SyncState.Id id)
+    {
+        Job job = job(desc);
+        if (job == null)
+            return null;
+        return job.sync(id);
+    }
+
+    public RegisterStatus register(SyncState state)
+    {
+        return getOrCreateJob(state.id.desc).register(state);
     }
 
     public Collection<ValidationState> validations()
     {
-        return validations.values();
+        return jobs.values().stream()
+                   .map(j -> j.validation())
+                   .filter(f -> f != null)
+                   .collect(Collectors.toList());
     }
 
     public Collection<UUID> validationIds()
     {
-        return validations.keySet();
+        return jobs.values().stream()
+                   .map(j -> j.validation())
+                   .filter(f -> f != null)
+                   .map(v -> v.id)
+                   .collect(Collectors.toList());
+    }
+
+    @Override
+    public String toString()
+    {
+        Result result = getResult();
+        return "ParticipateState{" +
+               "initiator=" + initiator +
+               ", status=" + (result == null ? "pending" : result.toString()) +
+               ", jobs=" + jobs.values() +
+               '}';
     }
 
     public class Phase extends BasePhase
     {
+        public void accept()
+        {
+            accepted = true;
+        }
+    }
 
+    public static class Job extends AbstractState<Job.State, RepairJobDesc>
+    {
+        public enum State { ACCEPT, SNAPSHOT, VALIDATION, SYNC }
+
+        private final AtomicReference<ValidationState> validation = new AtomicReference<>(null);
+        private final ConcurrentMap<SyncState.Id, SyncState> syncs = new ConcurrentHashMap<>();
+
+        public Job(Clock clock, RepairJobDesc desc)
+        {
+            super(clock, desc, State.class);
+        }
+
+        @Override
+        protected synchronized UpdateType maybeUpdateState(State state)
+        {
+            return super.maybeUpdateState(state);
+        }
+
+        public void snapshot()
+        {
+            updateState(State.SNAPSHOT);
+        }
+
+        public RegisterStatus register(ValidationState state)
+        {
+            return register(s -> validation.compareAndSet(null, s) ? null : validation(), State.VALIDATION, state);
+        }
+
+        @Nullable
+        public ValidationState validation()
+        {
+            return validation.get();
+        }
+
+        public RegisterStatus register(SyncState state)
+        {
+            return register(s -> syncs.putIfAbsent(s.id, s), State.SYNC, state);
+        }
+
+        private <I, S extends AbstractState<?, I>> RegisterStatus register(Function<S, S> putter, State state, S value)
+        {
+            UpdateType updateType = maybeUpdateState(state);
+            switch (updateType)
+            {
+                case ALREADY_COMPLETED:
+                    return RegisterStatus.ALREADY_COMPLETED;
+                case LARGER_STATE_SEEN:
+                    return RegisterStatus.STATUS_REJECTED;
+                case ACCEPTED:
+                case NO_CHANGE:
+                    // allow
+                    break;
+                default:
+                    throw new IllegalStateException("Unknown status: " + updateType);
+            }
+            S current = putter.apply(value);
+            return current == null ? RegisterStatus.ACCEPTED : RegisterStatus.EXISTS;
+        }
+
+        @Nullable
+        public SyncState sync(SyncState.Id id)
+        {
+            return syncs.get(id);
+        }
+
+        @Override
+        public String toString()
+        {
+            return super.toString();
+        }
     }
 }

src/java/org/apache/cassandra/repair/state/SessionState.java

@@ -22,9 +22,11 @@ import java.util.Set;
 import java.util.UUID;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
+import java.util.stream.Collectors;
 
 import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.repair.CommonRange;
+import org.apache.cassandra.utils.Clock;
 import org.apache.cassandra.utils.TimeUUID;
 
 import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID;
@@ -44,9 +46,9 @@ public class SessionState extends AbstractState<SessionState.State, TimeUUID>
 
     public final Phase phase = new Phase();
 
-    public SessionState(TimeUUID parentRepairSession, String keyspace, String[] cfnames, CommonRange commonRange)
+    public SessionState(Clock clock, TimeUUID parentRepairSession, String keyspace, String[] cfnames, CommonRange commonRange)
     {
-        super(nextTimeUUID(), State.class);
+        super(clock, nextTimeUUID(), State.class);
         this.parentRepairSession = parentRepairSession;
         this.keyspace = keyspace;
         this.cfnames = cfnames;
@@ -73,6 +75,21 @@ public class SessionState extends AbstractState<SessionState.State, TimeUUID>
         return commonRange.endpoints;
     }
 
+    @Override
+    public String status()
+    {
+        State state = getStatus();
+        Result result = getResult();
+        if (result != null)
+            return result.kind.name();
+        else if (state == null)
+            return "init";
+        else if (state == State.JOBS_START)
+            return state.name() + " " + jobs.entrySet().stream().map(e -> e.getKey() + " -> " + e.getValue().status()).collect(Collectors.toList());
+        else
+            return state.name();
+    }
+
     public void register(JobState state)
     {
         jobs.put(state.id, state);

src/java/org/apache/cassandra/repair/state/SyncState.java

@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.repair.state;
+
+import java.util.Objects;
+
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.repair.RepairJobDesc;
+import org.apache.cassandra.utils.Clock;
+
+public class SyncState extends AbstractState<SyncState.State, SyncState.Id>
+{
+    public enum State
+    { ACCEPT, PLANNING, START }
+
+    public final Phase phase = new Phase();
+
+    public SyncState(Clock clock, RepairJobDesc desc, InetAddressAndPort initiator, InetAddressAndPort src, InetAddressAndPort dst)
+    {
+        super(clock, new Id(desc, initiator, src, dst), State.class);
+    }
+
+    public final class Phase extends BaseSkipPhase
+    {
+        public void accept()
+        {
+            updateState(State.ACCEPT);
+        }
+
+        public void planning()
+        {
+            updateState(State.PLANNING);
+        }
+
+        public void start()
+        {
+            updateState(State.START);
+        }
+    }
+
+    public static class Id
+    {
+        public final RepairJobDesc desc;
+        public final InetAddressAndPort initiator, src, dst;
+
+        public Id(RepairJobDesc desc, InetAddressAndPort initiator, InetAddressAndPort src, InetAddressAndPort dst)
+        {
+            this.desc = desc;
+            this.initiator = initiator;
+            this.src = src;
+            this.dst = dst;
+        }
+
+        @Override
+        public boolean equals(Object o)
+        {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            Id id = (Id) o;
+            return desc.equals(id.desc) && initiator.equals(id.initiator) && src.equals(id.src) && dst.equals(id.dst);
+        }
+
+        @Override
+        public int hashCode()
+        {
+            return Objects.hash(desc, initiator, src, dst);
+        }
+    }
+}

src/java/org/apache/cassandra/repair/state/ValidationState.java

@@ -21,11 +21,12 @@ import java.util.UUID;
 
 import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.repair.RepairJobDesc;
+import org.apache.cassandra.utils.Clock;
 
 public class ValidationState extends AbstractState<ValidationState.State, UUID>
 {
     public enum State
-    { START, SENDING_TREES }
+    { ACCEPT, START, SENDING_TREES }
 
     public final Phase phase = new Phase();
     public final RepairJobDesc desc;
@@ -35,9 +36,10 @@ public class ValidationState extends AbstractState<ValidationState.State, UUID>
     public long partitionsProcessed;
     public long bytesRead;
 
-    public ValidationState(RepairJobDesc desc, InetAddressAndPort initiator)
+    public ValidationState(Clock clock, RepairJobDesc desc, InetAddressAndPort initiator)
     {
-        super(desc.determanisticId(), State.class);
+        // UUID is used to make the validations table easier for users to lookup by a single key rather than a composite key
+        super(clock, desc.determanisticId(), State.class);
         this.desc = desc;
         this.initiator = initiator;
     }
@@ -56,6 +58,11 @@ public class ValidationState extends AbstractState<ValidationState.State, UUID>
 
     public final class Phase extends BaseSkipPhase
     {
+        public void accept()
+        {
+            updateState(State.ACCEPT);
+        }
+
         public void start(long estimatedPartitions, long estimatedTotalBytes)
         {
             updateState(State.START);

src/java/org/apache/cassandra/schema/Tables.java

@@ -123,7 +123,7 @@ public final class Tables implements Iterable<TableMetadata>
     }
 
     @Nullable
-    TableMetadata getNullable(TableId id)
+    public TableMetadata getNullable(TableId id)
     {
         return tablesById.get(id);
     }

src/java/org/apache/cassandra/service/ActiveRepairService.java

@@ -26,8 +26,8 @@ import java.util.concurrent.ScheduledFuture;
 import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
-import java.util.concurrent.atomic.AtomicBoolean;
 import javax.management.openmbean.CompositeData;
+import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.Predicate;
 import java.util.stream.Collectors;
@@ -44,7 +44,7 @@ import com.google.common.collect.Multimap;
 import org.apache.cassandra.concurrent.ExecutorPlus;
 import org.apache.cassandra.config.Config;
 import org.apache.cassandra.config.DurationSpec;
-import org.apache.cassandra.db.compaction.CompactionManager;
+import org.apache.cassandra.repair.SharedContext;
 import org.apache.cassandra.exceptions.ConfigurationException;
 import org.apache.cassandra.locator.AbstractReplicationStrategy;
 import org.apache.cassandra.locator.EndpointsByRange;
@@ -59,11 +59,10 @@ import org.apache.cassandra.schema.Schema;
 import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.streaming.PreviewKind;
 import org.apache.cassandra.utils.TimeUUID;
-import org.apache.cassandra.utils.concurrent.CountDownLatch;
+import org.apache.cassandra.utils.concurrent.AsyncPromise;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import org.apache.cassandra.concurrent.ScheduledExecutors;
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.Keyspace;
@@ -73,10 +72,8 @@ import org.apache.cassandra.exceptions.RequestFailureReason;
 import org.apache.cassandra.gms.ApplicationState;
 import org.apache.cassandra.gms.EndpointState;
 import org.apache.cassandra.gms.FailureDetector;
-import org.apache.cassandra.gms.Gossiper;
 import org.apache.cassandra.gms.IEndpointStateChangeSubscriber;
 import org.apache.cassandra.gms.IFailureDetectionEventListener;
-import org.apache.cassandra.gms.IFailureDetector;
 import org.apache.cassandra.gms.VersionedValue;
 import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.locator.TokenMetadata;
@@ -84,7 +81,6 @@ import org.apache.cassandra.metrics.RepairMetrics;
 import org.apache.cassandra.net.RequestCallback;
 import org.apache.cassandra.net.Verb;
 import org.apache.cassandra.net.Message;
-import org.apache.cassandra.net.MessagingService;
 import org.apache.cassandra.repair.CommonRange;
 import org.apache.cassandra.repair.NoSuchRepairSessionException;
 import org.apache.cassandra.service.paxos.PaxosRepair;
@@ -106,8 +102,6 @@ import org.apache.cassandra.repair.messages.RepairOption;
 import org.apache.cassandra.repair.messages.SyncResponse;
 import org.apache.cassandra.repair.messages.ValidationResponse;
 import org.apache.cassandra.schema.TableId;
-import org.apache.cassandra.utils.FBUtilities;
-import org.apache.cassandra.utils.MBeanWrapper;
 import org.apache.cassandra.utils.MerkleTrees;
 import org.apache.cassandra.utils.Pair;
 import org.apache.cassandra.utils.concurrent.Future;
@@ -126,12 +120,9 @@ import static org.apache.cassandra.config.CassandraRelevantProperties.SKIP_PAXOS
 import static org.apache.cassandra.config.CassandraRelevantProperties.SKIP_PAXOS_REPAIR_ON_TOPOLOGY_CHANGE_KEYSPACES;
 import static org.apache.cassandra.config.Config.RepairCommandPoolFullStrategy.reject;
 import static org.apache.cassandra.config.DatabaseDescriptor.*;
-import static org.apache.cassandra.net.Message.out;
 import static org.apache.cassandra.net.Verb.PREPARE_MSG;
-import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis;
+import static org.apache.cassandra.repair.messages.RepairMessage.notDone;
 import static org.apache.cassandra.utils.Simulate.With.MONITORS;
-import static org.apache.cassandra.utils.Clock.Global.nanoTime;
-import static org.apache.cassandra.utils.concurrent.CountDownLatch.newCountDownLatch;
 
 /**
  * ActiveRepairService is the starting point for manual "active" repairs.
@@ -150,6 +141,7 @@ import static org.apache.cassandra.utils.concurrent.CountDownLatch.newCountDownL
 @Simulate(with = MONITORS)
 public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFailureDetectionEventListener, ActiveRepairServiceMBean
 {
+
     public enum ParentRepairStatus
     {
         IN_PROGRESS, COMPLETED, FAILED
@@ -157,21 +149,35 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai
 
     public static class ConsistentSessions
     {
-        public final LocalSessions local = new LocalSessions();
-        public final CoordinatorSessions coordinated = new CoordinatorSessions();
+        public final LocalSessions local;
+        public final CoordinatorSessions coordinated;
+
+        public ConsistentSessions(SharedContext ctx)
+        {
+            local = new LocalSessions(ctx);
+            coordinated = new CoordinatorSessions(ctx);
+        }
     }
 
-    public final ConsistentSessions consistent = new ConsistentSessions();
+    public final ConsistentSessions consistent;
 
     private boolean registeredForEndpointChanges = false;
 
     private static final Logger logger = LoggerFactory.getLogger(ActiveRepairService.class);
-    // singleton enforcement
-    public static final ActiveRepairService instance = new ActiveRepairService(FailureDetector.instance, Gossiper.instance);
 
     public static final long UNREPAIRED_SSTABLE = 0;
     public static final TimeUUID NO_PENDING_REPAIR = null;
 
+    public static ActiveRepairService instance()
+    {
+        return Holder.instance;
+    }
+
+    private static class Holder
+    {
+        private static final ActiveRepairService instance = new ActiveRepairService();
+    }
+
     /**
      * A map of active coordinator session.
      */
@@ -182,6 +188,7 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai
     private final Cache<TimeUUID, CoordinatorState> repairs;
     // map of top level repair id (parent repair id) -> participate state
     private final Cache<TimeUUID, ParticipateState> participates;
+    public final SharedContext ctx;
 
     private volatile ScheduledFuture<?> irCleanup;
 
@@ -213,18 +220,22 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai
         return RepairCommandExecutorHandle.repairCommandExecutor;
     }
 
-    private final IFailureDetector failureDetector;
-    private final Gossiper gossiper;
     private final Cache<Integer, Pair<ParentRepairStatus, List<String>>> repairStatusByCmd;
+    public final ExecutorPlus snapshotExecutor;
 
-    public final ExecutorPlus snapshotExecutor = executorFactory().configurePooled("RepairSnapshotExecutor", 1)
-                                                                  .withKeepAlive(1, TimeUnit.HOURS)
-                                                                  .build();
-
-    public ActiveRepairService(IFailureDetector failureDetector, Gossiper gossiper)
+    public ActiveRepairService()
     {
-        this.failureDetector = failureDetector;
-        this.gossiper = gossiper;
+        this(SharedContext.Global.instance);
+    }
+
+    @VisibleForTesting
+    public ActiveRepairService(SharedContext ctx)
+    {
+        this.ctx = ctx;
+        consistent = new ConsistentSessions(ctx);
+        this.snapshotExecutor = ctx.executorFactory().configurePooled("RepairSnapshotExecutor", 1)
+                                   .withKeepAlive(1, TimeUnit.HOURS)
+                                   .build();
         this.repairStatusByCmd = CacheBuilder.newBuilder()
                                              .expireAfterWrite(PARENT_REPAIR_STATUS_EXPIRY_SECONDS.getLong(), TimeUnit.SECONDS)
                                              // using weight wouldn't work so well, since it doesn't reflect mutation of cached data
@@ -245,15 +256,15 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai
                                    .maximumSize(numElements)
                                    .build();
 
-        MBeanWrapper.instance.registerMBean(this, MBEAN_NAME);
+        ctx.mbean().registerMBean(this, MBEAN_NAME);
     }
 
     public void start()
     {
         consistent.local.start();
-        this.irCleanup = ScheduledExecutors.optionalTasks.scheduleAtFixedRate(consistent.local::cleanup, 0,
-                                                                              LocalSessions.CLEANUP_INTERVAL,
-                                                                              TimeUnit.SECONDS);
+        this.irCleanup = ctx.optionalTasks().scheduleAtFixedRate(consistent.local::cleanup, 0,
+                                                                 LocalSessions.CLEANUP_INTERVAL,
+                                                                 TimeUnit.SECONDS);
     }
 
     @VisibleForTesting
@@ -428,7 +439,7 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai
         if (cfnames.length == 0)
             return null;
 
-        final RepairSession session = new RepairSession(parentRepairSession, range, keyspace,
+        final RepairSession session = new RepairSession(ctx, parentRepairSession, range, keyspace,
                                                         parallelismDegree, isIncremental, pullRepair,
                                                         previewKind, optimiseStreams, repairPaxos, paxosOnly, cfnames);
         repairs.getIfPresent(parentRepairSession).register(session.state);
@@ -463,8 +474,8 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai
                IEndpointStateChangeSubscriber &
                IFailureDetectionEventListener> void registerOnFdAndGossip(final T task)
     {
-        gossiper.register(task);
-        failureDetector.registerFailureDetectionEventListener(task);
+        ctx.gossiper().register(task);
+        ctx.failureDetector().registerFailureDetectionEventListener(task);
 
         // unregister listeners at completion
         task.addListener(new Runnable()
@@ -474,8 +485,8 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai
              */
             public void run()
             {
-                failureDetector.unregisterFailureDetectionEventListener(task);
-                gossiper.unregister(task);
+                ctx.failureDetector().unregisterFailureDetectionEventListener(task);
+                ctx.gossiper().unregister(task);
             }
         });
     }
@@ -511,9 +522,9 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai
      * @param dataCenters         the data centers to involve in the repair
      * @return neighbors with whom we share the provided range
      */
-    public static EndpointsForRange getNeighbors(String keyspaceName, Iterable<Range<Token>> keyspaceLocalRanges,
-                                                 Range<Token> toRepair, Collection<String> dataCenters,
-                                                 Collection<String> hosts)
+    public EndpointsForRange getNeighbors(String keyspaceName, Iterable<Range<Token>> keyspaceLocalRanges,
+                                          Range<Token> toRepair, Collection<String> dataCenters,
+                                          Collection<String> hosts)
     {
         StorageService ss = StorageService.instance;
         EndpointsByRange replicaSets = ss.getRangeToAddressMap(keyspaceName);
@@ -536,7 +547,8 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai
         if (rangeSuperSet == null || !replicaSets.containsKey(rangeSuperSet))
             return EndpointsForRange.empty(toRepair);
 
-        EndpointsForRange neighbors = replicaSets.get(rangeSuperSet).withoutSelf();
+        // same as withoutSelf(), but done this way for testing
+        EndpointsForRange neighbors = replicaSets.get(rangeSuperSet).filter(r -> !ctx.broadcastAddressAndPort().equals(r.endpoint()));
 
         if (dataCenters != null && !dataCenters.isEmpty())
         {
@@ -553,7 +565,7 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai
                 try
                 {
                     final InetAddressAndPort endpoint = InetAddressAndPort.getByName(host.trim());
-                    if (endpoint.equals(FBUtilities.getBroadcastAddressAndPort()) || neighbors.endpoints().contains(endpoint))
+                    if (endpoint.equals(ctx.broadcastAddressAndPort()) || neighbors.endpoints().contains(endpoint))
                         specifiedHost.add(endpoint);
                 }
                 catch (UnknownHostException e)
@@ -562,7 +574,7 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai
                 }
             }
 
-            if (!specifiedHost.contains(FBUtilities.getBroadcastAddressAndPort()))
+            if (!specifiedHost.contains(ctx.broadcastAddressAndPort()))
                 throw new IllegalArgumentException("The current host must be part of the repair");
 
             if (specifiedHost.size() <= 1)
@@ -573,7 +585,7 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai
                 throw new IllegalArgumentException(String.format(msg, hosts, toRepair, neighbors));
             }
 
-            specifiedHost.remove(FBUtilities.getBroadcastAddressAndPort());
+            specifiedHost.remove(ctx.broadcastAddressAndPort());
             return neighbors.keep(specifiedHost);
         }
 
@@ -585,14 +597,14 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai
      * incremental repairs, forced incremental repairs, and full repairs, the UNREPAIRED_SSTABLE value will prevent
      * sstables from being promoted to repaired or preserve the repairedAt/pendingRepair values, respectively.
      */
-    static long getRepairedAt(RepairOption options, boolean force)
+    long getRepairedAt(RepairOption options, boolean force)
     {
         // we only want to set repairedAt for incremental repairs including all replicas for a token range. For non-global incremental repairs, full repairs, the UNREPAIRED_SSTABLE value will prevent
         // sstables from being promoted to repaired or preserve the repairedAt/pendingRepair values, respectively. For forced repairs, repairedAt time is only set to UNREPAIRED_SSTABLE if we actually
         // end up skipping replicas
         if (options.isIncremental() && options.isGlobal() && !force)
         {
-            return currentTimeMillis();
+            return ctx.clock().currentTimeMillis();
         }
         else
         {
@@ -600,11 +612,11 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai
         }
     }
 
-    public static boolean verifyCompactionsPendingThreshold(TimeUUID parentRepairSession, PreviewKind previewKind)
+    public boolean verifyCompactionsPendingThreshold(TimeUUID parentRepairSession, PreviewKind previewKind)
     {
         // Snapshot values so failure message is consistent with decision
-        int pendingCompactions = CompactionManager.instance.getPendingTasks();
-        int pendingThreshold = ActiveRepairService.instance.getRepairPendingCompactionRejectThreshold();
+        int pendingCompactions = ctx.compactionManager().getPendingTasks();
+        int pendingThreshold = getRepairPendingCompactionRejectThreshold();
         if (pendingCompactions > pendingThreshold)
         {
             logger.error("[{}] Rejecting incoming repair, pending compactions ({}) above threshold ({})",
@@ -614,54 +626,28 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai
         return true;
     }
 
-    public TimeUUID prepareForRepair(TimeUUID parentRepairSession, InetAddressAndPort coordinator, Set<InetAddressAndPort> endpoints, RepairOption options, boolean isForcedRepair, List<ColumnFamilyStore> columnFamilyStores)
+    public Future<?> prepareForRepair(TimeUUID parentRepairSession, InetAddressAndPort coordinator, Set<InetAddressAndPort> endpoints, RepairOption options, boolean isForcedRepair, List<ColumnFamilyStore> columnFamilyStores)
     {
         if (!verifyCompactionsPendingThreshold(parentRepairSession, options.getPreviewKind()))
             failRepair(parentRepairSession, "Rejecting incoming repair, pending compactions above threshold"); // failRepair throws exception
 
         long repairedAt = getRepairedAt(options, isForcedRepair);
         registerParentRepairSession(parentRepairSession, coordinator, columnFamilyStores, options.getRanges(), options.isIncremental(), repairedAt, options.isGlobal(), options.getPreviewKind());
-        final CountDownLatch prepareLatch = newCountDownLatch(endpoints.size());
-        final AtomicBoolean status = new AtomicBoolean(true);
-        final Set<String> failedNodes = synchronizedSet(new HashSet<String>());
-        final AtomicInteger timeouts = new AtomicInteger(0);
-        RequestCallback callback = new RequestCallback()
-        {
-            @Override
-            public void onResponse(Message msg)
-            {
-                prepareLatch.decrement();
-            }
-
-            @Override
-            public void onFailure(InetAddressAndPort from, RequestFailureReason failureReason)
-            {
-                status.set(false);
-                failedNodes.add(from.toString());
-                if (failureReason == RequestFailureReason.TIMEOUT)
-                    timeouts.incrementAndGet();
-                prepareLatch.decrement();
-            }
-
-            @Override
-            public boolean invokeOnFailure()
-            {
-                return true;
-            }
-        };
+        AtomicInteger pending = new AtomicInteger(endpoints.size());
+        Set<String> failedNodes = synchronizedSet(new HashSet<>());
+        AsyncPromise<Void> promise = new AsyncPromise<>();
 
         List<TableId> tableIds = new ArrayList<>(columnFamilyStores.size());
         for (ColumnFamilyStore cfs : columnFamilyStores)
             tableIds.add(cfs.metadata.id);
 
         PrepareMessage message = new PrepareMessage(parentRepairSession, tableIds, options.getRanges(), options.isIncremental(), repairedAt, options.isGlobal(), options.getPreviewKind());
-        register(new ParticipateState(FBUtilities.getBroadcastAddressAndPort(), message));
+        register(new ParticipateState(ctx.clock(), ctx.broadcastAddressAndPort(), message));
         for (InetAddressAndPort neighbour : endpoints)
         {
-            if (FailureDetector.instance.isAlive(neighbour))
+            if (ctx.failureDetector().isAlive(neighbour))
             {
-                Message<RepairMessage> msg = out(PREPARE_MSG, message);
-                MessagingService.instance().sendWithCallback(msg, neighbour, callback);
+                sendPrepareWithRetries(parentRepairSession, pending, failedNodes, promise, neighbour, message);
             }
             else
             {
@@ -669,7 +655,7 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai
                 // remaining ones go down, we still want to fail so we don't create repair sessions that can't complete
                 if (isForcedRepair && !options.isIncremental())
                 {
-                    prepareLatch.decrement();
+                    pending.decrementAndGet();
                 }
                 else
                 {
@@ -678,22 +664,66 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai
                 }
             }
         }
-        try
-        {
-            if (!prepareLatch.await(getRpcTimeout(MILLISECONDS), MILLISECONDS) || timeouts.get() > 0)
-                failRepair(parentRepairSession, "Did not get replies from all endpoints.");
-        }
-        catch (InterruptedException e)
-        {
-            failRepair(parentRepairSession, "Interrupted while waiting for prepare repair response.");
-        }
+        // implement timeout to bound the runtime of the future
+        long timeoutMillis = getRepairRetrySpec().isEnabled() ? getRepairRpcTimeout(MILLISECONDS)
+                                                              : getRpcTimeout(MILLISECONDS);
+        ctx.optionalTasks().schedule(() -> {
+            if (promise.isDone())
+                return;
+            String errorMsg = "Did not get replies from all endpoints.";
+            if (promise.tryFailure(new RuntimeException(errorMsg)))
+                participateFailed(parentRepairSession, errorMsg);
+        }, timeoutMillis, MILLISECONDS);
 
-        if (!status.get())
-        {
-            failRepair(parentRepairSession, "Got negative replies from endpoints " + failedNodes);
-        }
+        return promise;
+    }
+
+    private void sendPrepareWithRetries(TimeUUID parentRepairSession,
+                                        AtomicInteger pending,
+                                        Set<String> failedNodes,
+                                        AsyncPromise<Void> promise,
+                                        InetAddressAndPort to,
+                                        RepairMessage msg)
+    {
+        RepairMessage.sendMessageWithRetries(ctx, notDone(promise), msg, PREPARE_MSG, to, new RequestCallback<>()
+        {
+            @Override
+            public void onResponse(Message<Object> msg)
+            {
+                ack();
+            }
+
+            @Override
+            public void onFailure(InetAddressAndPort from, RequestFailureReason failureReason)
+            {
+                failedNodes.add(from.toString());
+                if (failureReason == RequestFailureReason.TIMEOUT)
+                {
+                    pending.set(-1);
+                    promise.setFailure(failRepairException(parentRepairSession, "Did not get replies from all endpoints."));
+                }
+                else
+                {
+                    ack();
+                }
+            }
+
+            private void ack()
+            {
+                if (pending.decrementAndGet() == 0)
+                {
+                    if (failedNodes.isEmpty())
+                    {
+                        promise.setSuccess(null);
+                    }
+                    else
+                    {
+                        promise.setFailure(failRepairException(parentRepairSession, "Got negative replies from endpoints " + failedNodes));
+                    }
+                }
+            }
+        });
 
-        return parentRepairSession;
     }
 
     /**
@@ -707,10 +737,9 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai
         {
             try
             {
-                if (FailureDetector.instance.isAlive(endpoint))
+                if (ctx.failureDetector().isAlive(endpoint))
                 {
                     CleanupMessage message = new CleanupMessage(parentRepairSession);
-                    Message<CleanupMessage> msg = Message.out(Verb.CLEANUP_MSG, message);
 
                     RequestCallback loggingCallback = new RequestCallback()
                     {
@@ -728,8 +757,7 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai
                                          "of messages like this.", parentRepairSession, endpoint);
                         }
                     };
-
-                    MessagingService.instance().sendWithCallback(msg, endpoint, loggingCallback);
+                    RepairMessage.sendMessageWithRetries(ctx, message, Verb.CLEANUP_MSG, endpoint, loggingCallback);
                 }
             }
             catch (Exception exc)
@@ -743,12 +771,22 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai
     }
 
     private void failRepair(TimeUUID parentRepairSession, String errorMsg)
+    {
+        throw failRepairException(parentRepairSession, errorMsg);
+    }
+
+    private RuntimeException failRepairException(TimeUUID parentRepairSession, String errorMsg)
+    {
+        participateFailed(parentRepairSession, errorMsg);
+        removeParentRepairSession(parentRepairSession);
+        return new RuntimeException(errorMsg);
+    }
+
+    private void participateFailed(TimeUUID parentRepairSession, String errorMsg)
     {
         ParticipateState state = participate(parentRepairSession);
         if (state != null)
             state.phase.fail(errorMsg);
-        removeParentRepairSession(parentRepairSession);
-        throw new RuntimeException(errorMsg);
     }
 
     public synchronized void registerParentRepairSession(TimeUUID parentRepairSession, InetAddressAndPort coordinator, List<ColumnFamilyStore> columnFamilyStores, Collection<Range<Token>> ranges, boolean isIncremental, long repairedAt, boolean isGlobal, PreviewKind previewKind)
@@ -756,8 +794,8 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai
         assert isIncremental || repairedAt == ActiveRepairService.UNREPAIRED_SSTABLE;
         if (!registeredForEndpointChanges)
         {
-            Gossiper.instance.register(this);
-            FailureDetector.instance.registerFailureDetectionEventListener(this);
+            ctx.gossiper().register(this);
+            ctx.failureDetector().registerFailureDetectionEventListener(this);
             registeredForEndpointChanges = true;
         }
 
@@ -796,25 +834,25 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai
      */
     public synchronized ParentRepairSession removeParentRepairSession(TimeUUID parentSessionId)
     {
-        String snapshotName = parentSessionId.toString();
         ParentRepairSession session = parentRepairSessions.remove(parentSessionId);
         if (session == null)
             return null;
 
-        if (session.hasSnapshots)
+        String snapshotName = parentSessionId.toString();
+        if (session.hasSnapshots.get())
         {
             snapshotExecutor.submit(() -> {
                 logger.info("[repair #{}] Clearing snapshots for {}", parentSessionId,
                             session.columnFamilyStores.values()
                                                       .stream()
                                                       .map(cfs -> cfs.metadata().toString()).collect(Collectors.joining(", ")));
-                long startNanos = nanoTime();
+                long startNanos = ctx.clock().nanoTime();
                 for (ColumnFamilyStore cfs : session.columnFamilyStores.values())
                 {
                     if (cfs.snapshotExists(snapshotName))
                         cfs.clearSnapshot(snapshotName);
                 }
-                logger.info("[repair #{}] Cleared snapshots in {}ms", parentSessionId, TimeUnit.NANOSECONDS.toMillis(nanoTime() - startNanos));
+                logger.info("[repair #{}] Cleared snapshots in {}ms", parentSessionId, TimeUnit.NANOSECONDS.toMillis(ctx.clock().nanoTime() - startNanos));
             });
         }
         return session;
@@ -828,6 +866,13 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai
 
         if (session == null)
         {
+            switch (message.verb())
+            {
+                case VALIDATION_RSP:
+                case SYNC_RSP:
+                    ctx.messaging().send(message.emptyResponse(), message.from());
+                    break;
+            }
             if (payload instanceof ValidationResponse)
             {
                 // The trees may be off-heap, and will therefore need to be released.
@@ -845,13 +890,10 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai
         switch (message.verb())
         {
             case VALIDATION_RSP:
-                ValidationResponse validation = (ValidationResponse) payload;
-                session.validationComplete(desc, message.from(), validation.trees);
+                session.validationComplete(desc, (Message<ValidationResponse>) message);
                 break;
             case SYNC_RSP:
-                // one of replica is synced.
-                SyncResponse sync = (SyncResponse) payload;
-                session.syncComplete(desc, sync.nodes, sync.success, sync.summaries);
+                session.syncComplete(desc, (Message<SyncResponse>) message);
                 break;
             default:
                 break;
@@ -873,7 +915,7 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai
         public final long repairedAt;
         public final InetAddressAndPort coordinator;
         public final PreviewKind previewKind;
-        public volatile boolean hasSnapshots = false;
+        public final AtomicBoolean hasSnapshots = new AtomicBoolean(false);
 
         public ParentRepairSession(InetAddressAndPort coordinator, List<ColumnFamilyStore> columnFamilyStores, Collection<Range<Token>> ranges, boolean isIncremental, long repairedAt, boolean isGlobal, PreviewKind previewKind)
         {
@@ -930,9 +972,9 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai
                    '}';
         }
 
-        public void setHasSnapshots()
+        public boolean setHasSnapshots()
         {
-            hasSnapshots = true;
+            return hasSnapshots.compareAndSet(false, true);
         }
     }
 

src/java/org/apache/cassandra/service/CassandraDaemon.java

@@ -367,7 +367,7 @@ public class CassandraDaemon
         if (sizeRecorderInterval > 0)
             ScheduledExecutors.optionalTasks.scheduleWithFixedDelay(SizeEstimatesRecorder.instance, 30, sizeRecorderInterval, TimeUnit.SECONDS);
 
-        ActiveRepairService.instance.start();
+        ActiveRepairService.instance().start();
         StreamManager.instance.start();
 
         // Prepared statements

src/java/org/apache/cassandra/service/StorageService.java

@@ -179,7 +179,7 @@ import org.apache.cassandra.metrics.StorageMetrics;
 import org.apache.cassandra.net.AsyncOneResponse;
 import org.apache.cassandra.net.Message;
 import org.apache.cassandra.net.MessagingService;
-import org.apache.cassandra.repair.RepairRunnable;
+import org.apache.cassandra.repair.RepairCoordinator;
 import org.apache.cassandra.repair.messages.RepairOption;
 import org.apache.cassandra.schema.CompactionParams.TombstoneOption;
 import org.apache.cassandra.schema.KeyspaceMetadata;
@@ -361,8 +361,12 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
 
     public RangesAtEndpoint getLocalReplicas(String keyspaceName)
     {
-        return Keyspace.open(keyspaceName).getReplicationStrategy()
-                .getAddressReplicas(FBUtilities.getBroadcastAddressAndPort());
+        return getReplicas(keyspaceName, FBUtilities.getBroadcastAddressAndPort());
+    }
+
+    public RangesAtEndpoint getReplicas(String keyspaceName, InetAddressAndPort endpoint)
+    {
+        return Keyspace.open(keyspaceName).getReplicationStrategy().getAddressReplicas(endpoint);
     }
 
     public List<Range<Token>> getLocalRanges(String ks)
@@ -4593,7 +4597,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
      * @param cfNames CFs
      * @throws java.lang.IllegalArgumentException when given CF name does not exist
      */
-    public Iterable<ColumnFamilyStore> getValidColumnFamilies(boolean allowIndexes, boolean autoAddIndexes, String keyspaceName, String... cfNames) throws IOException
+    public Iterable<ColumnFamilyStore> getValidColumnFamilies(boolean allowIndexes, boolean autoAddIndexes, String keyspaceName, String... cfNames)
     {
         Keyspace keyspace = getValidKeyspace(keyspaceName);
         return keyspace.getValidColumnFamilies(allowIndexes, autoAddIndexes, cfNames);
@@ -4714,7 +4718,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
     {
         if (!options.getDataCenters().isEmpty() && !options.getDataCenters().contains(DatabaseDescriptor.getLocalDataCenter()))
         {
-            throw new IllegalArgumentException("the local data center must be part of the repair");
+            throw new IllegalArgumentException("the local data center must be part of the repair; requested " + options.getDataCenters() + " but DC is " + DatabaseDescriptor.getLocalDataCenter());
         }
         Set<String> existingDatacenters = tokenMetadata.cloneOnlyTokenMap().getTopology().getDatacenterEndpoints().keys().elementSet();
         List<String> datacenters = new ArrayList<>(options.getDataCenters());
@@ -4724,7 +4728,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
             throw new IllegalArgumentException("data center(s) " + datacenters.toString() + " not found");
         }
 
-        RepairRunnable task = new RepairRunnable(this, cmd, options, keyspace);
+        RepairCoordinator task = new RepairCoordinator(this, cmd, options, keyspace);
         task.addProgressListener(progressSupport);
         for (ProgressListener listener : listeners)
             task.addProgressListener(listener);
@@ -4807,7 +4811,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
                 continue;
 
             List<Range<Token>> ranges = getLocalAndPendingRanges(ksName);
-            futures.add(ActiveRepairService.instance.repairPaxosForTopologyChange(ksName, ranges, reason));
+            futures.add(ActiveRepairService.instance().repairPaxosForTopologyChange(ksName, ranges, reason));
         }
 
         return FutureCombiner.allOf(futures);
@@ -4827,13 +4831,13 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
 
     public void forceTerminateAllRepairSessions()
     {
-        ActiveRepairService.instance.terminateSessions();
+        ActiveRepairService.instance().terminateSessions();
     }
 
     @Nullable
     public List<String> getParentRepairStatus(int cmd)
     {
-        Pair<ParentRepairStatus, List<String>> pair = ActiveRepairService.instance.getRepairStatus(cmd);
+        Pair<ParentRepairStatus, List<String>> pair = ActiveRepairService.instance().getRepairStatus(cmd);
         return pair == null ? null :
                ImmutableList.<String>builder().add(pair.left.name()).addAll(pair.right).build();
     }
@@ -5729,7 +5733,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
                 shutdownClientServers();
             ScheduledExecutors.optionalTasks.shutdown();
             Gossiper.instance.stop();
-            ActiveRepairService.instance.stop();
+            ActiveRepairService.instance().stop();
 
             if (!isFinalShutdown)
                 setMode(Mode.DRAINING, "shutting down MessageService", false);

src/java/org/apache/cassandra/service/StorageServiceMBean.java

@@ -477,7 +477,7 @@ public interface StorageServiceMBean extends NotificationEmitter
     /**
      * Get the status of a given parent repair session.
      * @param cmd the int reference returned when issuing the repair
-     * @return status of parent repair from enum {@link org.apache.cassandra.repair.RepairRunnable.Status}
+     * @return status of parent repair from enum {@link ActiveRepairService.ParentRepairStatus}
      * followed by final message or messages of the session
      */
     @Nullable

src/java/org/apache/cassandra/streaming/PreviewKind.java

@@ -86,7 +86,7 @@ public enum PreviewKind
             StatsMetadata sstableMetadata = sstable.getSSTableMetadata();
             if (sstableMetadata.pendingRepair != null)
             {
-                LocalSession session = ActiveRepairService.instance.consistent.local.getSession(sstableMetadata.pendingRepair);
+                LocalSession session = ActiveRepairService.instance().consistent.local.getSession(sstableMetadata.pendingRepair);
                 if (session == null)
                     return false;
                 else if (session.getState() == ConsistentSession.State.FINALIZED)

src/java/org/apache/cassandra/streaming/StreamPlan.java

@@ -151,6 +151,22 @@ public class StreamPlan
         return this;
     }
 
+    public TimeUUID planId()
+    {
+        return planId;
+    }
+
+    public StreamOperation streamOperation()
+    {
+        return streamOperation;
+    }
+
+    @VisibleForTesting
+    public List<StreamEventHandler> handlers()
+    {
+        return handlers;
+    }
+
     /**
      * Set custom StreamConnectionFactory to be used for establishing connection
      *

src/java/org/apache/cassandra/streaming/StreamSession.java

@@ -542,7 +542,9 @@ public class StreamSession
                     this.failureReason = failureReason;
 
                     sink.onClose(peer);
-                    streamResult.handleSessionComplete(this);
+                    // closed before init?
+                    if (streamResult != null)
+                        streamResult.handleSessionComplete(this);
                 }}).flatMap(ignore -> {
                     List<Future<?>> futures = new ArrayList<>();
                     // ensure aborting the tasks do not happen on the network IO thread (read: netty event loop)
@@ -959,7 +961,7 @@ public class StreamSession
             }
         }
         Collections.sort(tables);
-        int pendingThreshold = ActiveRepairService.instance.getRepairPendingCompactionRejectThreshold();
+        int pendingThreshold = ActiveRepairService.instance().getRepairPendingCompactionRejectThreshold();
         if (pendingCompactionsAfterStreaming > pendingThreshold)
         {
             logger.error("[Stream #{}] Rejecting incoming files based on pending compactions calculation " +

src/java/org/apache/cassandra/tools/nodetool/Repair.java

@@ -20,6 +20,7 @@ package org.apache.cassandra.tools.nodetool;
 import static com.google.common.collect.Lists.newArrayList;
 import static org.apache.commons.lang3.StringUtils.EMPTY;
 import io.airlift.airline.Arguments;
+import io.airlift.airline.Cli;
 import io.airlift.airline.Command;
 import io.airlift.airline.Option;
 
@@ -28,6 +29,7 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.function.Supplier;
 
 import com.google.common.collect.Sets;
 
@@ -141,39 +143,7 @@ public class Repair extends NodeToolCmd
             if ((args == null || args.isEmpty()) && ONLY_EXPLICITLY_REPAIRED.contains(keyspace))
                 continue;
 
-            Map<String, String> options = new HashMap<>();
-            RepairParallelism parallelismDegree = RepairParallelism.PARALLEL;
-            if (sequential)
-                parallelismDegree = RepairParallelism.SEQUENTIAL;
-            else if (dcParallel)
-                parallelismDegree = RepairParallelism.DATACENTER_AWARE;
-            options.put(RepairOption.PARALLELISM_KEY, parallelismDegree.getName());
-            options.put(RepairOption.PRIMARY_RANGE_KEY, Boolean.toString(primaryRange));
-            options.put(RepairOption.INCREMENTAL_KEY, Boolean.toString(!fullRepair && !(paxosOnly && getPreviewKind() == PreviewKind.NONE)));
-            options.put(RepairOption.JOB_THREADS_KEY, Integer.toString(numJobThreads));
-            options.put(RepairOption.TRACE_KEY, Boolean.toString(trace));
-            options.put(RepairOption.COLUMNFAMILIES_KEY, StringUtils.join(cfnames, ","));
-            options.put(RepairOption.PULL_REPAIR_KEY, Boolean.toString(pullRepair));
-            options.put(RepairOption.FORCE_REPAIR_KEY, Boolean.toString(force));
-            options.put(RepairOption.PREVIEW, getPreviewKind().toString());
-            options.put(RepairOption.OPTIMISE_STREAMS_KEY, Boolean.toString(optimiseStreams));
-            options.put(RepairOption.IGNORE_UNREPLICATED_KS, Boolean.toString(ignoreUnreplicatedKeyspaces));
-            options.put(RepairOption.REPAIR_PAXOS_KEY, Boolean.toString(!skipPaxos && getPreviewKind() == PreviewKind.NONE));
-            options.put(RepairOption.PAXOS_ONLY_KEY, Boolean.toString(paxosOnly && getPreviewKind() == PreviewKind.NONE));
-
-            if (!startToken.isEmpty() || !endToken.isEmpty())
-            {
-                options.put(RepairOption.RANGES_KEY, startToken + ":" + endToken);
-            }
-            if (localDC)
-            {
-                options.put(RepairOption.DATACENTERS_KEY, StringUtils.join(newArrayList(probe.getDataCenter()), ","));
-            }
-            else
-            {
-                options.put(RepairOption.DATACENTERS_KEY, StringUtils.join(specificDataCenters, ","));
-            }
-            options.put(RepairOption.HOSTS_KEY, StringUtils.join(specificHosts, ","));
+            Map<String, String> options = createOptions(probe::getDataCenter, cfnames);
             try
             {
                 probe.repairAsync(probe.output().out, keyspace, options);
@@ -183,4 +153,53 @@ public class Repair extends NodeToolCmd
             }
         }
     }
+
+    public static Map<String, String> parseOptionMap(Supplier<String> localDCOption, List<String> args)
+    {
+        List<String> realArgs = new ArrayList<>(args.size() + 1);
+        realArgs.add("repair");
+        realArgs.addAll(args);
+        Cli<Object> parser = Cli.builder("fortesting").withCommand(Repair.class).build();
+        Repair repair = (Repair) parser.parse(realArgs);
+        String[] cfnames = repair.parseOptionalTables(repair.args);
+        return repair.createOptions(localDCOption, cfnames);
+    }
+
+    private Map<String, String> createOptions(Supplier<String> localDCOption, String[] cfnames)
+    {
+        Map<String, String> options = new HashMap<>();
+        RepairParallelism parallelismDegree = RepairParallelism.PARALLEL;
+        if (sequential)
+            parallelismDegree = RepairParallelism.SEQUENTIAL;
+        else if (dcParallel)
+            parallelismDegree = RepairParallelism.DATACENTER_AWARE;
+        options.put(RepairOption.PARALLELISM_KEY, parallelismDegree.getName());
+        options.put(RepairOption.PRIMARY_RANGE_KEY, Boolean.toString(primaryRange));
+        options.put(RepairOption.INCREMENTAL_KEY, Boolean.toString(!fullRepair && !(paxosOnly && getPreviewKind() == PreviewKind.NONE)));
+        options.put(RepairOption.JOB_THREADS_KEY, Integer.toString(numJobThreads));
+        options.put(RepairOption.TRACE_KEY, Boolean.toString(trace));
+        options.put(RepairOption.COLUMNFAMILIES_KEY, StringUtils.join(cfnames, ","));
+        options.put(RepairOption.PULL_REPAIR_KEY, Boolean.toString(pullRepair));
+        options.put(RepairOption.FORCE_REPAIR_KEY, Boolean.toString(force));
+        options.put(RepairOption.PREVIEW, getPreviewKind().toString());
+        options.put(RepairOption.OPTIMISE_STREAMS_KEY, Boolean.toString(optimiseStreams));
+        options.put(RepairOption.IGNORE_UNREPLICATED_KS, Boolean.toString(ignoreUnreplicatedKeyspaces));
+        options.put(RepairOption.REPAIR_PAXOS_KEY, Boolean.toString(!skipPaxos && getPreviewKind() == PreviewKind.NONE));
+        options.put(RepairOption.PAXOS_ONLY_KEY, Boolean.toString(paxosOnly && getPreviewKind() == PreviewKind.NONE));
+
+        if (!startToken.isEmpty() || !endToken.isEmpty())
+        {
+            options.put(RepairOption.RANGES_KEY, startToken + ":" + endToken);
+        }
+        if (localDC)
+        {
+            options.put(RepairOption.DATACENTERS_KEY, StringUtils.join(newArrayList(localDCOption.get()), ","));
+        }
+        else
+        {
+            options.put(RepairOption.DATACENTERS_KEY, StringUtils.join(specificDataCenters, ","));
+        }
+        options.put(RepairOption.HOSTS_KEY, StringUtils.join(specificHosts, ","));
+        return options;
+    }
 }

src/java/org/apache/cassandra/utils/Backoff.java

@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.utils;
+
+import java.util.concurrent.TimeUnit;
+import java.util.function.DoubleSupplier;
+
+public interface Backoff
+{
+    /**
+     * @return max attempts allowed, {@code == 0} implies no retries are allowed
+     */
+    int maxAttempts();
+    long computeWaitTime(int retryCount);
+    TimeUnit unit();
+
+    enum None implements Backoff
+    {
+        INSTANCE;
+
+        @Override
+        public int maxAttempts()
+        {
+            return 0;
+        }
+
+        @Override
+        public long computeWaitTime(int retryCount)
+        {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public TimeUnit unit()
+        {
+            throw new UnsupportedOperationException();
+        }
+    }
+
+    class ExponentialBackoff implements Backoff
+    {
+        private final int maxAttempts;
+        private final long baseSleepTimeMillis;
+        private final long maxSleepMillis;
+        private final DoubleSupplier randomSource;
+
+        public ExponentialBackoff(int maxAttempts, long baseSleepTimeMillis, long maxSleepMillis, DoubleSupplier randomSource)
+        {
+            this.maxAttempts = maxAttempts;
+            this.baseSleepTimeMillis = baseSleepTimeMillis;
+            this.maxSleepMillis = maxSleepMillis;
+            this.randomSource = randomSource;
+        }
+
+        @Override
+        public int maxAttempts()
+        {
+            return maxAttempts;
+        }
+
+        @Override
+        public long computeWaitTime(int retryCount)
+        {
+            long baseTimeMillis = baseSleepTimeMillis * (1L << retryCount);
+            // it's possible that this overflows, so fall back to max;
+            if (baseTimeMillis <= 0)
+                baseTimeMillis = maxSleepMillis;
+            // now make sure this is capped to target max
+            baseTimeMillis = Math.min(baseTimeMillis, maxSleepMillis);
+
+            return (long) (baseTimeMillis * (randomSource.getAsDouble() + 0.5));
+        }
+
+        @Override
+        public TimeUnit unit()
+        {
+            return TimeUnit.MILLISECONDS;
+        }
+    }
+}

src/java/org/apache/cassandra/utils/Clock.java

@@ -87,6 +87,11 @@ public interface Clock
             INITIALIZE_MESSAGE = null;
         }
 
+        public static Clock clock()
+        {
+            return instance;
+        }
+
         /**
          * Semantically equivalent to {@link System#nanoTime()}
          */
@@ -133,6 +138,11 @@ public interface Clock
      */
     public long currentTimeMillis();
 
+    public default long nowInSeconds()
+    {
+        return currentTimeMillis() / 1000L;
+    }
+
     @Intercept
     public static void waitUntil(long deadlineNanos) throws InterruptedException
     {

src/java/org/apache/cassandra/utils/FailingBiConsumer.java

@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.utils;
+
+import java.util.function.BiConsumer;
+
+public interface FailingBiConsumer<A, B> extends BiConsumer<A, B>
+{
+    void acceptOrFail(A a, B b) throws Throwable;
+
+    @Override
+    default void accept(A a, B b)
+    {
+        try
+        {
+            acceptOrFail(a, b);
+        }
+        catch (Throwable e)
+        {
+            throw Throwables.throwAsUncheckedException(e);
+        }
+    }
+
+    static <A, B> FailingBiConsumer<A, B> orFail(FailingBiConsumer<A, B> fn)
+    {
+        return fn;
+    }
+}

src/java/org/apache/cassandra/utils/MerkleTree.java

@@ -1015,7 +1015,7 @@ public class MerkleTree
         default void serialize(DataOutputPlus out, int version) throws IOException
         {
             byte[] hash = hash();
-            assert hash.length == HASH_SIZE;
+            assert hash.length == HASH_SIZE: String.format("Expected hash length to be %d, but given %d", HASH_SIZE, hash.length);
 
             out.writeByte(Leaf.IDENT);
 

test/distributed/org/apache/cassandra/distributed/impl/Instance.java

@@ -747,7 +747,7 @@ public class Instance extends IsolatedExecutor implements IInvokableInstance
                     FBUtilities.getBroadcastAddressAndPort().getPort() != broadcastAddress().getPort())
                     throw new IllegalStateException(String.format("%s != %s", FBUtilities.getBroadcastAddressAndPort(), broadcastAddress()));
 
-                ActiveRepairService.instance.start();
+                ActiveRepairService.instance().start();
                 StreamManager.instance.start();
 
                 PaxosState.startAutoRepairs();
@@ -898,7 +898,7 @@ public class Instance extends IsolatedExecutor implements IInvokableInstance
                                 () -> DiagnosticSnapshotService.instance.shutdownAndWait(1L, MINUTES),
                                 () -> SSTableReader.shutdownBlocking(1L, MINUTES),
                                 () -> shutdownAndWait(Collections.singletonList(ActiveRepairService.repairCommandExecutor())),
-                                () -> ActiveRepairService.instance.shutdownNowAndWait(1L, MINUTES),
+                                () -> ActiveRepairService.instance().shutdownNowAndWait(1L, MINUTES),
                                 () -> SnapshotManager.shutdownAndWait(1L, MINUTES)
             );
 

test/distributed/org/apache/cassandra/distributed/mock/nodetool/InternalNodeProbe.java

@@ -79,7 +79,7 @@ public class InternalNodeProbe extends NodeProbe
         gcProxy = new GCInspector();
         gossProxy = Gossiper.instance;
         bmProxy = BatchlogManager.instance;
-        arsProxy = ActiveRepairService.instance;
+        arsProxy = ActiveRepairService.instance();
         memProxy = ManagementFactory.getMemoryMXBean();
         runtimeProxy = ManagementFactory.getRuntimeMXBean();
     }

test/distributed/org/apache/cassandra/distributed/test/ClearSnapshotTest.java

@@ -100,7 +100,7 @@ public class ClearSnapshotTest extends TestBaseImpl
             long activeRepairs;
             do
             {
-                activeRepairs = cluster.get(1).callOnInstance(() -> ActiveRepairService.instance.parentRepairSessionCount());
+                activeRepairs = cluster.get(1).callOnInstance(() -> ActiveRepairService.instance().parentRepairSessionCount());
                 Thread.sleep(50);
             }
             while (activeRepairs < 35);

test/distributed/org/apache/cassandra/distributed/test/IncRepairAdminTest.java

@@ -149,14 +149,14 @@ public class IncRepairAdminTest extends TestBaseImpl
                 ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore("tbl");
                 Range<Token> range = new Range<>(cfs.metadata().partitioner.getMinimumToken(),
                                                  cfs.metadata().partitioner.getRandomToken());
-                ActiveRepairService.instance.registerParentRepairSession(sessionId,
-                                                                         InetAddressAndPort.getByAddress(coordinator.getAddress()),
-                                                                         Lists.newArrayList(cfs),
-                                                                         Sets.newHashSet(range),
-                                                                         true,
-                                                                         currentTimeMillis(),
-                                                                         true,
-                                                                         PreviewKind.NONE);
+                ActiveRepairService.instance().registerParentRepairSession(sessionId,
+                                                                           InetAddressAndPort.getByAddress(coordinator.getAddress()),
+                                                                           Lists.newArrayList(cfs),
+                                                                           Sets.newHashSet(range),
+                                                                           true,
+                                                                           currentTimeMillis(),
+                                                                           true,
+                                                                           PreviewKind.NONE);
                 LocalSessionAccessor.prepareUnsafe(sessionId,
                                                    InetAddressAndPort.getByAddress(coordinator.getAddress()),
                                                    participants.stream().map(participant -> InetAddressAndPort.getByAddress(participant.getAddress())).collect(Collectors.toSet()));

test/distributed/org/apache/cassandra/distributed/test/IncRepairCoordinatorErrorTest.java

@@ -48,7 +48,7 @@ public class IncRepairCoordinatorErrorTest extends TestBaseImpl
             cluster.get(1).nodetoolResult("repair", KEYSPACE).asserts().success();
             TimeUUID result = (TimeUUID) cluster.get(1).executeInternal("select parent_id from system_distributed.repair_history")[0][0];
             cluster.get(3).runOnInstance(() -> {
-                ActiveRepairService.instance.failSession(result.toString(), true);
+                ActiveRepairService.instance().failSession(result.toString(), true);
             });
             assertThat(cluster.get(1).logs().watchFor("Can't transition endpoints .* to FAILED").getResult()).isNotEmpty();
         }

test/distributed/org/apache/cassandra/distributed/test/OptimiseStreamsRepairTest.java

@@ -39,6 +39,7 @@ import net.bytebuddy.ByteBuddy;
 import net.bytebuddy.dynamic.loading.ClassLoadingStrategy;
 import net.bytebuddy.implementation.MethodDelegation;
 import net.bytebuddy.implementation.bind.annotation.SuperCall;
+import org.apache.cassandra.repair.SharedContext;
 import org.apache.cassandra.dht.Range;
 import org.apache.cassandra.dht.Token;
 import org.apache.cassandra.distributed.Cluster;
@@ -114,7 +115,8 @@ public class OptimiseStreamsRepairTest extends TestBaseImpl
                            .load(cl, ClassLoadingStrategy.Default.INJECTION);
         }
 
-        public static List<SyncTask> createOptimisedSyncingSyncTasks(RepairJobDesc desc,
+        public static List<SyncTask> createOptimisedSyncingSyncTasks(SharedContext ctx,
+                                                                     RepairJobDesc desc,
                                                                      List<TreeResponse> trees,
                                                                      InetAddressAndPort local,
                                                                      Predicate<InetAddressAndPort> isTransient,

test/distributed/org/apache/cassandra/distributed/test/PaxosRepair2Test.java

@@ -161,7 +161,7 @@ public class PaxosRepair2Test extends TestBaseImpl
                 {
                     throw new AssertionError(e);
                 }
-                Pair<ActiveRepairService.ParentRepairStatus, List<String>> status = ActiveRepairService.instance.getRepairStatus(cmd);
+                Pair<ActiveRepairService.ParentRepairStatus, List<String>> status = ActiveRepairService.instance().getRepairStatus(cmd);
                 if (status == null)
                     continue;
 

test/distributed/org/apache/cassandra/distributed/test/PaxosRepairTest.java

@@ -175,7 +175,7 @@ public class PaxosRepairTest extends TestBaseImpl
                 {
                     throw new AssertionError(e);
                 }
-                Pair<ActiveRepairService.ParentRepairStatus, List<String>> status = ActiveRepairService.instance.getRepairStatus(cmd);
+                Pair<ActiveRepairService.ParentRepairStatus, List<String>> status = ActiveRepairService.instance().getRepairStatus(cmd);
                 if (status == null)
                     continue;
 

test/distributed/org/apache/cassandra/distributed/test/RepairErrorsTest.java

@@ -201,7 +201,7 @@ public class RepairErrorsTest extends TestBaseImpl
     private void assertNoActiveRepairSessions(IInvokableInstance instance)
     {
         // Make sure we've cleaned up local sessions:
-        Integer sessions = instance.callOnInstance(() -> ActiveRepairService.instance.sessionCount());
+        Integer sessions = instance.callOnInstance(() -> ActiveRepairService.instance().sessionCount());
         assertEquals(0, sessions.intValue());
     }
 

test/unit/accord/utils/DefaultRandom.java

@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package accord.utils;
+
+import java.util.Random;
+
+public class DefaultRandom implements RandomSource
+{
+    private final Random delegate;
+    public DefaultRandom()
+    {
+        this.delegate = new Random();
+    }
+
+    public DefaultRandom(long seed)
+    {
+        this.delegate = new Random(seed);
+    }
+
+    @Override
+    public void nextBytes(byte[] bytes)
+    {
+        delegate.nextBytes(bytes);
+    }
+
+    @Override
+    public boolean nextBoolean()
+    {
+        return delegate.nextBoolean();
+    }
+
+    @Override
+    public int nextInt()
+    {
+        return delegate.nextInt();
+    }
+
+    @Override
+    public long nextLong()
+    {
+        return delegate.nextLong();
+    }
+
+    @Override
+    public float nextFloat()
+    {
+        return delegate.nextFloat();
+    }
+
+    @Override
+    public double nextDouble()
+    {
+        return delegate.nextDouble();
+    }
+
+    @Override
+    public double nextGaussian()
+    {
+        return delegate.nextGaussian();
+    }
+
+    @Override
+    public void setSeed(long seed)
+    {
+        delegate.setSeed(seed);
+    }
+
+    @Override
+    public DefaultRandom fork() {
+        return new DefaultRandom(nextLong());
+    }
+
+    @Override
+    public Random asJdkRandom()
+    {
+        return delegate;
+    }
+}

test/unit/accord/utils/Gen.java

@@ -0,0 +1,179 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package accord.utils;
+
+import java.util.function.BiFunction;
+import java.util.function.Function;
+import java.util.function.IntPredicate;
+import java.util.function.IntSupplier;
+import java.util.function.LongPredicate;
+import java.util.function.LongSupplier;
+import java.util.function.Predicate;
+import java.util.function.Supplier;
+import java.util.function.ToIntFunction;
+import java.util.function.ToLongFunction;
+import java.util.stream.IntStream;
+import java.util.stream.LongStream;
+import java.util.stream.Stream;
+
+public interface Gen<A> {
+    /**
+     * For cases where method handles isn't able to detect the proper type, this method acts as a cast
+     * to inform the compiler of the desired type.
+     */
+    static <A> Gen<A> of(Gen<A> fn)
+    {
+        return fn;
+    }
+
+    A next(RandomSource random);
+
+    default <B> Gen<B> map(Function<? super A, ? extends B> fn)
+    {
+        return r -> fn.apply(this.next(r));
+    }
+
+    default <B> Gen<B> map(BiFunction<RandomSource, ? super A, ? extends B> fn)
+    {
+        return r -> fn.apply(r, this.next(r));
+    }
+
+    default IntGen mapToInt(ToIntFunction<A> fn)
+    {
+        return r -> fn.applyAsInt(next(r));
+    }
+
+    default LongGen mapToLong(ToLongFunction<A> fn)
+    {
+        return r -> fn.applyAsLong(next(r));
+    }
+
+    default <B> Gen<B> flatMap(Function<? super A, Gen<? extends B>> mapper)
+    {
+        return rs -> mapper.apply(this.next(rs)).next(rs);
+    }
+
+    default <B> Gen<B> flatMap(BiFunction<RandomSource, ? super A, Gen<? extends B>> mapper)
+    {
+        return rs -> mapper.apply(rs, this.next(rs)).next(rs);
+    }
+
+    default Gen<A> filter(Predicate<A> fn)
+    {
+        Gen<A> self = this;
+        return r -> {
+            A value;
+            do {
+                value = self.next(r);
+            }
+            while (!fn.test(value));
+            return value;
+        };
+    }
+
+    default Supplier<A> asSupplier(RandomSource rs)
+    {
+        return () -> next(rs);
+    }
+
+    default Stream<A> asStream(RandomSource rs)
+    {
+        return Stream.generate(() -> next(rs));
+    }
+
+    interface IntGen extends Gen<Integer>
+    {
+        int nextInt(RandomSource random);
+
+        @Override
+        default Integer next(RandomSource random)
+        {
+            return nextInt(random);
+        }
+
+        default Gen.IntGen filterInt(IntPredicate fn)
+        {
+            return rs -> {
+                int value;
+                do
+                {
+                    value = nextInt(rs);
+                }
+                while (!fn.test(value));
+                return value;
+            };
+        }
+
+        @Override
+        default Gen.IntGen filter(Predicate<Integer> fn)
+        {
+            return filterInt(i -> fn.test(i));
+        }
+
+        default IntSupplier asIntSupplier(RandomSource rs)
+        {
+            return () -> nextInt(rs);
+        }
+
+        default IntStream asIntStream(RandomSource rs)
+        {
+            return IntStream.generate(() -> nextInt(rs));
+        }
+    }
+
+    interface LongGen extends Gen<Long>
+    {
+        long nextLong(RandomSource random);
+
+        @Override
+        default Long next(RandomSource random)
+        {
+            return nextLong(random);
+        }
+
+        default Gen.LongGen filterLong(LongPredicate fn)
+        {
+            return rs -> {
+                long value;
+                do
+                {
+                    value = nextLong(rs);
+                }
+                while (!fn.test(value));
+                return value;
+            };
+        }
+
+        @Override
+        default Gen.LongGen filter(Predicate<Long> fn)
+        {
+            return filterLong(i -> fn.test(i));
+        }
+
+        default LongSupplier asLongSupplier(RandomSource rs)
+        {
+            return () -> nextLong(rs);
+        }
+
+        default LongStream asLongStream(RandomSource rs)
+        {
+            return LongStream.generate(() -> nextLong(rs));
+        }
+    }
+}

test/unit/accord/utils/Gens.java

@@ -0,0 +1,576 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package accord.utils;
+
+import java.lang.reflect.Array;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.EnumMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.NavigableSet;
+import java.util.Objects;
+import java.util.Set;
+import java.util.function.Function;
+import java.util.function.Supplier;
+import java.util.stream.Collectors;
+
+
+public class Gens {
+    private Gens() {
+    }
+
+    public static <T> Gen<T> constant(T constant)
+    {
+        return ignore -> constant;
+    }
+
+    public static <T> Gen<T> constant(Supplier<T> constant)
+    {
+        return ignore -> constant.get();
+    }
+
+    public static <T> Gen<T> pick(T... ts)
+    {
+        return pick(Arrays.asList(ts));
+    }
+
+    public static <T> Gen<T> pick(List<T> ts)
+    {
+        Gen.IntGen offset = ints().between(0, ts.size() - 1);
+        return rs -> ts.get(offset.nextInt(rs));
+    }
+
+    public static <T extends Comparable<T>> Gen<T> pick(Set<T> set)
+    {
+        List<T> list = new ArrayList<>(set);
+        // Non-ordered sets may have different iteration order on different environments, which would make a seed produce different histories!
+        // To avoid such a problem, make sure to apply a deterministic function (sort).
+        if (!(set instanceof NavigableSet))
+            list.sort(Comparator.naturalOrder());
+        return pick(list);
+    }
+
+    public static <T> Gen<T> pick(Map<T, Integer> values)
+    {
+        if (values == null || values.isEmpty())
+            throw new IllegalArgumentException("values is empty");
+        double totalWeight = values.values().stream().mapToDouble(Integer::intValue).sum();
+        List<Weight<T>> list = values.entrySet().stream().map(e -> new Weight<>(e.getKey(), e.getValue())).collect(Collectors.toList());
+        Collections.sort(list);
+        return rs -> {
+            double value = rs.nextDouble() * totalWeight;
+            for (Weight<T> w : list)
+            {
+                value -= w.weight;
+                if (value <= 0)
+                    return w.value;
+            }
+            return list.get(list.size() - 1).value;
+        };
+    }
+
+    public static Gen<char[]> charArray(Gen.IntGen sizes, char[] domain)
+    {
+        return charArray(sizes, domain, (a, b) -> true);
+    }
+
+    public interface IntCharBiPredicate
+    {
+        boolean test(int a, char b);
+    }
+
+    public static Gen<char[]> charArray(Gen.IntGen sizes, char[] domain, IntCharBiPredicate fn)
+    {
+        Gen.IntGen indexGen = ints().between(0, domain.length - 1);
+        return rs -> {
+            int size = sizes.nextInt(rs);
+            char[] is = new char[size];
+            for (int i = 0; i != size; i++)
+            {
+                char c;
+                do
+                {
+                    c = domain[indexGen.nextInt(rs)];
+                }
+                while (!fn.test(i, c));
+                is[i] = c;
+            }
+            return is;
+        };
+    }
+
+    public static Gen<RandomSource> random() {
+        return r -> r;
+    }
+
+    public static BooleanDSL bools()
+    {
+        return new BooleanDSL();
+    }
+
+    public static IntDSL ints()
+    {
+        return new IntDSL();
+    }
+
+    public static LongDSL longs() {
+        return new LongDSL();
+    }
+
+    public static <T> ListDSL<T> lists(Gen<T> fn) {
+        return new ListDSL<>(fn);
+    }
+
+    public static <T> ArrayDSL<T> arrays(Class<T> type, Gen<T> fn) {
+        return new ArrayDSL<>(type, fn);
+    }
+
+    public static IntArrayDSL arrays(Gen.IntGen fn) {
+        return new IntArrayDSL(fn);
+    }
+
+    public static LongArrayDSL arrays(Gen.LongGen fn) {
+        return new LongArrayDSL(fn);
+    }
+
+    public static EnumDSL enums()
+    {
+        return new EnumDSL();
+    }
+
+    public static StringDSL strings()
+    {
+        return new StringDSL();
+    }
+
+    public static class BooleanDSL
+    {
+        public Gen<Boolean> all()
+        {
+            return RandomSource::nextBoolean;
+        }
+
+        public Gen<Boolean> runs(double ratio, int maxRuns)
+        {
+            Invariants.checkArgument(ratio > 0 && ratio <= 1, "Expected %d to be larger than 0 and <= 1", ratio);
+            double lower = ratio * .8;
+            double upper = ratio * 1.2;
+            return new Gen<>() {
+                // run represents how many consecutaive true values should be returned; -1 implies no active "run" exists
+                private int run = -1;
+                private long falseCount = 0, trueCount = 0;
+                @Override
+                public Boolean next(RandomSource rs)
+                {
+                    if (run != -1)
+                    {
+                        run--;
+                        trueCount++;
+                        return true;
+                    }
+                    double currentRatio = trueCount / (double) (falseCount + trueCount);
+                    if (currentRatio < lower)
+                    {
+                        // not enough true
+                        trueCount++;
+                        return true;
+                    }
+                    if (currentRatio > upper)
+                    {
+                        // not enough false
+                        falseCount++;
+                        return false;
+                    }
+                    if (rs.decide(ratio))
+                    {
+                        run = rs.nextInt(maxRuns);
+                        run--;
+                        trueCount++;
+                        return true;
+                    }
+                    falseCount++;
+                    return false;
+                }
+            };
+        }
+    }
+
+    public static class IntDSL
+    {
+        public Gen.IntGen of(int value)
+        {
+            return r -> value;
+        }
+
+        public Gen.IntGen all()
+        {
+            return RandomSource::nextInt;
+        }
+
+        public Gen.IntGen between(int min, int max)
+        {
+            Invariants.checkArgument(max >= min, "max (%d) < min (%d)", max, min);
+            if (min == max)
+                return of(min);
+            // since bounds is exclusive, if max == max_value unable to do +1 to include... so will return a gen
+            // that does not include
+            if (max == Integer.MAX_VALUE)
+                return r -> r.nextInt(min, max);
+            return r -> r.nextInt(min, max + 1);
+        }
+    }
+
+    public static class LongDSL {
+        public Gen.LongGen of(long value)
+        {
+            return r -> value;
+        }
+
+        public Gen.LongGen all() {
+            return RandomSource::nextLong;
+        }
+
+        public Gen.LongGen between(long min, long max) {
+            Invariants.checkArgument(max >= min);
+            if (min == max)
+                return of(min);
+            // since bounds is exclusive, if max == max_value unable to do +1 to include... so will return a gen
+            // that does not include
+            if (max == Long.MAX_VALUE)
+                return r -> r.nextLong(min, max);
+            return r -> r.nextLong(min, max + 1);
+        }
+    }
+
+    public static class EnumDSL
+    {
+        public <T extends Enum<T>> Gen<T> all(Class<T> klass)
+        {
+            return pick(klass.getEnumConstants());
+        }
+
+        public <T extends Enum<T>> Gen<T> allWithWeights(Class<T> klass, int... weights)
+        {
+            T[] constants = klass.getEnumConstants();
+            if (constants.length != weights.length)
+                throw new IllegalArgumentException(String.format("Total number of weights (%s) does not match the enum (%s)", Arrays.toString(weights), Arrays.toString(constants)));
+            Map<T, Integer> values = new EnumMap<>(klass);
+            for (int i = 0; i < constants.length; i++)
+                values.put(constants[i], weights[i]);
+            return pick(values);
+        }
+    }
+
+    public static class StringDSL
+    {
+        public Gen<String> of(Gen.IntGen sizes, char[] domain)
+        {
+            // note, map is overloaded so String::new is ambugious to javac, so need a lambda here
+            return charArray(sizes, domain).map(c -> new String(c));
+        }
+
+        public SizeBuilder<String> of(char[] domain)
+        {
+            return new SizeBuilder<>(sizes -> of(sizes, domain));
+        }
+
+        public Gen<String> of(Gen.IntGen sizes, char[] domain, IntCharBiPredicate fn)
+        {
+            // note, map is overloaded so String::new is ambugious to javac, so need a lambda here
+            return charArray(sizes, domain, fn).map(c -> new String(c));
+        }
+
+        public SizeBuilder<String> of(char[] domain, IntCharBiPredicate fn)
+        {
+            return new SizeBuilder<>(sizes -> of(sizes, domain, fn));
+        }
+
+        public Gen<String> all(Gen.IntGen sizes)
+        {
+            return betweenCodePoints(sizes, Character.MIN_CODE_POINT, Character.MAX_CODE_POINT);
+        }
+
+        public SizeBuilder<String> all()
+        {
+            return new SizeBuilder<>(this::all);
+        }
+
+        public Gen<String> ascii(Gen.IntGen sizes)
+        {
+            return betweenCodePoints(sizes, 0, 127);
+        }
+
+        public SizeBuilder<String> ascii()
+        {
+            return new SizeBuilder<>(this::ascii);
+        }
+
+        public Gen<String> betweenCodePoints(Gen.IntGen sizes, int min, int max)
+        {
+            Gen.IntGen codePointGen = ints().between(min, max).filter(Character::isDefined);
+            return rs -> {
+                int[] array = new int[sizes.nextInt(rs)];
+                for (int i = 0; i < array.length; i++)
+                    array[i] = codePointGen.nextInt(rs);
+                return new String(array, 0, array.length);
+            };
+        }
+
+        public SizeBuilder<String> betweenCodePoints(int min, int max)
+        {
+            return new SizeBuilder<>(sizes -> betweenCodePoints(sizes, min, max));
+        }
+    }
+
+    public static class SizeBuilder<T>
+    {
+        private final Function<Gen.IntGen, Gen<T>> fn;
+
+        public SizeBuilder(Function<Gen.IntGen, Gen<T>> fn)
+        {
+            this.fn = fn;
+        }
+
+        public Gen<T> ofLength(int fixed)
+        {
+            return ofLengthBetween(fixed, fixed);
+        }
+
+        public Gen<T> ofLengthBetween(int min, int max)
+        {
+            return fn.apply(ints().between(min, max));
+        }
+    }
+
+    public static class ListDSL<T> implements BaseSequenceDSL<ListDSL<T>, List<T>> {
+        private final Gen<T> fn;
+
+        public ListDSL(Gen<T> fn) {
+            this.fn = Objects.requireNonNull(fn);
+        }
+
+        @Override
+        public ListDSL<T> unique()
+        {
+            return new ListDSL<>(new GenReset<>(fn));
+        }
+
+        @Override
+        public Gen<List<T>> ofSizeBetween(int minSize, int maxSize) {
+            Gen.IntGen sizeGen = ints().between(minSize, maxSize);
+            return r ->
+            {
+                Reset.tryReset(fn);
+                int size = sizeGen.nextInt(r);
+                List<T> list = new ArrayList<>(size);
+                for (int i = 0; i < size; i++)
+                    list.add(fn.next(r));
+                return list;
+            };
+        }
+    }
+
+    public static class ArrayDSL<T> implements BaseSequenceDSL<ArrayDSL<T>, T[]> {
+        private final Class<T> type;
+        private final Gen<T> fn;
+
+        public ArrayDSL(Class<T> type, Gen<T> fn) {
+            this.type = Objects.requireNonNull(type);
+            this.fn = Objects.requireNonNull(fn);
+        }
+
+        @Override
+        public ArrayDSL<T> unique()
+        {
+            return new ArrayDSL<>(type, new GenReset<>(fn));
+        }
+
+        @Override
+        public Gen<T[]> ofSizeBetween(int minSize, int maxSize) {
+            Gen.IntGen sizeGen = ints().between(minSize, maxSize);
+            return r ->
+            {
+                Reset.tryReset(fn);
+                int size = sizeGen.nextInt(r);
+                T[] list = (T[]) Array.newInstance(type, size);
+                for (int i = 0; i < size; i++)
+                    list[i] = fn.next(r);
+                return list;
+            };
+        }
+    }
+
+    public static class IntArrayDSL implements BaseSequenceDSL<IntArrayDSL, int[]> {
+        private final Gen.IntGen fn;
+
+        public IntArrayDSL(Gen.IntGen fn) {
+            this.fn = Objects.requireNonNull(fn);
+        }
+
+        @Override
+        public IntArrayDSL unique()
+        {
+            return new IntArrayDSL(new IntGenReset(fn));
+        }
+
+        @Override
+        public Gen<int[]> ofSizeBetween(int minSize, int maxSize) {
+            Gen.IntGen sizeGen = ints().between(minSize, maxSize);
+            return r ->
+            {
+                int size = sizeGen.nextInt(r);
+                int[] list = new int[size];
+                for (int i = 0; i < size; i++)
+                    list[i] = fn.nextInt(r);
+                return list;
+            };
+        }
+    }
+
+    public static class LongArrayDSL implements BaseSequenceDSL<LongArrayDSL, long[]> {
+        private final Gen.LongGen fn;
+
+        public LongArrayDSL(Gen.LongGen fn) {
+            this.fn = Objects.requireNonNull(fn);
+        }
+
+        @Override
+        public LongArrayDSL unique()
+        {
+            return new LongArrayDSL(new LongGenReset(fn));
+        }
+
+        @Override
+        public Gen<long[]> ofSizeBetween(int minSize, int maxSize) {
+            Gen.IntGen sizeGen = ints().between(minSize, maxSize);
+            return r ->
+            {
+                int size = sizeGen.nextInt(r);
+                long[] list = new long[size];
+                for (int i = 0; i < size; i++)
+                    list[i] = fn.nextLong(r);
+                return list;
+            };
+        }
+    }
+
+    public interface BaseSequenceDSL<A extends BaseSequenceDSL<A, B>, B>
+    {
+        A unique();
+
+        Gen<B> ofSizeBetween(int min, int max);
+
+        default Gen<B> ofSize(int size) {
+            return ofSizeBetween(size, size);
+        }
+    }
+
+    private interface Reset {
+        static void tryReset(Object o)
+        {
+            if (o instanceof Reset)
+                ((Reset) o).reset();
+        }
+
+        void reset();
+    }
+
+    private static class GenReset<T> implements Gen<T>, Reset
+    {
+        private final Set<T> seen = new HashSet<>();
+        private final Gen<T> fn;
+
+        private GenReset(Gen<T> fn)
+        {
+            this.fn = fn;
+        }
+
+        @Override
+        public T next(RandomSource random)
+        {
+            T value;
+            while (!seen.add((value = fn.next(random)))) {}
+            return value;
+        }
+
+        @Override
+        public void reset()
+        {
+            seen.clear();
+        }
+    }
+
+    private static class IntGenReset implements Gen.IntGen, Reset
+    {
+        private final GenReset<Integer> base;
+
+        private IntGenReset(Gen.IntGen fn)
+        {
+            this.base = new GenReset<>(fn);
+        }
+        @Override
+        public int nextInt(RandomSource random) {
+            return base.next(random);
+        }
+
+        @Override
+        public void reset() {
+            base.reset();
+        }
+    }
+
+    private static class LongGenReset implements Gen.LongGen, Reset
+    {
+        private final GenReset<Long> base;
+
+        private LongGenReset(Gen.LongGen fn)
+        {
+            this.base = new GenReset<>(fn);
+        }
+        @Override
+        public long nextLong(RandomSource random) {
+            return base.next(random);
+        }
+
+        @Override
+        public void reset() {
+            base.reset();
+        }
+    }
+
+    private static class Weight<T> implements Comparable<Weight<T>>
+    {
+        private final T value;
+        private final double weight;
+
+        private Weight(T value, double weight) {
+            this.value = value;
+            this.weight = weight;
+        }
+
+        @Override
+        public int compareTo(Weight<T> o) {
+            return Double.compare(weight, o.weight);
+        }
+    }
+}

test/unit/accord/utils/Invariants.java

@@ -0,0 +1,327 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package accord.utils;
+
+import net.nicoulaj.compilecommand.annotations.Inline;
+
+import javax.annotation.Nullable;
+import java.util.function.Predicate;
+
+import static java.lang.String.format;
+
+public class Invariants
+{
+    private static final boolean PARANOID = true;
+    private static final boolean DEBUG = true;
+
+    public static boolean isParanoid()
+    {
+        return PARANOID;
+    }
+    public static boolean debug()
+    {
+        return DEBUG;
+    }
+
+    private static void illegalState(String msg)
+    {
+        throw new IllegalStateException(msg);
+    }
+
+    private static void illegalState()
+    {
+        illegalState(null);
+    }
+
+    private static void illegalArgument(String msg)
+    {
+        throw new IllegalArgumentException(msg);
+    }
+
+
+    private static void illegalArgument()
+    {
+        illegalArgument(null);
+    }
+
+    public static <T1, T2 extends T1> T2 checkType(T1 cast)
+    {
+        return (T2)cast;
+    }
+
+    public static <T1, T2 extends T1> T2 checkType(Class<T2> to, T1 cast)
+    {
+        if (cast != null && !to.isInstance(cast))
+            illegalState();
+        return (T2)cast;
+    }
+
+    public static <T1, T2 extends T1> T2 checkType(Class<T2> to, T1 cast, String msg)
+    {
+        if (cast != null && !to.isInstance(cast))
+            illegalState(msg);
+        return (T2)cast;
+    }
+
+    public static void paranoid(boolean condition)
+    {
+        if (PARANOID && !condition)
+            illegalState();
+    }
+
+    public static void checkState(boolean condition)
+    {
+        if (!condition)
+            illegalState();
+    }
+
+    public static void checkState(boolean condition, String msg)
+    {
+        if (!condition)
+            illegalState(msg);
+    }
+
+    public static void checkState(boolean condition, String fmt, int p1)
+    {
+        if (!condition)
+            illegalState(format(fmt, p1));
+    }
+
+    public static void checkState(boolean condition, String fmt, int p1, int p2)
+    {
+        if (!condition)
+            illegalState(format(fmt, p1, p2));
+    }
+
+    public static void checkState(boolean condition, String fmt, long p1)
+    {
+        if (!condition)
+            illegalState(format(fmt, p1));
+    }
+
+    public static void checkState(boolean condition, String fmt, long p1, long p2)
+    {
+        if (!condition)
+            illegalState(format(fmt, p1, p2));
+    }
+
+    public static void checkState(boolean condition, String fmt, @Nullable Object p1)
+    {
+        if (!condition)
+            illegalState(format(fmt, p1));
+    }
+
+    public static void checkState(boolean condition, String fmt, @Nullable Object p1, @Nullable Object p2)
+    {
+        if (!condition)
+            illegalState(format(fmt, p1, p2));
+    }
+
+    public static void checkState(boolean condition, String fmt, Object... args)
+    {
+        if (!condition)
+            illegalState(format(fmt, args));
+    }
+
+    public static <T> T nonNull(T param)
+    {
+        if (param == null)
+            throw new NullPointerException();
+        return param;
+    }
+
+    public static <T> T nonNull(T param, String fmt, Object... args)
+    {
+        if (param == null)
+            throw new NullPointerException(format(fmt, args));
+        return param;
+    }
+
+    public static int isNatural(int input)
+    {
+        if (input < 0)
+            illegalState();
+        return input;
+    }
+
+    public static long isNatural(long input)
+    {
+        if (input < 0)
+            illegalState();
+        return input;
+    }
+
+    public static void checkArgument(boolean condition)
+    {
+        if (!condition)
+            illegalArgument();
+    }
+
+    public static void checkArgument(boolean condition, String msg)
+    {
+        if (!condition)
+            illegalArgument(msg);
+    }
+
+    public static void checkArgument(boolean condition, String fmt, int p1)
+    {
+        if (!condition)
+            illegalArgument(format(fmt, p1));
+    }
+
+    public static void checkArgument(boolean condition, String fmt, int p1, int p2)
+    {
+        if (!condition)
+            illegalArgument(format(fmt, p1, p2));
+    }
+
+    public static void checkArgument(boolean condition, String fmt, long p1)
+    {
+        if (!condition)
+            illegalArgument(format(fmt, p1));
+    }
+
+    public static void checkArgument(boolean condition, String fmt, long p1, long p2)
+    {
+        if (!condition)
+            illegalArgument(format(fmt, p1, p2));
+    }
+
+    public static void checkArgument(boolean condition, String fmt, @Nullable Object p1)
+    {
+        if (!condition)
+            illegalArgument(format(fmt, p1));
+    }
+
+    public static void checkArgument(boolean condition, String fmt, @Nullable Object p1, @Nullable Object p2)
+    {
+        if (!condition)
+            illegalArgument(format(fmt, p1, p2));
+    }
+
+    public static void checkArgument(boolean condition, String fmt, Object... args)
+    {
+        if (!condition)
+            illegalArgument(format(fmt, args));
+    }
+
+    public static <T> T checkArgument(T param, boolean condition)
+    {
+        if (!condition)
+            illegalArgument();
+        return param;
+    }
+
+    public static <T> T checkArgument(T param, boolean condition, String msg)
+    {
+        if (!condition)
+            illegalArgument(msg);
+        return param;
+    }
+
+    public static <T> T checkArgument(T param, boolean condition, String fmt, int p1)
+    {
+        if (!condition)
+            illegalArgument(format(fmt, p1));
+        return param;
+    }
+
+    public static <T> T checkArgument(T param, boolean condition, String fmt, int p1, int p2)
+    {
+        if (!condition)
+            illegalArgument(format(fmt, p1, p2));
+        return param;
+    }
+
+    public static <T> T checkArgument(T param, boolean condition, String fmt, long p1)
+    {
+        if (!condition)
+            illegalArgument(format(fmt, p1));
+        return param;
+    }
+
+    public static <T> T checkArgument(T param, boolean condition, String fmt, long p1, long p2)
+    {
+        if (!condition)
+            illegalArgument(format(fmt, p1, p2));
+        return param;
+    }
+
+    public static <T> T checkArgument(T param, boolean condition, String fmt, @Nullable Object p1)
+    {
+        if (!condition)
+            illegalArgument(format(fmt, p1));
+        return param;
+    }
+
+    public static <T> T checkArgument(T param, boolean condition, String fmt, @Nullable Object p1, @Nullable Object p2)
+    {
+        if (!condition)
+            illegalArgument(format(fmt, p1, p2));
+        return param;
+    }
+
+    public static <T> T checkArgument(T param, boolean condition, String fmt, Object... args)
+    {
+        if (!condition)
+            illegalArgument(format(fmt, args));
+        return param;
+    }
+
+    @Inline
+    public static <T> T checkArgument(T param, Predicate<T> condition)
+    {
+        if (!condition.test(param))
+            illegalArgument();
+        return param;
+    }
+
+    @Inline
+    public static <T> T checkArgument(T param, Predicate<T> condition, String msg)
+    {
+        if (!condition.test(param))
+            illegalArgument(msg);
+        return param;
+    }
+
+    public static <O> O cast(Object o, Class<O> klass)
+    {
+        try
+        {
+            return klass.cast(o);
+        }
+        catch (ClassCastException e)
+        {
+            throw new IllegalArgumentException(format("Unable to cast %s to %s", o, klass.getName()));
+        }
+    }
+
+    public static void checkIndexInBounds(int realLength, int offset, int length)
+    {
+        if (realLength == 0 || length == 0)
+            throw new IndexOutOfBoundsException("Unable to access offset " + offset + "; empty");
+        if (offset < 0)
+            throw new IndexOutOfBoundsException("Offset " + offset + " must not be negative");
+        if (length < 0)
+            throw new IndexOutOfBoundsException("Length " + length + " must not be negative");
+        int endOffset = offset + length;
+        if (endOffset > realLength)
+            throw new IndexOutOfBoundsException(String.format("Offset %d, length = %d; real length was %d", offset, length, realLength));
+    }
+}

test/unit/accord/utils/Property.java

@@ -0,0 +1,382 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package accord.utils;
+
+import java.time.Duration;
+import java.util.Arrays;
+import java.util.Objects;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import javax.annotation.Nullable;
+
+import org.apache.cassandra.transport.ProtocolException;
+import org.apache.cassandra.utils.concurrent.AsyncPromise;
+
+public class Property
+{
+    public static abstract class Common<T extends Common<T>>
+    {
+        protected long seed = ThreadLocalRandom.current().nextLong();
+        protected int examples = 1000;
+
+        protected boolean pure = true;
+        @Nullable
+        protected Duration timeout = null;
+
+        protected Common() {
+        }
+
+        protected Common(Common<?> other) {
+            this.seed = other.seed;
+            this.examples = other.examples;
+            this.pure = other.pure;
+            this.timeout = other.timeout;
+        }
+
+        public T withSeed(long seed)
+        {
+            this.seed = seed;
+            return (T) this;
+        }
+
+        public T withExamples(int examples)
+        {
+            if (examples <= 0)
+                throw new IllegalArgumentException("Examples must be positive");
+            this.examples = examples;
+            return (T) this;
+        }
+
+        public T withPure(boolean pure)
+        {
+            this.pure = pure;
+            return (T) this;
+        }
+
+        public T withTimeout(Duration timeout)
+        {
+            this.timeout = timeout;
+            return (T) this;
+        }
+
+        protected void checkWithTimeout(Runnable fn)
+        {
+            AsyncPromise<?> promise = new AsyncPromise<>();
+            Thread t = new Thread(() -> {
+                try
+                {
+                    fn.run();
+                    promise.setSuccess(null);
+                }
+                catch (Throwable e)
+                {
+                    promise.setFailure(e);
+                }
+            });
+            t.setName("property with timeout");
+            t.setDaemon(true);
+            try
+            {
+                t.start();
+                promise.get(timeout.toNanos(), TimeUnit.NANOSECONDS);
+            }
+            catch (ExecutionException e)
+            {
+                throw new ProtocolException(propertyError(this, e.getCause()));
+            }
+            catch (InterruptedException e)
+            {
+                t.interrupt();
+                throw new PropertyError(propertyError(this, e));
+            }
+            catch (TimeoutException e)
+            {
+                t.interrupt();
+                TimeoutException override = new TimeoutException("property test did not complete within " + this.timeout);
+                override.setStackTrace(new StackTraceElement[0]);
+                throw new PropertyError(propertyError(this, override));
+            }
+        }
+    }
+
+    public static class ForBuilder extends Common<ForBuilder>
+    {
+        public void check(FailingConsumer<RandomSource> fn)
+        {
+            forAll(Gens.random()).check(fn);
+        }
+
+        public <T> SingleBuilder<T> forAll(Gen<T> gen)
+        {
+            return new SingleBuilder<>(gen, this);
+        }
+
+        public <A, B> DoubleBuilder<A, B> forAll(Gen<A> a, Gen<B> b)
+        {
+            return new DoubleBuilder<>(a, b, this);
+        }
+
+        public <A, B, C> TrippleBuilder<A, B, C> forAll(Gen<A> a, Gen<B> b, Gen<C> c)
+        {
+            return new TrippleBuilder<>(a, b, c, this);
+        }
+    }
+
+    private static Object normalizeValue(Object value)
+    {
+        if (value == null)
+            return null;
+        // one day java arrays will have a useful toString... one day...
+        if (value.getClass().isArray())
+        {
+            Class<?> subType = value.getClass().getComponentType();
+            if (!subType.isPrimitive())
+                return Arrays.asList((Object[]) value);
+            if (Byte.TYPE == subType)
+                return Arrays.toString((byte[]) value);
+            if (Character.TYPE == subType)
+                return Arrays.toString((char[]) value);
+            if (Short.TYPE == subType)
+                return Arrays.toString((short[]) value);
+            if (Integer.TYPE == subType)
+                return Arrays.toString((int[]) value);
+            if (Long.TYPE == subType)
+                return Arrays.toString((long[]) value);
+            if (Float.TYPE == subType)
+                return Arrays.toString((float[]) value);
+            if (Double.TYPE == subType)
+                return Arrays.toString((double[]) value);
+        }
+        try
+        {
+            return value.toString();
+        }
+        catch (Throwable t)
+        {
+            return "Object.toString failed: " + t.getClass().getCanonicalName() + ": " + t.getMessage();
+        }
+    }
+
+    private static String propertyError(Common<?> input, Throwable cause, Object... values)
+    {
+        StringBuilder sb = new StringBuilder();
+        // return "Seed=" + seed + "\nExamples=" + examples;
+        sb.append("Property error detected:\nSeed = ").append(input.seed).append('\n');
+        sb.append("Examples = ").append(input.examples).append('\n');
+        sb.append("Pure = ").append(input.pure).append('\n');
+        if (cause != null)
+        {
+            String msg = cause.getMessage();
+            sb.append("Error: ");
+            // to improve readability, if a newline is detected move the error msg to the next line
+            if (msg != null && msg.contains("\n"))
+                msg = "\n\t" + msg.replace("\n", "\n\t");
+            if (msg == null)
+                msg = cause.getClass().getCanonicalName();
+            sb.append(msg).append('\n');
+        }
+        if (values != null)
+        {
+            sb.append("Values:\n");
+            for (int i = 0; i < values.length; i++)
+                sb.append('\t').append(i).append(" = ").append(normalizeValue(values[i])).append('\n');
+        }
+        return sb.toString();
+    }
+
+    public interface FailingConsumer<A>
+    {
+        void accept(A value) throws Exception;
+    }
+
+    public static class SingleBuilder<T> extends Common<SingleBuilder<T>>
+    {
+        private final Gen<T> gen;
+
+        private SingleBuilder(Gen<T> gen, Common<?> other) {
+            super(other);
+            this.gen = Objects.requireNonNull(gen);
+        }
+
+        public void check(FailingConsumer<T> fn)
+        {
+            if (timeout != null)
+            {
+                checkWithTimeout(() -> checkInternal(fn));
+                return;
+            }
+            checkInternal(fn);
+        }
+
+        private void checkInternal(FailingConsumer<T> fn)
+        {
+            RandomSource random = new DefaultRandom(seed);
+            for (int i = 0; i < examples; i++)
+            {
+                T value = null;
+                try
+                {
+                    checkInterrupted();
+                    fn.accept(value = gen.next(random));
+                }
+                catch (Throwable t)
+                {
+                    throw new PropertyError(propertyError(this, t, value), t);
+                }
+                if (pure)
+                {
+                    seed = random.nextLong();
+                    random.setSeed(seed);
+                }
+            }
+        }
+    }
+
+    public interface FailingBiConsumer<A, B>
+    {
+        void accept(A a, B b) throws Exception;
+    }
+
+    public static class DoubleBuilder<A, B> extends Common<DoubleBuilder<A, B>>
+    {
+        private final Gen<A> aGen;
+        private final Gen<B> bGen;
+
+        private DoubleBuilder(Gen<A> aGen, Gen<B> bGen, Common<?> other) {
+            super(other);
+            this.aGen = Objects.requireNonNull(aGen);
+            this.bGen = Objects.requireNonNull(bGen);
+        }
+
+        public void check(FailingBiConsumer<A, B> fn)
+        {
+            if (timeout != null)
+            {
+                checkWithTimeout(() -> checkInternal(fn));
+                return;
+            }
+            checkInternal(fn);
+        }
+
+        private void checkInternal(FailingBiConsumer<A, B> fn)
+        {
+            RandomSource random = new DefaultRandom(seed);
+            for (int i = 0; i < examples; i++)
+            {
+                A a = null;
+                B b = null;
+                try
+                {
+                    checkInterrupted();
+                    fn.accept(a = aGen.next(random), b = bGen.next(random));
+                }
+                catch (Throwable t)
+                {
+                    throw new PropertyError(propertyError(this, t, a, b), t);
+                }
+                if (pure)
+                {
+                    seed = random.nextLong();
+                    random.setSeed(seed);
+                }
+            }
+        }
+    }
+
+    public interface FailingTriConsumer<A, B, C>
+    {
+        void accept(A a, B b, C c) throws Exception;
+    }
+
+    public static class TrippleBuilder<A, B, C> extends Common<TrippleBuilder<A, B, C>>
+    {
+        private final Gen<A> as;
+        private final Gen<B> bs;
+        private final Gen<C> cs;
+
+        public TrippleBuilder(Gen<A> as, Gen<B> bs, Gen<C> cs, Common<?> other)
+        {
+            super(other);
+            this.as = as;
+            this.bs = bs;
+            this.cs = cs;
+        }
+
+        public void check(FailingTriConsumer<A, B, C> fn)
+        {
+            if (timeout != null)
+            {
+                checkWithTimeout(() -> checkInternal(fn));
+                return;
+            }
+            checkInternal(fn);
+        }
+
+        private void checkInternal(FailingTriConsumer<A, B, C> fn)
+        {
+            RandomSource random = new DefaultRandom(seed);
+            for (int i = 0; i < examples; i++)
+            {
+                A a = null;
+                B b = null;
+                C c = null;
+                try
+                {
+                    checkInterrupted();
+                    fn.accept(a = as.next(random), b = bs.next(random), c = cs.next(random));
+                }
+                catch (Throwable t)
+                {
+                    throw new PropertyError(propertyError(this, t, a, b, c), t);
+                }
+                if (pure)
+                {
+                    seed = random.nextLong();
+                    random.setSeed(seed);
+                }
+            }
+        }
+    }
+
+    private static void checkInterrupted() throws InterruptedException
+    {
+        if (Thread.currentThread().isInterrupted())
+            throw new InterruptedException();
+    }
+
+    public static class PropertyError extends AssertionError
+    {
+        public PropertyError(String message, Throwable cause)
+        {
+            super(message, cause);
+        }
+
+        public PropertyError(String message)
+        {
+            super(message);
+        }
+    }
+
+    public static ForBuilder qt()
+    {
+        return new ForBuilder();
+    }
+}

test/unit/accord/utils/RandomSource.java

@@ -0,0 +1,361 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package accord.utils;
+
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import java.util.NavigableSet;
+import java.util.Random;
+import java.util.Set;
+import java.util.stream.DoubleStream;
+import java.util.stream.IntStream;
+import java.util.stream.LongStream;
+
+public interface RandomSource
+{
+    static RandomSource wrap(Random random)
+    {
+        return new WrappedRandomSource(random);
+    }
+
+    void nextBytes(byte[] bytes);
+
+    boolean nextBoolean();
+
+    int nextInt();
+
+    default int nextInt(int maxExclusive)
+    {
+        return nextInt(0, maxExclusive);
+    }
+
+    default int nextInt(int minInclusive, int maxExclusive)
+    {
+        // this is diff behavior than ThreadLocalRandom, which returns nextInt
+        if (minInclusive >= maxExclusive)
+            throw new IllegalArgumentException(String.format("Min (%s) should be less than max (%d).", minInclusive, maxExclusive));
+
+        int result = nextInt();
+        int delta = maxExclusive - minInclusive;
+        int mask = delta - 1;
+        if ((delta & mask) == 0) // power of two
+            result = (result & mask) + minInclusive;
+        else if (delta > 0)
+        {
+            // reject over-represented candidates
+            for (int u = result >>> 1;                  // ensure nonnegative
+                 u + mask - (result = u % delta) < 0;   // rejection check
+                 u = nextInt() >>> 1)                   // retry
+                ;
+            result += minInclusive;
+        }
+        else
+        {
+            // range not representable as int
+            while (result < minInclusive || result >= maxExclusive)
+                result = nextInt();
+        }
+        return result;
+    }
+
+    default IntStream ints()
+    {
+        return IntStream.generate(this::nextInt);
+    }
+
+    default IntStream ints(int maxExclusive)
+    {
+        return IntStream.generate(() -> nextInt(maxExclusive));
+    }
+
+    default IntStream ints(int minInclusive, int maxExclusive)
+    {
+        return IntStream.generate(() -> nextInt(minInclusive, maxExclusive));
+    }
+
+    long nextLong();
+
+    default long nextLong(long maxExclusive)
+    {
+        return nextLong(0, maxExclusive);
+    }
+
+    default long nextLong(long minInclusive, long maxExclusive)
+    {
+        // this is diff behavior than ThreadLocalRandom, which returns nextLong
+        if (minInclusive >= maxExclusive)
+            throw new IllegalArgumentException(String.format("Min (%s) should be less than max (%d).", minInclusive, maxExclusive));
+
+        long result = nextLong();
+        long delta = maxExclusive - minInclusive;
+        long mask = delta - 1;
+        if ((delta & mask) == 0L) // power of two
+            result = (result & mask) + minInclusive;
+        else if (delta > 0L)
+        {
+            // reject over-represented candidates
+            for (long u = result >>> 1;                 // ensure nonnegative
+                 u + mask - (result = u % delta) < 0L;  // rejection check
+                 u = nextLong() >>> 1)                  // retry
+                ;
+            result += minInclusive;
+        }
+        else
+        {
+            // range not representable as long
+            while (result < minInclusive || result >= maxExclusive)
+                result = nextLong();
+        }
+        return result;
+    }
+
+    default LongStream longs()
+    {
+        return LongStream.generate(this::nextLong);
+    }
+
+    default LongStream longs(long maxExclusive)
+    {
+        return LongStream.generate(() -> nextLong(maxExclusive));
+    }
+
+    default LongStream longs(long minInclusive, long maxExclusive)
+    {
+        return LongStream.generate(() -> nextLong(minInclusive, maxExclusive));
+    }
+
+    float nextFloat();
+
+    double nextDouble();
+
+    default double nextDouble(double maxExclusive)
+    {
+        return nextDouble(0, maxExclusive);
+    }
+
+    default double nextDouble(double minInclusive, double maxExclusive)
+    {
+        if (minInclusive >= maxExclusive)
+            throw new IllegalArgumentException(String.format("Min (%s) should be less than max (%d).", minInclusive, maxExclusive));
+
+        double result = nextDouble();
+        result = result * (maxExclusive - minInclusive) + minInclusive;
+        if (result >= maxExclusive) // correct for rounding
+            result = Double.longBitsToDouble(Double.doubleToLongBits(maxExclusive) - 1);
+        return result;
+    }
+
+    default DoubleStream doubles()
+    {
+        return DoubleStream.generate(this::nextDouble);
+    }
+
+    default DoubleStream doubles(double maxExclusive)
+    {
+        return DoubleStream.generate(() -> nextDouble(maxExclusive));
+    }
+
+    default DoubleStream doubles(double minInclusive, double maxExclusive)
+    {
+        return DoubleStream.generate(() -> nextDouble(minInclusive, maxExclusive));
+    }
+
+    double nextGaussian();
+
+    default int pickInt(int first, int second, int... rest)
+    {
+        int offset = nextInt(0, rest.length + 2);
+        switch (offset)
+        {
+            case 0:  return first;
+            case 1:  return second;
+            default: return rest[offset - 2];
+        }
+    }
+
+    default int pickInt(int[] array)
+    {
+        return pickInt(array, 0, array.length);
+    }
+
+    default int pickInt(int[] array, int offset, int length)
+    {
+        Invariants.checkIndexInBounds(array.length, offset, length);
+        if (length == 1)
+            return array[offset];
+        return array[nextInt(offset, offset + length)];
+    }
+
+    default long pickLong(long first, long second, long... rest)
+    {
+        int offset = nextInt(0, rest.length + 2);
+        switch (offset)
+        {
+            case 0:  return first;
+            case 1:  return second;
+            default: return rest[offset - 2];
+        }
+    }
+
+    default long pickLong(long[] array)
+    {
+        return pickLong(array, 0, array.length);
+    }
+
+    default long pickLong(long[] array, int offset, int length)
+    {
+        Invariants.checkIndexInBounds(array.length, offset, length);
+        if (length == 1)
+            return array[offset];
+        return array[nextInt(offset, offset + length)];
+    }
+
+    default <T extends Comparable<T>> T pick(Set<T> set)
+    {
+        List<T> values = new ArrayList<>(set);
+        // Non-ordered sets may have different iteration order on different environments, which would make a seed produce different histories!
+        // To avoid such a problem, make sure to apply a deterministic function (sort).
+        if (!(set instanceof NavigableSet))
+            values.sort(Comparator.naturalOrder());
+        return pick(values);
+    }
+
+    default <T> T pick(T first, T second, T... rest)
+    {
+        int offset = nextInt(0, rest.length + 2);
+        switch (offset)
+        {
+            case 0:  return first;
+            case 1:  return second;
+            default: return rest[offset - 2];
+        }
+    }
+
+    default <T> T pick(T[] array)
+    {
+        return array[nextInt(array.length)];
+    }
+
+    default <T> T pick(List<T> values)
+    {
+        return pick(values, 0, values.size());
+    }
+
+    default <T> T pick(List<T> values, int offset, int length)
+    {
+        Invariants.checkIndexInBounds(values.size(), offset, length);
+        if (length == 1)
+            return values.get(offset);
+        return values.get(nextInt(offset, offset + length));
+    }
+
+    void setSeed(long seed);
+
+    RandomSource fork();
+
+    /**
+     * Returns true with a probability of {@code chance}.  This logic is logically the same as
+     * <pre>{@code nextFloat() < chance}</pre>
+     *
+     * @param chance cumulative probability in range [0..1]
+     */
+    default boolean decide(float chance)
+    {
+        return nextFloat() < chance;
+    }
+
+    /**
+     * Returns true with a probability of {@code chance}.  This logic is logically the same as
+     * <pre>{@code nextDouble() < chance}</pre>
+     *
+     * @param chance cumulative probability in range [0..1]
+     */
+    default boolean decide(double chance)
+    {
+        return nextDouble() < chance;
+    }
+
+    default long reset()
+    {
+        long seed = nextLong();
+        setSeed(seed);
+        return seed;
+    }
+
+    default Random asJdkRandom()
+    {
+        return new Random()
+        {
+            @Override
+            public void setSeed(long seed)
+            {
+                RandomSource.this.setSeed(seed);
+            }
+
+            @Override
+            public void nextBytes(byte[] bytes)
+            {
+                RandomSource.this.nextBytes(bytes);
+            }
+
+            @Override
+            public int nextInt()
+            {
+                return RandomSource.this.nextInt();
+            }
+
+            @Override
+            public int nextInt(int bound)
+            {
+                return RandomSource.this.nextInt(bound);
+            }
+
+            @Override
+            public long nextLong()
+            {
+                return RandomSource.this.nextLong();
+            }
+
+            @Override
+            public boolean nextBoolean()
+            {
+                return RandomSource.this.nextBoolean();
+            }
+
+            @Override
+            public float nextFloat()
+            {
+                return RandomSource.this.nextFloat();
+            }
+
+            @Override
+            public double nextDouble()
+            {
+                return RandomSource.this.nextDouble();
+            }
+
+            @Override
+            public double nextGaussian()
+            {
+                return RandomSource.this.nextGaussian();
+            }
+        };
+    }
+}

test/unit/accord/utils/WrappedRandomSource.java

@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package accord.utils;
+
+import java.util.Random;
+
+class WrappedRandomSource implements RandomSource
+{
+    private final Random random;
+
+    WrappedRandomSource(Random random)
+    {
+        this.random = random;
+    }
+
+    @Override
+    public Random asJdkRandom()
+    {
+        return random;
+    }
+
+    @Override
+    public void nextBytes(byte[] bytes)
+    {
+        random.nextBytes(bytes);
+    }
+
+    @Override
+    public boolean nextBoolean()
+    {
+        return random.nextBoolean();
+    }
+
+    @Override
+    public int nextInt()
+    {
+        return random.nextInt();
+    }
+
+    @Override
+    public int nextInt(int maxExclusive)
+    {
+        return random.nextInt(maxExclusive);
+    }
+
+    @Override
+    public long nextLong()
+    {
+        return random.nextLong();
+    }
+
+    @Override
+    public float nextFloat()
+    {
+        return random.nextFloat();
+    }
+
+    @Override
+    public double nextDouble()
+    {
+        return random.nextDouble();
+    }
+
+    @Override
+    public double nextGaussian()
+    {
+        return random.nextGaussian();
+    }
+
+    @Override
+    public void setSeed(long seed)
+    {
+        random.setSeed(seed);
+    }
+
+    @Override
+    public RandomSource fork()
+    {
+        return new WrappedRandomSource(new Random(nextLong()));
+    }
+}

test/unit/org/apache/cassandra/concurrent/ForwardingExecutorPlus.java

@@ -0,0 +1,219 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.concurrent;
+
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+
+import com.google.common.util.concurrent.FutureCallback;
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.common.util.concurrent.MoreExecutors;
+
+import org.apache.cassandra.utils.WithResources;
+import org.apache.cassandra.utils.concurrent.AsyncPromise;
+import org.apache.cassandra.utils.concurrent.Future;
+
+public class ForwardingExecutorPlus implements ExecutorPlus
+{
+    private final ExecutorService delegate;
+
+    public ForwardingExecutorPlus(ExecutorService delegate)
+    {
+        this.delegate = delegate;
+    }
+
+    protected ExecutorService delegate()
+    {
+        return delegate;
+    }
+
+    @Override
+    public void shutdown()
+    {
+        delegate().shutdown();
+    }
+
+    @Override
+    public List<Runnable> shutdownNow()
+    {
+        return delegate().shutdownNow();
+    }
+
+    @Override
+    public boolean isShutdown()
+    {
+        return delegate().isShutdown();
+    }
+
+    @Override
+    public boolean isTerminated()
+    {
+        return delegate().isTerminated();
+    }
+
+    @Override
+    public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException
+    {
+        return delegate().awaitTermination(timeout, unit);
+    }
+
+    @Override
+    public <T> Future<T> submit(Callable<T> task)
+    {
+        return wrap(delegate().submit(task));
+    }
+
+    @Override
+    public <T> Future<T> submit(Runnable task, T result)
+    {
+        return wrap(delegate().submit(task, result));
+    }
+
+    @Override
+    public Future<?> submit(Runnable task)
+    {
+        return wrap(delegate().submit(task));
+    }
+
+    @Override
+    public void execute(WithResources withResources, Runnable task)
+    {
+        execute(TaskFactory.standard().toExecute(withResources, task));
+    }
+
+    @Override
+    public <T> Future<T> submit(WithResources withResources, Callable<T> task)
+    {
+        class Catch
+        {
+            T value;
+        }
+        Catch c = new Catch();
+        Runnable exec = TaskFactory.standard().toExecute(withResources, () -> {
+            try
+            {
+                c.value = task.call();
+            }
+            catch (Exception e)
+            {
+                throw new RuntimeException(e);
+            }
+        });
+        return submit(() -> {
+            exec.run();
+            return c.value;
+        });
+    }
+
+    @Override
+    public Future<?> submit(WithResources withResources, Runnable task)
+    {
+        return submit(TaskFactory.standard().toExecute(withResources, task));
+    }
+
+    @Override
+    public <T> Future<T> submit(WithResources withResources, Runnable task, T result)
+    {
+        return submit(Executors.callable(TaskFactory.standard().toSubmit(withResources, task), result));
+    }
+
+    @Override
+    public boolean inExecutor()
+    {
+        return false;
+    }
+
+    @Override
+    public void execute(Runnable command)
+    {
+        delegate().execute(command);
+    }
+
+    @Override
+    public int getCorePoolSize()
+    {
+        return 0;
+    }
+
+    @Override
+    public void setCorePoolSize(int newCorePoolSize)
+    {
+
+    }
+
+    @Override
+    public int getMaximumPoolSize()
+    {
+        return 0;
+    }
+
+    @Override
+    public void setMaximumPoolSize(int newMaximumPoolSize)
+    {
+
+    }
+
+    @Override
+    public int getActiveTaskCount()
+    {
+        return 0;
+    }
+
+    @Override
+    public long getCompletedTaskCount()
+    {
+        return 0;
+    }
+
+    @Override
+    public int getPendingTaskCount()
+    {
+        return 0;
+    }
+
+    private <T> Future<T> wrap(java.util.concurrent.Future<T> submit)
+    {
+        if (submit instanceof Future)
+            return (Future<T>) submit;
+        if (submit instanceof ListenableFuture)
+        {
+            AsyncPromise<T> promise = new AsyncPromise<>();
+            Futures.addCallback((ListenableFuture<T>) submit, new FutureCallback<T>()
+            {
+                @Override
+                public void onSuccess(T result)
+                {
+                    promise.setSuccess(result);
+                }
+
+                @Override
+                public void onFailure(Throwable t)
+                {
+                    promise.setFailure(t);
+                }
+            }, MoreExecutors.directExecutor());
+            return promise;
+        }
+        throw new IllegalStateException("Unexpected future type: " + submit.getClass());
+    }
+}

test/unit/org/apache/cassandra/concurrent/ForwardingLocalAwareExecutorPlus.java

@@ -0,0 +1,167 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.concurrent;
+
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.cassandra.utils.WithResources;
+import org.apache.cassandra.utils.concurrent.Future;
+
+public class ForwardingLocalAwareExecutorPlus implements LocalAwareExecutorPlus
+{
+    private final ExecutorPlus delegate;
+
+    public ForwardingLocalAwareExecutorPlus(ExecutorPlus delegate)
+    {
+        this.delegate = delegate;
+    }
+
+    protected ExecutorPlus delegate()
+    {
+        return delegate;
+    }
+
+    @Override
+    public void shutdown()
+    {
+        delegate().shutdown();
+    }
+
+    @Override
+    public List<Runnable> shutdownNow()
+    {
+        return delegate().shutdownNow();
+    }
+
+    @Override
+    public boolean isShutdown()
+    {
+        return delegate().isShutdown();
+    }
+
+    @Override
+    public boolean isTerminated()
+    {
+        return delegate().isTerminated();
+    }
+
+    @Override
+    public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException
+    {
+        return delegate().awaitTermination(timeout, unit);
+    }
+
+    @Override
+    public <T> Future<T> submit(Callable<T> task)
+    {
+        return delegate().submit(task);
+    }
+
+    @Override
+    public <T> Future<T> submit(Runnable task, T result)
+    {
+        return delegate().submit(task, result);
+    }
+
+    @Override
+    public Future<?> submit(Runnable task)
+    {
+        return delegate().submit(task);
+    }
+
+    @Override
+    public void execute(WithResources withResources, Runnable task)
+    {
+        delegate().execute(withResources, task);
+    }
+
+    @Override
+    public <T> Future<T> submit(WithResources withResources, Callable<T> task)
+    {
+        return delegate().submit(withResources, task);
+    }
+
+    @Override
+    public Future<?> submit(WithResources withResources, Runnable task)
+    {
+        return delegate().submit(withResources, task);
+    }
+
+    @Override
+    public <T> Future<T> submit(WithResources withResources, Runnable task, T result)
+    {
+        return delegate().submit(withResources, task, result);
+    }
+
+    @Override
+    public boolean inExecutor()
+    {
+        return delegate().inExecutor();
+    }
+
+    @Override
+    public void execute(Runnable command)
+    {
+        delegate().execute(command);
+    }
+
+    @Override
+    public int getCorePoolSize()
+    {
+        return delegate().getCorePoolSize();
+    }
+
+    @Override
+    public void setCorePoolSize(int newCorePoolSize)
+    {
+        delegate().setCorePoolSize(newCorePoolSize);
+    }
+
+    @Override
+    public int getMaximumPoolSize()
+    {
+        return delegate().getMaximumPoolSize();
+    }
+
+    @Override
+    public void setMaximumPoolSize(int newMaximumPoolSize)
+    {
+        delegate().setMaximumPoolSize(newMaximumPoolSize);
+    }
+
+    @Override
+    public int getActiveTaskCount()
+    {
+        return delegate().getActiveTaskCount();
+    }
+
+    @Override
+    public long getCompletedTaskCount()
+    {
+        return delegate().getCompletedTaskCount();
+    }
+
+    @Override
+    public int getPendingTaskCount()
+    {
+        return delegate().getPendingTaskCount();
+    }
+}

test/unit/org/apache/cassandra/concurrent/ForwardingScheduledExecutorPlus.java

@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.concurrent;
+
+import java.util.concurrent.Callable;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.TimeUnit;
+
+import static com.google.common.primitives.Longs.max;
+import static java.util.concurrent.TimeUnit.NANOSECONDS;
+import static org.apache.cassandra.utils.Clock.Global.nanoTime;
+
+public class ForwardingScheduledExecutorPlus extends ForwardingExecutorPlus implements ScheduledExecutorPlus
+{
+    private final ScheduledExecutorService delegate;
+
+    public ForwardingScheduledExecutorPlus(ScheduledExecutorService delegate)
+    {
+        super(delegate);
+        this.delegate = delegate;
+    }
+
+    protected ScheduledExecutorService delegate()
+    {
+        return delegate;
+    }
+
+    @Override
+    public ScheduledFuture<?> scheduleSelfRecurring(Runnable run, long delay, TimeUnit units)
+    {
+        return schedule(run, delay, units);
+    }
+
+    @Override
+    public ScheduledFuture<?> scheduleAt(Runnable run, long deadline)
+    {
+        return schedule(run, max(0, deadline - nanoTime()), NANOSECONDS);
+    }
+
+    @Override
+    public ScheduledFuture<?> scheduleTimeoutAt(Runnable run, long deadline)
+    {
+        return scheduleTimeoutWithDelay(run, max(0, deadline - nanoTime()), NANOSECONDS);
+    }
+
+    @Override
+    public ScheduledFuture<?> scheduleTimeoutWithDelay(Runnable run, long delay, TimeUnit units)
+    {
+        return schedule(run, delay, units);
+    }
+
+    @Override
+    public ScheduledFuture<?> schedule(Runnable command, long delay, TimeUnit unit)
+    {
+        return delegate().schedule(command, delay, unit);
+    }
+
+    @Override
+    public <V> ScheduledFuture<V> schedule(Callable<V> callable, long delay, TimeUnit unit)
+    {
+        return delegate().schedule(callable, delay, unit);
+    }
+
+    @Override
+    public ScheduledFuture<?> scheduleAtFixedRate(Runnable command, long initialDelay, long period, TimeUnit unit)
+    {
+        return delegate().scheduleAtFixedRate(command, initialDelay, period, unit);
+    }
+
+    @Override
+    public ScheduledFuture<?> scheduleWithFixedDelay(Runnable command, long initialDelay, long delay, TimeUnit unit)
+    {
+        return delegate().scheduleWithFixedDelay(command, initialDelay, delay, unit);
+    }
+}

test/unit/org/apache/cassandra/concurrent/SimulatedExecutorFactory.java

@@ -0,0 +1,506 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.concurrent;
+
+import java.util.Collections;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.PriorityQueue;
+import java.util.Queue;
+import java.util.concurrent.AbstractExecutorService;
+import java.util.concurrent.Callable;
+import java.util.concurrent.Delayed;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.RejectedExecutionException;
+import java.util.concurrent.RejectedExecutionHandler;
+import java.util.concurrent.RunnableFuture;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.function.LongSupplier;
+
+import accord.utils.Gens;
+import accord.utils.RandomSource;
+import org.apache.cassandra.utils.Clock;
+
+import static java.util.concurrent.TimeUnit.NANOSECONDS;
+
+public class SimulatedExecutorFactory implements ExecutorFactory, Clock
+{
+    private static class Item implements Comparable<Item>
+    {
+        private final long runAtNanos;
+        private final long seq;
+        private final org.apache.cassandra.utils.concurrent.RunnableFuture<?>  action;
+
+        private Item(long runAtNanos, long seq, org.apache.cassandra.utils.concurrent.RunnableFuture<?>  action)
+        {
+            if (runAtNanos < 0)
+                throw new IllegalArgumentException("Time went backwards!  Given " + runAtNanos);
+            this.runAtNanos = runAtNanos;
+            this.seq = seq;
+            this.action = action;
+        }
+
+        @Override
+        public int compareTo(Item o)
+        {
+            int rc = Long.compare(runAtNanos, o.runAtNanos);
+            if (rc != 0)
+                return rc;
+            return Long.compare(seq, o.seq);
+        }
+
+        @Override
+        public String toString()
+        {
+            return "Item{" +
+                   "runAtNanos=" + runAtNanos +
+                   ", seq=" + seq +
+                   '}';
+        }
+    }
+
+    private final RandomSource rs;
+    private final long startTimeNanos;
+    private final PriorityQueue<Item> queue = new PriorityQueue<>();
+    private long seq = 0;
+    private long nowNanos;
+    private int repeatedTasks = 0;
+
+    public SimulatedExecutorFactory(RandomSource rs, long startTimeNanos)
+    {
+        this.rs = rs;
+        this.startTimeNanos = startTimeNanos;
+    }
+
+    public boolean processOne()
+    {
+        // if we count the repeated tasks, then processAll will never complete
+        if (queue.size() == repeatedTasks)
+            return false;
+        Item item = queue.poll();
+        if (item == null)
+            return false;
+        nowNanos = Math.max(nowNanos + 1, item.runAtNanos);
+        item.action.run();
+        return true;
+    }
+
+    @Override
+    public long nanoTime()
+    {
+        return nowNanos++;
+    }
+
+    @Override
+    public long currentTimeMillis()
+    {
+        return TimeUnit.NANOSECONDS.toMillis(startTimeNanos + nanoTime());
+    }
+
+
+    @Override
+    public ExecutorBuilder<? extends SequentialExecutorPlus> configureSequential(String name)
+    {
+        return new SimulatedExecutorBuilder<>().configureSequential(name);
+    }
+
+    @Override
+    public ExecutorBuilder<? extends ExecutorPlus> configurePooled(String name, int threads)
+    {
+        return new SimulatedExecutorBuilder<>().configurePooled(name, threads);
+    }
+
+    @Override
+    public ExecutorBuilderFactory<ExecutorPlus, SequentialExecutorPlus> withJmx(String jmxPath)
+    {
+        return this;
+    }
+
+    @Override
+    public LocalAwareSubFactory localAware()
+    {
+        return new SimulatedExecutorBuilder<>();
+    }
+
+    @Override
+    public ScheduledExecutorPlus scheduled(boolean executeOnShutdown, String name, int priority, SimulatorSemantics simulatorSemantics)
+    {
+        return new ForwardingScheduledExecutorPlus(new UnorderedScheduledExecutorService());
+    }
+
+    @Override
+    public Thread startThread(String name, Runnable runnable, InfiniteLoopExecutor.Daemon daemon)
+    {
+        throw new UnsupportedOperationException("Thread can't be simualted");
+    }
+
+    @Override
+    public Interruptible infiniteLoop(String name, Interruptible.Task task, InfiniteLoopExecutor.SimulatorSafe simulatorSafe, InfiniteLoopExecutor.Daemon daemon, InfiniteLoopExecutor.Interrupts interrupts)
+    {
+        throw new UnsupportedOperationException("TODO");
+    }
+
+    @Override
+    public ThreadGroup newThreadGroup(String name)
+    {
+        throw new UnsupportedOperationException("Thread can't be simualted");
+    }
+
+    private class SimulatedExecutorBuilder<E extends ExecutorService> implements ExecutorBuilder<E>, LocalAwareSubFactory, LocalAwareSubFactoryWithJMX
+    {
+        private int threads = -1;
+
+        @Override
+        public ExecutorBuilder<E> withKeepAlive(long keepAlive, TimeUnit keepAliveUnits)
+        {
+            return this;
+        }
+
+        @Override
+        public ExecutorBuilder<E> withKeepAlive()
+        {
+            return this;
+        }
+
+        @Override
+        public ExecutorBuilder<E> withThreadPriority(int threadPriority)
+        {
+            return this;
+        }
+
+        @Override
+        public ExecutorBuilder<E> withThreadGroup(ThreadGroup threadGroup)
+        {
+            return this;
+        }
+
+        @Override
+        public ExecutorBuilder<E> withDefaultThreadGroup()
+        {
+            return this;
+        }
+
+        @Override
+        public ExecutorBuilder<E> withQueueLimit(int queueLimit)
+        {
+            throw new UnsupportedOperationException("TODO");
+        }
+
+        @Override
+        public ExecutorBuilder<E> withRejectedExecutionHandler(RejectedExecutionHandler rejectedExecutionHandler)
+        {
+            throw new UnsupportedOperationException("TODO");
+        }
+
+        @Override
+        public ExecutorBuilder<E> withUncaughtExceptionHandler(Thread.UncaughtExceptionHandler uncaughtExceptionHandler)
+        {
+            throw new UnsupportedOperationException("TODO");
+        }
+
+        @Override
+        public ExecutorBuilder<? extends LocalAwareSequentialExecutorPlus> configureSequential(String name)
+        {
+            threads = 1;
+            return (ExecutorBuilder<? extends LocalAwareSequentialExecutorPlus>) this;
+        }
+
+        @Override
+        public ExecutorBuilder<? extends LocalAwareExecutorPlus> configurePooled(String name, int threads)
+        {
+            this.threads = threads;
+            return (ExecutorBuilder<? extends LocalAwareExecutorPlus>) this;
+        }
+
+        @Override
+        public LocalAwareSubFactoryWithJMX withJmx(String jmxPath)
+        {
+            return this;
+        }
+
+        @Override
+        public LocalAwareSubFactoryWithJMX withJmxInternal()
+        {
+            return this;
+        }
+
+        @Override
+        public LocalAwareExecutorPlus shared(String name, int threads, ExecutorPlus.MaximumPoolSizeListener onSetMaxSize)
+        {
+            return new ForwardingLocalAwareExecutorPlus(build0());
+        }
+
+        @Override
+        public E build()
+        {
+            return (E) build0();
+        }
+
+        private ForwardingExecutorPlus build0()
+        {
+            return new ForwardingExecutorPlus(threads == 1 ?
+                                              new OrderedExecutorService() :
+                                              new UnorderedExecutorService());
+        }
+    }
+
+    private class UnorderedExecutorService extends AbstractExecutorService
+    {
+        private final LongSupplier jitterNanos;
+        private boolean shutdown = false;
+
+        UnorderedExecutorService()
+        {
+            long maxSmall = TimeUnit.MICROSECONDS.toNanos(50);
+            long max = TimeUnit.MILLISECONDS.toNanos(5);
+            LongSupplier small = () -> rs.nextLong(0, maxSmall);
+            LongSupplier large = () -> rs.nextLong(maxSmall, max);
+            this.jitterNanos = Gens.bools().runs(rs.nextInt(1, 11) / 100.0D, rs.nextInt(3, 15)).mapToLong(b -> b ? large.getAsLong() : small.getAsLong()).asLongSupplier(rs);
+        }
+
+        @Override
+        protected <T> RunnableFuture<T> newTaskFor(Runnable runnable, T value)
+        {
+            return new FutureTask<>(Executors.callable(runnable, value));
+        }
+
+        @Override
+        protected <T> RunnableFuture<T> newTaskFor(Callable<T> callable)
+        {
+            return new FutureTask<>(callable);
+        }
+
+        protected org.apache.cassandra.utils.concurrent.RunnableFuture<?> taskFor(Runnable command)
+        {
+            if (command instanceof org.apache.cassandra.utils.concurrent.RunnableFuture<?>)
+                return (org.apache.cassandra.utils.concurrent.RunnableFuture<?>) command;
+            return new FutureTask<>(Executors.callable(command));
+        }
+
+        @Override
+        public void shutdown()
+        {
+            shutdown = true;
+        }
+
+        @Override
+        public List<Runnable> shutdownNow()
+        {
+            return Collections.emptyList();
+        }
+
+        @Override
+        public boolean isShutdown()
+        {
+            return shutdown;
+        }
+
+        @Override
+        public boolean isTerminated()
+        {
+            return shutdown;
+        }
+
+        @Override
+        public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException
+        {
+            return shutdown;
+        }
+
+        @Override
+        public void execute(Runnable command)
+        {
+            checkNotShutdown();
+            queue.add(new Item(nowWithJitter(), SimulatedExecutorFactory.this.seq++, taskFor(command)));
+        }
+
+        protected void checkNotShutdown()
+        {
+            if (isShutdown())
+                throw new RejectedExecutionException("Shutdown");
+        }
+
+        protected long nowWithJitter()
+        {
+            return nanoTime() + jitterNanos.getAsLong();
+        }
+    }
+
+    private class OrderedExecutorService extends UnorderedExecutorService
+    {
+        private final Queue<Item> pending = new LinkedList<>();
+
+        @Override
+        public void execute(Runnable command)
+        {
+            checkNotShutdown();
+            boolean wasEmpty = pending.isEmpty();
+            Item task = new Item(nowWithJitter(), SimulatedExecutorFactory.this.seq++, taskFor(command));
+            pending.add(task);
+            if (wasEmpty)
+                runNextTask();
+        }
+
+        private void runNextTask()
+        {
+            Item next = pending.peek();
+            if (next == null)
+                return;
+
+            next.action.addCallback((s, f) -> afterExecution());
+            queue.add(next);
+        }
+
+        private void afterExecution()
+        {
+            pending.poll();
+            runNextTask();
+        }
+    }
+
+    private class UnorderedScheduledExecutorService extends UnorderedExecutorService implements ScheduledExecutorService
+    {
+        private class ScheduledFuture<T> extends FutureTask<T> implements java.util.concurrent.ScheduledFuture<T>
+        {
+            private final long sequenceNumber;
+            private final long periodNanos;
+            private long nextExecuteAtNanos;
+
+            ScheduledFuture(long sequenceNumber, long initialDelay, long value, TimeUnit unit, Callable<? extends T> call)
+            {
+                super(call);
+                this.sequenceNumber = sequenceNumber;
+                periodNanos = unit.toNanos(value);
+                nextExecuteAtNanos = triggerTimeNanos(initialDelay, unit);
+            }
+
+            private long triggerTimeNanos(long delay, TimeUnit unit)
+            {
+                long delayNanos = unit.toNanos(delay < 0 ? 0 : delay);
+                return nanoTime() + delayNanos;
+            }
+
+            @Override
+            public long getDelay(TimeUnit unit)
+            {
+                return unit.convert(nextExecuteAtNanos - nanoTime(), TimeUnit.NANOSECONDS);
+            }
+
+            @Override
+            public int compareTo(Delayed other)
+            {
+                if (other == this) // compare zero if same object
+                    return 0;
+                if (other instanceof ScheduledFuture)
+                {
+                    ScheduledFuture<?> x = (ScheduledFuture<?>) other;
+                    long diff = nextExecuteAtNanos - x.nextExecuteAtNanos;
+                    if (diff < 0)
+                        return -1;
+                    else if (diff > 0)
+                        return 1;
+                    else if (sequenceNumber < x.sequenceNumber)
+                        return -1;
+                    else
+                        return 1;
+                }
+                long diff = getDelay(NANOSECONDS) - other.getDelay(NANOSECONDS);
+                return (diff < 0) ? -1 : (diff > 0) ? 1 : 0;
+            }
+
+            @Override
+            public void run()
+            {
+                boolean periodic = periodNanos != 0;
+                if (!periodic)
+                {
+                    super.run();
+                }
+                else
+                {
+                    if (isCancelled())
+                        return;
+                    // run without setting the result
+                    try
+                    {
+                        call();
+                        long nowNanos = nanoTime();
+                        if (periodNanos > 0)
+                        {
+                            // scheduleAtFixedRate
+                            nextExecuteAtNanos += periodNanos;
+                        }
+                        else
+                        {
+                            // scheduleWithFixedDelay
+                            nextExecuteAtNanos = nowNanos + (-periodNanos);
+                        }
+                        long delayNanos = nextExecuteAtNanos - nowNanos;
+                        if (delayNanos < 0)
+                            delayNanos = 0;
+                        schedule(this, delayNanos, NANOSECONDS);
+                    }
+                    catch (Throwable t)
+                    {
+                        tryFailure(t);
+                    }
+                }
+            }
+        }
+
+        @Override
+        public ScheduledFuture<?> schedule(Runnable command, long delay, TimeUnit unit)
+        {
+            return schedule(Executors.callable(command), delay, unit);
+        }
+
+        @Override
+        public <V> ScheduledFuture<V> schedule(Callable<V> callable, long delay, TimeUnit unit)
+        {
+            checkNotShutdown();
+            ScheduledFuture<V> task = new ScheduledFuture<>(seq++, delay, 0, NANOSECONDS, callable);
+            queue.add(new Item(nowWithJitter() + unit.toNanos(delay), task.sequenceNumber, task));
+            return task;
+        }
+
+        @Override
+        public ScheduledFuture<?> scheduleAtFixedRate(Runnable command, long initialDelay, long period, TimeUnit unit)
+        {
+            checkNotShutdown();
+            ScheduledFuture<?> task = new ScheduledFuture<>(seq++, initialDelay, period, unit, Executors.callable(command));
+            repeatedTasks++;
+            task.addCallback((s, f) -> repeatedTasks--);
+            queue.add(new Item(nowWithJitter() + unit.toNanos(initialDelay), task.sequenceNumber, task));
+            return task;
+        }
+
+        @Override
+        public ScheduledFuture<?> scheduleWithFixedDelay(Runnable command, long initialDelay, long delay, TimeUnit unit)
+        {
+            checkNotShutdown();
+            ScheduledFuture<?> task = new ScheduledFuture<>(seq++, initialDelay, -delay, unit, Executors.callable(command));
+            repeatedTasks++;
+            task.addCallback((s, f) -> repeatedTasks--);
+            queue.add(new Item(nowWithJitter() + unit.toNanos(initialDelay), task.sequenceNumber, task));
+            return task;
+        }
+    }
+}

test/unit/org/apache/cassandra/config/DatabaseDescriptorRefTest.java

@@ -140,6 +140,11 @@ public class DatabaseDescriptorRefTest
     "org.apache.cassandra.config.GuardrailsOptions$ConsistencyLevels",
     "org.apache.cassandra.config.GuardrailsOptions$TableProperties",
     "org.apache.cassandra.config.ParameterizedClass",
+    "org.apache.cassandra.config.RepairConfig",
+    "org.apache.cassandra.config.RepairRetrySpec",
+    "org.apache.cassandra.config.RetrySpec",
+    "org.apache.cassandra.config.RetrySpec$MaxAttempt",
+    "org.apache.cassandra.config.RetrySpec$Type",
     "org.apache.cassandra.config.ReplicaFilteringProtectionOptions",
     "org.apache.cassandra.config.StartupChecksOptions",
     "org.apache.cassandra.config.SubnetGroups",

test/unit/org/apache/cassandra/config/UnitConfigOverride.java

@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.config;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.UncheckedIOException;
+import java.nio.file.Files;
+import java.util.Arrays;
+import java.util.List;
+import javax.annotation.Nullable;
+
+public class UnitConfigOverride
+{
+    private enum ConfigType
+    {
+        CDC("cassandra.yaml", "cdc.yaml"),
+        COMPRESSION("cassandra.yaml", "commitlog_compression_LZ4.yaml"), // not 100% correct, LZ4 could be changed at build time
+        OA("cassandra.yaml", "storage_compatibility_mode_none.yaml"),
+        SYSTEM_KEYSPACE_DIRECTORY("cassandra.yaml", "system_keyspaces_directory.yaml"),
+        TRIE("cassandra.yaml", "trie_memtable.yaml", "storage_compatibility_mode_none.yaml");
+
+        public final List<String> configs;
+
+        ConfigType(String... configs)
+        {
+            this.configs = Arrays.asList(configs);
+        }
+    }
+
+    @Nullable
+    private static final ConfigType CONFIG_TYPE = null;
+
+    @SuppressWarnings("ConstantConditions")
+    public static void maybeOverrideConfig()
+    {
+        // CONFIG_TYPE is meant to be changed by users while debugging, so this condition won't be false on every machine
+        if (CONFIG_TYPE != null)
+            setConfigType(CONFIG_TYPE);
+    }
+
+    @SuppressWarnings("SameParameterValue")
+    private static void setConfigType(ConfigType type)
+    {
+        File confDir = new File("test/conf");
+        try
+        {
+            File tmp = Files.createTempFile("cassandra-conf", ".yaml").toFile();
+            tmp.deleteOnExit();
+            for (String name : type.configs)
+            {
+                try (FileOutputStream out = new FileOutputStream(tmp, true);
+                     InputStream in = new FileInputStream(new File(confDir, name)))
+                {
+                    in.transferTo(out);
+                }
+            }
+            CassandraRelevantProperties.CASSANDRA_CONFIG.setString(tmp.toURI().toString());
+        } catch (IOException e)
+        {
+            throw new UncheckedIOException(e);
+        }
+    }
+}

test/unit/org/apache/cassandra/config/YamlConfigurationLoaderTest.java

@@ -49,6 +49,51 @@ import static org.junit.Assert.assertTrue;
 
 public class YamlConfigurationLoaderTest
 {
+    @Test
+    public void repairRetryEmpty()
+    {
+        RepairRetrySpec repair_retries = loadRepairRetry(ImmutableMap.of());
+        // repair is empty
+        assertThat(repair_retries.isEnabled()).isFalse();
+        assertThat(repair_retries.isMerkleTreeRetriesEnabled()).isFalse();
+    }
+
+    @Test
+    public void repairRetryInheritance()
+    {
+        RepairRetrySpec repair_retries = loadRepairRetry(ImmutableMap.of("max_attempts", "3"));
+        assertThat(repair_retries.isEnabled()).isTrue();
+        assertThat(repair_retries.getMaxAttempts()).isEqualTo(3);
+        RetrySpec spec = repair_retries.getMerkleTreeResponseSpec();
+        assertThat(spec.isEnabled()).isTrue();
+        assertThat(spec.getMaxAttempts()).isEqualTo(3);
+    }
+
+    @Test
+    public void repairRetryOverride()
+    {
+        RepairRetrySpec repair_retries = loadRepairRetry(ImmutableMap.of(
+        "merkle_tree_response", ImmutableMap.of("max_attempts", 10,
+                                                "base_sleep_time", "1s",
+                                                "max_sleep_time", "10s")
+        ));
+        assertThat(repair_retries.isEnabled()).isFalse();
+        assertThat(repair_retries.getMaxAttempts()).isNull();
+        assertThat(repair_retries.baseSleepTime).isEqualTo(RetrySpec.DEFAULT_BASE_SLEEP);
+        assertThat(repair_retries.maxSleepTime).isEqualTo(RetrySpec.DEFAULT_MAX_SLEEP);
+
+        RetrySpec spec = repair_retries.getMerkleTreeResponseSpec();
+        assertThat(spec.isEnabled()).isTrue();
+        assertThat(spec.maxAttempts).isEqualTo(10);
+        assertThat(spec.baseSleepTime).isEqualTo(RetrySpec.DEFAULT_MAX_SLEEP);
+        assertThat(spec.maxSleepTime).isEqualTo(new DurationSpec.LongMillisecondsBound("10s"));
+    }
+
+    private static RepairRetrySpec loadRepairRetry(Map<String, Object> map)
+    {
+        return YamlConfigurationLoader.fromMap(ImmutableMap.of("repair", ImmutableMap.of("retries", map)), true, Config.class).repair.retries;
+    }
+
     @Test
     public void validateTypes()
     {