diff --git a/CHANGES.txt b/CHANGES.txt index 20588112aa..83ffa9a7d7 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -3,6 +3,7 @@ * Add the ability to disable bulk loading of SSTables (CASSANDRA-18781) * Clean up obsolete functions and simplify cql_version handling in cqlsh (CASSANDRA-18787) Merged from 5.0: + * Add support for repair coordinator to retry messages that timeout (CASSANDRA-18816) * Upgrade slf4j-api to 1.7.36 (CASSANDRA-18882) * Make the output of ON/OFF commands in cqlsh consistent (CASSANDRA-18547) * Do not create sstable files before registering in txn (CASSANDRA-18737) diff --git a/conf/cassandra.yaml b/conf/cassandra.yaml index 1e861a2144..f8e865cf05 100644 --- a/conf/cassandra.yaml +++ b/conf/cassandra.yaml @@ -723,6 +723,17 @@ memtable_allocation_type: heap_buffers # Min unit: MiB # repair_session_space: +# repair: +# # Configure the retries for each of the repair messages that support it. As of this moment retries use an exponential algorithm where each attempt sleeps longer based off the base_sleep_time and attempt. +# retries: +# max_attempts: 10 +# base_sleep_time: 200ms +# max_sleep_time: 1s +# # Increase the timeout of validation responses due to them containing the merkle tree +# merkle_tree_response: +# base_sleep_time: 30s +# max_sleep_time: 1m + # Total space to use for commit logs on disk. # # If space gets above this value, Cassandra will flush every dirty CF diff --git a/src/java/org/apache/cassandra/concurrent/Stage.java b/src/java/org/apache/cassandra/concurrent/Stage.java index ac609aa774..05b6c612fa 100644 --- a/src/java/org/apache/cassandra/concurrent/Stage.java +++ b/src/java/org/apache/cassandra/concurrent/Stage.java @@ -144,6 +144,12 @@ public enum Stage return executor; } + @VisibleForTesting + public void unsafeSetExecutor(ExecutorPlus executor) + { + this.executor = executor; + } + private static List executors() { return Stream.of(Stage.values()) diff --git a/src/java/org/apache/cassandra/config/Config.java b/src/java/org/apache/cassandra/config/Config.java index d7fc1df3fe..9f195ac606 100644 --- a/src/java/org/apache/cassandra/config/Config.java +++ b/src/java/org/apache/cassandra/config/Config.java @@ -1106,6 +1106,8 @@ public class Config public volatile long min_tracked_partition_tombstone_count = 5000; public volatile boolean top_partitions_enabled = true; + public final RepairConfig repair = new RepairConfig(); + /** * Default compaction configuration, used if a table does not specify any. */ diff --git a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java index 298ed3f8f9..e6d62a9ff9 100644 --- a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java +++ b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java @@ -3695,6 +3695,12 @@ public class DatabaseDescriptor return localDC; } + @VisibleForTesting + public static void setLocalDataCenter(String value) + { + localDC = value; + } + public static Comparator getLocalComparator() { return localComparator; @@ -4906,4 +4912,9 @@ public class DatabaseDescriptor { return conf.sai_options.segment_write_buffer_size; } + + public static RepairRetrySpec getRepairRetrySpec() + { + return conf == null ? new RepairRetrySpec() : conf.repair.retries; + } } diff --git a/src/java/org/apache/cassandra/config/RepairConfig.java b/src/java/org/apache/cassandra/config/RepairConfig.java new file mode 100644 index 0000000000..4912f51dd3 --- /dev/null +++ b/src/java/org/apache/cassandra/config/RepairConfig.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.config; + +public class RepairConfig +{ + public final RepairRetrySpec retries = new RepairRetrySpec(); +} diff --git a/src/java/org/apache/cassandra/config/RepairRetrySpec.java b/src/java/org/apache/cassandra/config/RepairRetrySpec.java new file mode 100644 index 0000000000..42900dee01 --- /dev/null +++ b/src/java/org/apache/cassandra/config/RepairRetrySpec.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.config; + +import com.fasterxml.jackson.annotation.JsonIgnore; + +public class RepairRetrySpec extends RetrySpec +{ + public RetrySpec.Partial merkle_tree_response = null; + + public boolean isMerkleTreeRetriesEnabled() + { + RetrySpec.Partial partial = merkle_tree_response; + if (partial == null || partial.maxAttempts == null) + return isEnabled(); + return partial.isEnabled(); + } + + @JsonIgnore + public RetrySpec getMerkleTreeResponseSpec() + { + RetrySpec.Partial partial = merkle_tree_response; + if (partial == null) + return this; + return partial.withDefaults(this); + } +} diff --git a/src/java/org/apache/cassandra/config/RetrySpec.java b/src/java/org/apache/cassandra/config/RetrySpec.java new file mode 100644 index 0000000000..4f113af962 --- /dev/null +++ b/src/java/org/apache/cassandra/config/RetrySpec.java @@ -0,0 +1,165 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.config; + +import java.util.Objects; + +import javax.annotation.Nullable; + +import org.apache.cassandra.config.DurationSpec.LongMillisecondsBound; + +public class RetrySpec +{ + public static class MaxAttempt + { + public static final MaxAttempt DISABLED = new MaxAttempt(); + + public final int value; + + public MaxAttempt(int value) + { + if (value < 1) + throw new IllegalArgumentException("max attempt must be positive; but given " + value); + this.value = value; + } + + private MaxAttempt() + { + value = 0; + } + + @Override + public boolean equals(Object o) + { + if (this == o) return true; + if (o == null) return false; + if (o instanceof Integer) return this.value == ((Integer) o).intValue(); + if (getClass() != o.getClass()) return false; + MaxAttempt that = (MaxAttempt) o; + return value == that.value; + } + + @Override + public int hashCode() + { + return Objects.hash(value); + } + + @Override + public String toString() + { + return Integer.toString(value); + } + } + + public static class Partial extends RetrySpec + { + public Partial() + { + this.maxAttempts = null; + this.baseSleepTime = null; + this.maxSleepTime = null; + } + + public RetrySpec withDefaults(RetrySpec defaultValues) + { + MaxAttempt maxAttempts = nonNull(this.maxAttempts, defaultValues.getMaxAttempts(), DEFAULT_MAX_ATTEMPTS); + LongMillisecondsBound baseSleepTime = nonNull(this.baseSleepTime, defaultValues.getBaseSleepTime(), DEFAULT_BASE_SLEEP); + LongMillisecondsBound maxSleepTime = nonNull(this.maxSleepTime, defaultValues.getMaxSleepTime(), DEFAULT_MAX_SLEEP); + return new RetrySpec(maxAttempts, baseSleepTime, maxSleepTime); + } + + private static T nonNull(@Nullable T left, @Nullable T right, T defaultValue) + { + if (left != null) + return left; + if (right != null) + return right; + return defaultValue; + } + } + + public static final MaxAttempt DEFAULT_MAX_ATTEMPTS = MaxAttempt.DISABLED; + public static final LongMillisecondsBound DEFAULT_BASE_SLEEP = new LongMillisecondsBound("200ms"); + public static final LongMillisecondsBound DEFAULT_MAX_SLEEP = new LongMillisecondsBound("1s"); + + /** + * Represents how many retry attempts are allowed. If the value is 2, this will cause 2 retries + 1 original request, for a total of 3 requests! + *

+ * To disable, set to 0. + */ + public MaxAttempt maxAttempts = DEFAULT_MAX_ATTEMPTS; // 2 retries, 1 original request; so 3 total + public LongMillisecondsBound baseSleepTime = DEFAULT_BASE_SLEEP; + public LongMillisecondsBound maxSleepTime = DEFAULT_MAX_SLEEP; + + public RetrySpec() + { + } + + public RetrySpec(MaxAttempt maxAttempts, LongMillisecondsBound baseSleepTime, LongMillisecondsBound maxSleepTime) + { + this.maxAttempts = maxAttempts; + this.baseSleepTime = baseSleepTime; + this.maxSleepTime = maxSleepTime; + } + + public boolean isEnabled() + { + return maxAttempts != MaxAttempt.DISABLED; + } + + public void setEnabled(boolean enabled) + { + if (!enabled) + { + maxAttempts = MaxAttempt.DISABLED; + } + else if (maxAttempts == MaxAttempt.DISABLED) + { + maxAttempts = new MaxAttempt(2); + } + } + + @Nullable + public MaxAttempt getMaxAttempts() + { + return !isEnabled() ? null : maxAttempts; + } + + @Nullable + public LongMillisecondsBound getBaseSleepTime() + { + return !isEnabled() ? null : baseSleepTime; + } + + public LongMillisecondsBound getMaxSleepTime() + { + return !isEnabled() ? null : maxSleepTime; + } + + @Override + public String toString() + { + return "RetrySpec{" + + "maxAttempts=" + maxAttempts + + ", baseSleepTime=" + baseSleepTime + + ", maxSleepTime=" + maxSleepTime + + '}'; + } +} diff --git a/src/java/org/apache/cassandra/cql3/QueryProcessor.java b/src/java/org/apache/cassandra/cql3/QueryProcessor.java index 02369f373a..fba424d7d9 100644 --- a/src/java/org/apache/cassandra/cql3/QueryProcessor.java +++ b/src/java/org/apache/cassandra/cql3/QueryProcessor.java @@ -478,7 +478,7 @@ public class QueryProcessor implements QueryHandler } Future>> future = FutureCombiner.allOf(commands.stream() .map(rc -> Message.out(rc.verb(), rc)) - .map(m -> MessagingService.instance().sendWithResult(m, address)) + .map(m -> MessagingService.instance().sendWithResult(m, address)) .collect(Collectors.toList())); ResultSetBuilder result = new ResultSetBuilder(select.getResultMetadata(), select.getSelection().newSelectors(options), false); diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java index 97a4a27bdf..856542c946 100644 --- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java +++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java @@ -2726,7 +2726,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner // since truncation can happen at different times on different nodes, we need to make sure // that any repairs are aborted, otherwise we might clear the data on one node and then // stream in data that is actually supposed to have been deleted - ActiveRepairService.instance.abort((prs) -> prs.getTableIds().contains(metadata.id), + ActiveRepairService.instance().abort((prs) -> prs.getTableIds().contains(metadata.id), "Stopping parent sessions {} due to truncation of tableId="+metadata.id); data.notifyTruncated(truncatedAt); diff --git a/src/java/org/apache/cassandra/db/Keyspace.java b/src/java/org/apache/cassandra/db/Keyspace.java index a3f05e6048..b49eb3162a 100644 --- a/src/java/org/apache/cassandra/db/Keyspace.java +++ b/src/java/org/apache/cassandra/db/Keyspace.java @@ -674,7 +674,7 @@ public class Keyspace public Iterable getValidColumnFamilies(boolean allowIndexes, boolean autoAddIndexes, - String... cfNames) throws IOException + String... cfNames) { Set valid = new HashSet<>(); diff --git a/src/java/org/apache/cassandra/db/ReadCommand.java b/src/java/org/apache/cassandra/db/ReadCommand.java index 2ca47ea06f..671f400081 100644 --- a/src/java/org/apache/cassandra/db/ReadCommand.java +++ b/src/java/org/apache/cassandra/db/ReadCommand.java @@ -981,13 +981,13 @@ public abstract class ReadCommand extends AbstractReadQuery TimeUUID pendingRepair = sstable.getPendingRepair(); if (pendingRepair != ActiveRepairService.NO_PENDING_REPAIR) { - if (ActiveRepairService.instance.consistent.local.isSessionFinalized(pendingRepair)) + if (ActiveRepairService.instance().consistent.local.isSessionFinalized(pendingRepair)) return true; // In the edge case where compaction is backed up long enough for the session to // timeout and be purged by LocalSessions::cleanup, consider the sstable unrepaired // as it will be marked unrepaired when compaction catches up - if (!ActiveRepairService.instance.consistent.local.sessionExists(pendingRepair)) + if (!ActiveRepairService.instance().consistent.local.sessionExists(pendingRepair)) return false; repairedDataInfo.markInconclusive(); diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionManager.java b/src/java/org/apache/cassandra/db/compaction/CompactionManager.java index 44d6f4b833..ad68a09808 100644 --- a/src/java/org/apache/cassandra/db/compaction/CompactionManager.java +++ b/src/java/org/apache/cassandra/db/compaction/CompactionManager.java @@ -135,7 +135,7 @@ import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID; * a set via Tracker. New scheduling attempts will ignore currently compacting * sstables. */ -public class CompactionManager implements CompactionManagerMBean +public class CompactionManager implements CompactionManagerMBean, ICompactionManager { public static final String MBEAN_OBJECT_NAME = "org.apache.cassandra.db:type=CompactionManager"; private static final Logger logger = LoggerFactory.getLogger(CompactionManager.class); @@ -893,7 +893,7 @@ public class CompactionManager implements CompactionManagerMBean ActiveRepairService.ParentRepairSession prs; try { - prs = ActiveRepairService.instance.getParentRepairSession(sessionID); + prs = ActiveRepairService.instance().getParentRepairSession(sessionID); } catch (NoSuchRepairSessionException e) { @@ -2193,6 +2193,7 @@ public class CompactionManager implements CompactionManagerMBean return metrics.totalCompactionsCompleted.getCount(); } + @Override public int getPendingTasks() { return metrics.pendingTasks.getValue(); diff --git a/src/java/org/apache/cassandra/db/compaction/ICompactionManager.java b/src/java/org/apache/cassandra/db/compaction/ICompactionManager.java new file mode 100644 index 0000000000..e8e7c6830e --- /dev/null +++ b/src/java/org/apache/cassandra/db/compaction/ICompactionManager.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.db.compaction; + +public interface ICompactionManager +{ + int getPendingTasks(); +} diff --git a/src/java/org/apache/cassandra/db/compaction/PendingRepairManager.java b/src/java/org/apache/cassandra/db/compaction/PendingRepairManager.java index 87915c964a..2ddfab7060 100644 --- a/src/java/org/apache/cassandra/db/compaction/PendingRepairManager.java +++ b/src/java/org/apache/cassandra/db/compaction/PendingRepairManager.java @@ -272,7 +272,7 @@ class PendingRepairManager if (compactionStrategy == null) return null; Set sstables = compactionStrategy.getSSTables(); - long repairedAt = ActiveRepairService.instance.consistent.local.getFinalSessionRepairedAt(sessionID); + long repairedAt = ActiveRepairService.instance().consistent.local.getFinalSessionRepairedAt(sessionID); LifecycleTransaction txn = cfs.getTracker().tryModify(sstables, OperationType.COMPACTION); return txn == null ? null : new RepairFinishedCompactionTask(cfs, txn, sessionID, repairedAt); } @@ -426,7 +426,7 @@ class PendingRepairManager boolean canCleanup(TimeUUID sessionID) { - return !ActiveRepairService.instance.consistent.local.isSessionInProgress(sessionID); + return !ActiveRepairService.instance().consistent.local.isSessionInProgress(sessionID); } @SuppressWarnings("resource") diff --git a/src/java/org/apache/cassandra/db/repair/CassandraTableRepairManager.java b/src/java/org/apache/cassandra/db/repair/CassandraTableRepairManager.java index 053342e12b..4e54d6ee77 100644 --- a/src/java/org/apache/cassandra/db/repair/CassandraTableRepairManager.java +++ b/src/java/org/apache/cassandra/db/repair/CassandraTableRepairManager.java @@ -32,6 +32,7 @@ import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.metrics.TopPartitionTracker; +import org.apache.cassandra.repair.SharedContext; import org.apache.cassandra.repair.TableRepairManager; import org.apache.cassandra.repair.ValidationPartitionIterator; import org.apache.cassandra.repair.NoSuchRepairSessionException; @@ -41,16 +42,23 @@ import org.apache.cassandra.service.ActiveRepairService; public class CassandraTableRepairManager implements TableRepairManager { private final ColumnFamilyStore cfs; + private final SharedContext ctx; public CassandraTableRepairManager(ColumnFamilyStore cfs) + { + this(cfs, SharedContext.Global.instance); + } + + public CassandraTableRepairManager(ColumnFamilyStore cfs, SharedContext ctx) { this.cfs = cfs; + this.ctx = ctx; } @Override public ValidationPartitionIterator getValidationIterator(Collection> ranges, TimeUUID parentId, TimeUUID sessionID, boolean isIncremental, long nowInSec, TopPartitionTracker.Collector topPartitionCollector) throws IOException, NoSuchRepairSessionException { - return new CassandraValidationIterator(cfs, ranges, parentId, sessionID, isIncremental, nowInSec, topPartitionCollector); + return new CassandraValidationIterator(cfs, ctx, ranges, parentId, sessionID, isIncremental, nowInSec, topPartitionCollector); } @Override @@ -70,7 +78,7 @@ public class CassandraTableRepairManager implements TableRepairManager { try { - ActiveRepairService.instance.snapshotExecutor.submit(() -> { + ActiveRepairService.instance().snapshotExecutor.submit(() -> { if (force || !cfs.snapshotExists(name)) { cfs.snapshot(name, new Predicate() diff --git a/src/java/org/apache/cassandra/db/repair/CassandraValidationIterator.java b/src/java/org/apache/cassandra/db/repair/CassandraValidationIterator.java index 4e4f5600b2..a31a7038b0 100644 --- a/src/java/org/apache/cassandra/db/repair/CassandraValidationIterator.java +++ b/src/java/org/apache/cassandra/db/repair/CassandraValidationIterator.java @@ -52,6 +52,7 @@ import org.apache.cassandra.dht.Token; import org.apache.cassandra.io.sstable.ISSTableScanner; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.metrics.TopPartitionTracker; +import org.apache.cassandra.repair.SharedContext; import org.apache.cassandra.repair.ValidationPartitionIterator; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.service.ActiveRepairService; @@ -112,11 +113,11 @@ public class CassandraValidationIterator extends ValidationPartitionIterator } @VisibleForTesting - public static synchronized Refs getSSTablesToValidate(ColumnFamilyStore cfs, Collection> ranges, TimeUUID parentId, boolean isIncremental) throws NoSuchRepairSessionException + public static synchronized Refs getSSTablesToValidate(ColumnFamilyStore cfs, SharedContext ctx, Collection> ranges, TimeUUID parentId, boolean isIncremental) throws NoSuchRepairSessionException { Refs sstables; - ActiveRepairService.ParentRepairSession prs = ActiveRepairService.instance.getParentRepairSession(parentId); + ActiveRepairService.ParentRepairSession prs = ctx.repair().getParentRepairSession(parentId); Set sstablesToValidate = new HashSet<>(); @@ -158,6 +159,7 @@ public class CassandraValidationIterator extends ValidationPartitionIterator } private final ColumnFamilyStore cfs; + private final SharedContext ctx; private final Refs sstables; private final String snapshotName; private final boolean isGlobalSnapshotValidation; @@ -172,9 +174,10 @@ public class CassandraValidationIterator extends ValidationPartitionIterator private final long estimatedPartitions; private final Map, Long> rangePartitionCounts; - public CassandraValidationIterator(ColumnFamilyStore cfs, Collection> ranges, TimeUUID parentId, TimeUUID sessionID, boolean isIncremental, long nowInSec, TopPartitionTracker.Collector topPartitionCollector) throws IOException, NoSuchRepairSessionException + public CassandraValidationIterator(ColumnFamilyStore cfs, SharedContext ctx, Collection> ranges, TimeUUID parentId, TimeUUID sessionID, boolean isIncremental, long nowInSec, TopPartitionTracker.Collector topPartitionCollector) throws IOException, NoSuchRepairSessionException { this.cfs = cfs; + this.ctx = ctx; isGlobalSnapshotValidation = cfs.snapshotExists(parentId.toString()); if (isGlobalSnapshotValidation) @@ -198,7 +201,7 @@ public class CassandraValidationIterator extends ValidationPartitionIterator cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.VALIDATION); // Note: we also flush for incremental repair during the anti-compaction process. } - sstables = getSSTablesToValidate(cfs, ranges, parentId, isIncremental); + sstables = getSSTablesToValidate(cfs, ctx, ranges, parentId, isIncremental); } // Persistent memtables will not flush or snapshot to sstables, make an sstable with their data. @@ -208,7 +211,7 @@ public class CassandraValidationIterator extends ValidationPartitionIterator Preconditions.checkArgument(sstables != null); - ActiveRepairService.ParentRepairSession prs = ActiveRepairService.instance.getParentRepairSession(parentId); + ActiveRepairService.ParentRepairSession prs = ctx.repair().getParentRepairSession(parentId); logger.info("{}, parentSessionId={}: Performing validation compaction on {} sstables in {}.{}", prs.previewKind.logPrefix(sessionID), parentId, diff --git a/src/java/org/apache/cassandra/db/repair/PendingAntiCompaction.java b/src/java/org/apache/cassandra/db/repair/PendingAntiCompaction.java index 55138e8277..2711ace7da 100644 --- a/src/java/org/apache/cassandra/db/repair/PendingAntiCompaction.java +++ b/src/java/org/apache/cassandra/db/repair/PendingAntiCompaction.java @@ -137,7 +137,7 @@ public class PendingAntiCompaction // non-finalized sessions for a later error message if (metadata.pendingRepair != NO_PENDING_REPAIR) { - if (!ActiveRepairService.instance.consistent.local.isSessionFinalized(metadata.pendingRepair)) + if (!ActiveRepairService.instance().consistent.local.isSessionFinalized(metadata.pendingRepair)) { String message = String.format("Prepare phase for incremental repair session %s has failed because it encountered " + "intersecting sstables belonging to another incremental repair session (%s). This is " + diff --git a/src/java/org/apache/cassandra/db/virtual/LocalRepairTables.java b/src/java/org/apache/cassandra/db/virtual/LocalRepairTables.java index 1f440abc72..b88502988e 100644 --- a/src/java/org/apache/cassandra/db/virtual/LocalRepairTables.java +++ b/src/java/org/apache/cassandra/db/virtual/LocalRepairTables.java @@ -27,6 +27,7 @@ import java.util.UUID; import java.util.stream.Collectors; import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableSet; import org.apache.cassandra.cql3.statements.schema.CreateTableStatement; import org.apache.cassandra.db.DecoratedKey; @@ -115,7 +116,7 @@ public class LocalRepairTables public DataSet data() { SimpleDataSet result = new SimpleDataSet(metadata()); - ActiveRepairService.instance.coordinators().forEach(s -> updateDataset(result, s)); + ActiveRepairService.instance().coordinators().forEach(s -> updateDataset(result, s)); return result; } @@ -123,7 +124,7 @@ public class LocalRepairTables { TimeUUID id = TimeUUIDType.instance.compose(partitionKey.getKey()); SimpleDataSet result = new SimpleDataSet(metadata()); - CoordinatorState state = ActiveRepairService.instance.coordinator(id); + CoordinatorState state = ActiveRepairService.instance().coordinator(id); if (state != null) updateDataset(result, state); return result; @@ -212,7 +213,7 @@ public class LocalRepairTables public DataSet data() { SimpleDataSet result = new SimpleDataSet(metadata()); - ActiveRepairService.instance.coordinators().stream() + ActiveRepairService.instance().coordinators().stream() .flatMap(s -> s.getSessions().stream()) .forEach(s -> updateDataset(result, s)); return result; @@ -251,7 +252,7 @@ public class LocalRepairTables public DataSet data() { SimpleDataSet result = new SimpleDataSet(metadata()); - ActiveRepairService.instance.coordinators().stream() + ActiveRepairService.instance().coordinators().stream() .flatMap(s -> s.getSessions().stream()) .flatMap(s -> s.getJobs().stream()) .forEach(s -> updateDataset(result, s)); @@ -291,7 +292,7 @@ public class LocalRepairTables public DataSet data() { SimpleDataSet result = new SimpleDataSet(metadata()); - ActiveRepairService.instance.participates().stream() + ActiveRepairService.instance().participates().stream() .forEach(s -> updateDataset(result, s)); return result; } @@ -301,7 +302,7 @@ public class LocalRepairTables { TimeUUID id = TimeUUIDType.instance.compose(partitionKey.getKey()); SimpleDataSet result = new SimpleDataSet(metadata()); - ParticipateState state = ActiveRepairService.instance.participate(id); + ParticipateState state = ActiveRepairService.instance().participate(id); if (state != null) updateDataset(result, state); return result; @@ -322,7 +323,7 @@ public class LocalRepairTables result.column("preview_kind", state.previewKind.name()); if (state.repairedAt != 0) result.column("repaired_at", new Date(state.repairedAt)); - result.column("validations", state.validationIds()); + result.column("validations", ImmutableSet.copyOf(state.validationIds())); result.column("ranges", toStringList(state.ranges)); } } @@ -352,7 +353,7 @@ public class LocalRepairTables public DataSet data() { SimpleDataSet result = new SimpleDataSet(metadata()); - ActiveRepairService.instance.validations().stream() + ActiveRepairService.instance().validations().stream() .forEach(s -> updateDataset(result, s)); return result; } @@ -362,7 +363,7 @@ public class LocalRepairTables { UUID id = UUIDType.instance.compose(partitionKey.getKey()); SimpleDataSet result = new SimpleDataSet(metadata()); - ValidationState state = ActiveRepairService.instance.validation(id); + ValidationState state = ActiveRepairService.instance().validation(id); if (state != null) updateDataset(result, state); return result; diff --git a/src/java/org/apache/cassandra/exceptions/RepairException.java b/src/java/org/apache/cassandra/exceptions/RepairException.java index eb93c37192..554bdfcee4 100644 --- a/src/java/org/apache/cassandra/exceptions/RepairException.java +++ b/src/java/org/apache/cassandra/exceptions/RepairException.java @@ -17,6 +17,8 @@ */ package org.apache.cassandra.exceptions; +import javax.annotation.Nullable; + import org.apache.cassandra.repair.RepairJobDesc; import org.apache.cassandra.streaming.PreviewKind; @@ -27,9 +29,9 @@ public class RepairException extends Exception { private final boolean shouldLogWarn; - private RepairException(RepairJobDesc desc, PreviewKind previewKind, String message, boolean shouldLogWarn) + private RepairException(@Nullable RepairJobDesc desc, PreviewKind previewKind, String message, boolean shouldLogWarn) { - this(desc.toString(previewKind != null ? previewKind : PreviewKind.NONE) + ' ' + message, shouldLogWarn); + this((desc == null ? "" : desc.toString(previewKind != null ? previewKind : PreviewKind.NONE)) + ' ' + message, shouldLogWarn); } private RepairException(String msg, boolean shouldLogWarn) @@ -38,12 +40,12 @@ public class RepairException extends Exception this.shouldLogWarn = shouldLogWarn; } - public static RepairException error(RepairJobDesc desc, PreviewKind previewKind, String message) + public static RepairException error(@Nullable RepairJobDesc desc, PreviewKind previewKind, String message) { return new RepairException(desc, previewKind, message, false); } - public static RepairException warn(RepairJobDesc desc, PreviewKind previewKind, String message) + public static RepairException warn(@Nullable RepairJobDesc desc, PreviewKind previewKind, String message) { return new RepairException(desc, previewKind, message, true); } diff --git a/src/java/org/apache/cassandra/gms/Gossiper.java b/src/java/org/apache/cassandra/gms/Gossiper.java index e51d8f7ff0..43ee0aeea5 100644 --- a/src/java/org/apache/cassandra/gms/Gossiper.java +++ b/src/java/org/apache/cassandra/gms/Gossiper.java @@ -116,7 +116,7 @@ import static org.apache.cassandra.utils.FBUtilities.getBroadcastAddressAndPort; * This class is not threadsafe and any state changes should happen in the gossip stage. */ -public class Gossiper implements IFailureDetectionEventListener, GossiperMBean +public class Gossiper implements IFailureDetectionEventListener, GossiperMBean, IGossiper { public static final String MBEAN_NAME = "org.apache.cassandra.net:type=Gossiper"; @@ -448,6 +448,7 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean * * @param subscriber module which implements the IEndpointStateChangeSubscriber */ + @Override public void register(IEndpointStateChangeSubscriber subscriber) { subscribers.add(subscriber); @@ -458,6 +459,7 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean * * @param subscriber module which implements the IEndpointStateChangeSubscriber */ + @Override public void unregister(IEndpointStateChangeSubscriber subscriber) { subscribers.remove(subscriber); @@ -1147,6 +1149,7 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean return storedTime == null ? computeExpireTime() : storedTime; } + @Override public EndpointState getEndpointStateForEndpoint(InetAddressAndPort ep) { return endpointStateMap.get(ep); @@ -2319,13 +2322,6 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean return currentTimeMillis() + aVeryLongTime; } - @Nullable - public CassandraVersion getReleaseVersion(InetAddressAndPort ep) - { - EndpointState state = getEndpointStateForEndpoint(ep); - return state != null ? state.getReleaseVersion() : null; - } - public Map> getReleaseVersionsWithPort() { Map> results = new HashMap<>(); diff --git a/src/java/org/apache/cassandra/gms/IGossiper.java b/src/java/org/apache/cassandra/gms/IGossiper.java new file mode 100644 index 0000000000..0e33526d22 --- /dev/null +++ b/src/java/org/apache/cassandra/gms/IGossiper.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.gms; + +import javax.annotation.Nullable; + +import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.utils.CassandraVersion; + +public interface IGossiper +{ + void register(IEndpointStateChangeSubscriber subscriber); + void unregister(IEndpointStateChangeSubscriber subscriber); + + @Nullable + EndpointState getEndpointStateForEndpoint(InetAddressAndPort ep); + @Nullable + default CassandraVersion getReleaseVersion(InetAddressAndPort ep) + { + EndpointState state = getEndpointStateForEndpoint(ep); + return state != null ? state.getReleaseVersion() : null; + } +} diff --git a/src/java/org/apache/cassandra/net/Message.java b/src/java/org/apache/cassandra/net/Message.java index da50727d6f..c60ebee972 100644 --- a/src/java/org/apache/cassandra/net/Message.java +++ b/src/java/org/apache/cassandra/net/Message.java @@ -102,7 +102,7 @@ public class Message return header.verb; } - boolean isFailureResponse() + public boolean isFailureResponse() { return verb() == Verb.FAILURE_RSP; } @@ -301,6 +301,11 @@ public class Message return new Message<>(header.withParam(ParamType.FORWARD_TO, peers), payload); } + public Message withFrom(InetAddressAndPort from) + { + return new Message<>(header.withFrom(from), payload); + } + public Message withFlag(MessageFlag flag) { return new Message<>(header.withFlag(flag), payload); @@ -434,6 +439,11 @@ public class Message this.params = params; } + Header withFrom(InetAddressAndPort from) + { + return new Header(id, verb, from, createdAtNanos, expiresAtNanos, flags, params); + } + Header withFlag(MessageFlag flag) { return new Header(id, verb, from, createdAtNanos, expiresAtNanos, flag.addTo(flags), params); diff --git a/src/java/org/apache/cassandra/net/MessageDelivery.java b/src/java/org/apache/cassandra/net/MessageDelivery.java new file mode 100644 index 0000000000..dd8c6ceeda --- /dev/null +++ b/src/java/org/apache/cassandra/net/MessageDelivery.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.net; + +import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.utils.concurrent.Future; + +public interface MessageDelivery +{ + public void send(Message message, InetAddressAndPort to); + public void sendWithCallback(Message message, InetAddressAndPort to, RequestCallback cb); + public void sendWithCallback(Message message, InetAddressAndPort to, RequestCallback cb, ConnectionType specifyConnection); + public Future> sendWithResult(Message message, InetAddressAndPort to); + public void respond(V response, Message message); +} diff --git a/src/java/org/apache/cassandra/net/MessagingService.java b/src/java/org/apache/cassandra/net/MessagingService.java index a3ae7d8d09..c306fd87f6 100644 --- a/src/java/org/apache/cassandra/net/MessagingService.java +++ b/src/java/org/apache/cassandra/net/MessagingService.java @@ -205,11 +205,43 @@ import static org.apache.cassandra.utils.Throwables.maybeFail; * implemented in {@link org.apache.cassandra.db.virtual.InternodeInboundTable} and * {@link org.apache.cassandra.db.virtual.InternodeOutboundTable} respectively. */ -public class MessagingService extends MessagingServiceMBeanImpl +public class MessagingService extends MessagingServiceMBeanImpl implements MessageDelivery { private static final Logger logger = LoggerFactory.getLogger(MessagingService.class); // 8 bits version, so don't waste versions + public enum Version + { + @Deprecated + VERSION_30(10), + @Deprecated + VERSION_3014(11), + VERSION_40(12), + // c14227 TTL overflow, 'uint' timestamps + VERSION_50(13); + + public static final Version CURRENT = DatabaseDescriptor.getStorageCompatibilityMode().isBefore(5) ? VERSION_40 : VERSION_50; + + public final int value; + + Version(int value) + { + this.value = value; + } + + public static List supportedVersions() + { + List versions = Lists.newArrayList(); + for (Version version : values()) + if (minimum_version <= version.value) + versions.add(version); + + return Collections.unmodifiableList(versions); + } + } + // Maintance Note: + // Try to keep Version enum in-sync for testing. By having the versions in the enum tests can get access without forcing this class + // to load, which adds a lot of costs to each test @Deprecated public static final int VERSION_30 = 10; @Deprecated @@ -217,7 +249,7 @@ public class MessagingService extends MessagingServiceMBeanImpl public static final int VERSION_40 = 12; public static final int VERSION_50 = 13; // c14227 TTL overflow, 'uint' timestamps public static final int minimum_version = VERSION_40; - public static final int current_version = DatabaseDescriptor.getStorageCompatibilityMode().isBefore(5) ? VERSION_40 : VERSION_50; + public static final int current_version = Version.CURRENT.value; static AcceptVersions accept_messaging = new AcceptVersions(minimum_version, current_version); static AcceptVersions accept_streaming = new AcceptVersions(current_version, current_version); static Map versionOrdinalMap = Arrays.stream(Version.values()).collect(Collectors.toMap(v -> v.value, v -> v.ordinal())); @@ -238,33 +270,6 @@ public class MessagingService extends MessagingServiceMBeanImpl return ordinal; } - public enum Version - { - @Deprecated - VERSION_30(10), - @Deprecated - VERSION_3014(11), - VERSION_40(12), - VERSION_50(13); - - public final int value; - - Version(int value) - { - this.value = value; - } - - public static List supportedVersions() - { - List versions = Lists.newArrayList(); - for (Version version : values()) - if (minimum_version <= version.value) - versions.add(version); - - return Collections.unmodifiableList(versions); - } - } - private static class MSHandle { public static final MessagingService instance = new MessagingService(false); @@ -312,13 +317,14 @@ public class MessagingService extends MessagingServiceMBeanImpl OutboundConnections.scheduleUnusedConnectionMonitoring(this, ScheduledExecutors.scheduledTasks, 1L, TimeUnit.HOURS); } - public org.apache.cassandra.utils.concurrent.Future> sendWithResult(Message message, InetAddressAndPort to) + @Override + public org.apache.cassandra.utils.concurrent.Future> sendWithResult(Message message, InetAddressAndPort to) { - AsyncPromise> promise = new AsyncPromise<>(); - MessagingService.instance().sendWithCallback(message, to, new RequestCallback() + AsyncPromise> promise = new AsyncPromise<>(); + sendWithCallback(message, to, new RequestCallback() { @Override - public void onResponse(Message msg) + public void onResponse(Message msg) { promise.trySuccess(msg); } diff --git a/src/java/org/apache/cassandra/net/Verb.java b/src/java/org/apache/cassandra/net/Verb.java index e5b4983d77..2481674f09 100644 --- a/src/java/org/apache/cassandra/net/Verb.java +++ b/src/java/org/apache/cassandra/net/Verb.java @@ -159,22 +159,23 @@ public enum Verb SCHEMA_VERSION_REQ (20, P1, rpcTimeout, MIGRATION, () -> NoPayload.serializer, () -> SchemaVersionVerbHandler.instance, SCHEMA_VERSION_RSP ), // repair; mostly doesn't use callbacks and sends responses as their own request messages, with matching sessions by uuid; should eventually harmonize and make idiomatic + // for the repair messages that implement retry logic, use rpcTimeout so the single request fails faster, then retries can be used to recover REPAIR_RSP (100, P1, repairTimeout, REQUEST_RESPONSE, () -> NoPayload.serializer, () -> ResponseVerbHandler.instance ), - VALIDATION_RSP (102, P1, longTimeout, ANTI_ENTROPY, () -> ValidationResponse.serializer, () -> RepairMessageVerbHandler.instance, REPAIR_RSP ), - VALIDATION_REQ (101, P1, repairTimeout, ANTI_ENTROPY, () -> ValidationRequest.serializer, () -> RepairMessageVerbHandler.instance, REPAIR_RSP ), - SYNC_RSP (104, P1, repairTimeout, ANTI_ENTROPY, () -> SyncResponse.serializer, () -> RepairMessageVerbHandler.instance, REPAIR_RSP ), - SYNC_REQ (103, P1, repairTimeout, ANTI_ENTROPY, () -> SyncRequest.serializer, () -> RepairMessageVerbHandler.instance, REPAIR_RSP ), - PREPARE_MSG (105, P1, repairTimeout, ANTI_ENTROPY, () -> PrepareMessage.serializer, () -> RepairMessageVerbHandler.instance, REPAIR_RSP ), - SNAPSHOT_MSG (106, P1, repairTimeout, ANTI_ENTROPY, () -> SnapshotMessage.serializer, () -> RepairMessageVerbHandler.instance, REPAIR_RSP ), - CLEANUP_MSG (107, P1, repairTimeout, ANTI_ENTROPY, () -> CleanupMessage.serializer, () -> RepairMessageVerbHandler.instance, REPAIR_RSP ), - PREPARE_CONSISTENT_RSP (109, P1, repairTimeout, ANTI_ENTROPY, () -> PrepareConsistentResponse.serializer, () -> RepairMessageVerbHandler.instance, REPAIR_RSP ), - PREPARE_CONSISTENT_REQ (108, P1, repairTimeout, ANTI_ENTROPY, () -> PrepareConsistentRequest.serializer, () -> RepairMessageVerbHandler.instance, REPAIR_RSP ), - FINALIZE_PROPOSE_MSG (110, P1, repairTimeout, ANTI_ENTROPY, () -> FinalizePropose.serializer, () -> RepairMessageVerbHandler.instance, REPAIR_RSP ), - FINALIZE_PROMISE_MSG (111, P1, repairTimeout, ANTI_ENTROPY, () -> FinalizePromise.serializer, () -> RepairMessageVerbHandler.instance, REPAIR_RSP ), - FINALIZE_COMMIT_MSG (112, P1, repairTimeout, ANTI_ENTROPY, () -> FinalizeCommit.serializer, () -> RepairMessageVerbHandler.instance, REPAIR_RSP ), - FAILED_SESSION_MSG (113, P1, repairTimeout, ANTI_ENTROPY, () -> FailSession.serializer, () -> RepairMessageVerbHandler.instance, REPAIR_RSP ), - STATUS_RSP (115, P1, repairTimeout, ANTI_ENTROPY, () -> StatusResponse.serializer, () -> RepairMessageVerbHandler.instance, REPAIR_RSP ), - STATUS_REQ (114, P1, repairTimeout, ANTI_ENTROPY, () -> StatusRequest.serializer, () -> RepairMessageVerbHandler.instance, REPAIR_RSP ), + VALIDATION_RSP (102, P1, repairValidationRspTimeout, ANTI_ENTROPY, () -> ValidationResponse.serializer, () -> RepairMessageVerbHandler.instance(), REPAIR_RSP ), + VALIDATION_REQ (101, P1, repairWithBackoffTimeout, ANTI_ENTROPY, () -> ValidationRequest.serializer, () -> RepairMessageVerbHandler.instance(), REPAIR_RSP ), + SYNC_RSP (104, P1, repairWithBackoffTimeout, ANTI_ENTROPY, () -> SyncResponse.serializer, () -> RepairMessageVerbHandler.instance(), REPAIR_RSP ), + SYNC_REQ (103, P1, repairWithBackoffTimeout, ANTI_ENTROPY, () -> SyncRequest.serializer, () -> RepairMessageVerbHandler.instance(), REPAIR_RSP ), + PREPARE_MSG (105, P1, repairWithBackoffTimeout, ANTI_ENTROPY, () -> PrepareMessage.serializer, () -> RepairMessageVerbHandler.instance(), REPAIR_RSP ), + SNAPSHOT_MSG (106, P1, repairWithBackoffTimeout, ANTI_ENTROPY, () -> SnapshotMessage.serializer, () -> RepairMessageVerbHandler.instance(), REPAIR_RSP ), + CLEANUP_MSG (107, P1, repairWithBackoffTimeout, ANTI_ENTROPY, () -> CleanupMessage.serializer, () -> RepairMessageVerbHandler.instance(), REPAIR_RSP ), + PREPARE_CONSISTENT_RSP (109, P1, repairTimeout, ANTI_ENTROPY, () -> PrepareConsistentResponse.serializer, () -> RepairMessageVerbHandler.instance(), REPAIR_RSP ), + PREPARE_CONSISTENT_REQ (108, P1, repairTimeout, ANTI_ENTROPY, () -> PrepareConsistentRequest.serializer, () -> RepairMessageVerbHandler.instance(), REPAIR_RSP ), + FINALIZE_PROPOSE_MSG (110, P1, repairTimeout, ANTI_ENTROPY, () -> FinalizePropose.serializer, () -> RepairMessageVerbHandler.instance(), REPAIR_RSP ), + FINALIZE_PROMISE_MSG (111, P1, repairTimeout, ANTI_ENTROPY, () -> FinalizePromise.serializer, () -> RepairMessageVerbHandler.instance(), REPAIR_RSP ), + FINALIZE_COMMIT_MSG (112, P1, repairTimeout, ANTI_ENTROPY, () -> FinalizeCommit.serializer, () -> RepairMessageVerbHandler.instance(), REPAIR_RSP ), + FAILED_SESSION_MSG (113, P1, repairTimeout, ANTI_ENTROPY, () -> FailSession.serializer, () -> RepairMessageVerbHandler.instance(), REPAIR_RSP ), + STATUS_RSP (115, P1, repairTimeout, ANTI_ENTROPY, () -> StatusResponse.serializer, () -> RepairMessageVerbHandler.instance(), REPAIR_RSP ), + STATUS_REQ (114, P1, repairTimeout, ANTI_ENTROPY, () -> StatusRequest.serializer, () -> RepairMessageVerbHandler.instance(), REPAIR_RSP ), REPLICATION_DONE_RSP (82, P0, rpcTimeout, MISC, () -> NoPayload.serializer, () -> ResponseVerbHandler.instance ), REPLICATION_DONE_REQ (22, P0, rpcTimeout, MISC, () -> NoPayload.serializer, () -> ReplicationDoneVerbHandler.instance, REPLICATION_DONE_RSP), @@ -475,4 +476,16 @@ class VerbTimeouts static final ToLongFunction pingTimeout = DatabaseDescriptor::getPingTimeout; static final ToLongFunction longTimeout = units -> Math.max(DatabaseDescriptor.getRpcTimeout(units), units.convert(5L, TimeUnit.MINUTES)); static final ToLongFunction noTimeout = units -> { throw new IllegalStateException(); }; + + // repair verbs need to have different timeouts based off if retries are enabled or not + static final ToLongFunction repairWithBackoffTimeout = units -> { + if (!DatabaseDescriptor.getRepairRetrySpec().isEnabled()) + return repairTimeout.applyAsLong(units); + return rpcTimeout.applyAsLong(units); + }; + static final ToLongFunction repairValidationRspTimeout = units -> { + if (!DatabaseDescriptor.getRepairRetrySpec().isMerkleTreeRetriesEnabled()) + return longTimeout.applyAsLong(units); + return rpcTimeout.applyAsLong(units); + }; } diff --git a/src/java/org/apache/cassandra/repair/AbstractRepairTask.java b/src/java/org/apache/cassandra/repair/AbstractRepairTask.java index d2a6f1a786..c525597f5c 100644 --- a/src/java/org/apache/cassandra/repair/AbstractRepairTask.java +++ b/src/java/org/apache/cassandra/repair/AbstractRepairTask.java @@ -31,8 +31,8 @@ import org.slf4j.LoggerFactory; import org.apache.cassandra.concurrent.ExecutorPlus; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; +import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.repair.messages.RepairOption; -import org.apache.cassandra.service.ActiveRepairService; import org.apache.cassandra.utils.TimeUUID; import org.apache.cassandra.utils.concurrent.Future; import org.apache.cassandra.utils.concurrent.FutureCombiner; @@ -41,15 +41,17 @@ public abstract class AbstractRepairTask implements RepairTask { protected static final Logger logger = LoggerFactory.getLogger(AbstractRepairTask.class); + protected final RepairCoordinator coordinator; + protected final InetAddressAndPort broadcastAddressAndPort; protected final RepairOption options; protected final String keyspace; - protected final RepairNotifier notifier; - protected AbstractRepairTask(RepairOption options, String keyspace, RepairNotifier notifier) + protected AbstractRepairTask(RepairCoordinator coordinator) { - this.options = Objects.requireNonNull(options); - this.keyspace = Objects.requireNonNull(keyspace); - this.notifier = Objects.requireNonNull(notifier); + this.coordinator = Objects.requireNonNull(coordinator); + this.broadcastAddressAndPort = coordinator.ctx.broadcastAddressAndPort(); + this.options = Objects.requireNonNull(coordinator.state.options); + this.keyspace = Objects.requireNonNull(coordinator.state.keyspace); } private List submitRepairSessions(TimeUUID parentSession, @@ -63,18 +65,18 @@ public abstract class AbstractRepairTask implements RepairTask for (CommonRange commonRange : commonRanges) { logger.info("Starting RepairSession for {}", commonRange); - RepairSession session = ActiveRepairService.instance.submitRepairSession(parentSession, - commonRange, - keyspace, - options.getParallelism(), - isIncremental, - options.isPullRepair(), - options.getPreviewKind(), - options.optimiseStreams(), - options.repairPaxos(), - options.paxosOnly(), - executor, - cfnames); + RepairSession session = coordinator.ctx.repair().submitRepairSession(parentSession, + commonRange, + keyspace, + options.getParallelism(), + isIncremental, + options.isPullRepair(), + options.getPreviewKind(), + options.optimiseStreams(), + options.repairPaxos(), + options.paxosOnly(), + executor, + cfnames); if (session == null) continue; session.addCallback(new RepairSessionCallback(session)); @@ -107,18 +109,20 @@ public abstract class AbstractRepairTask implements RepairTask this.session = session; } + @Override public void onSuccess(RepairSessionResult result) { String message = String.format("Repair session %s for range %s finished", session.getId(), session.ranges().toString()); - notifier.notifyProgress(message); + coordinator.notifyProgress(message); } + @Override public void onFailure(Throwable t) { String message = String.format("Repair session %s for range %s failed with error %s", session.getId(), session.ranges().toString(), t.getMessage()); - notifier.notifyError(new RuntimeException(message, t)); + coordinator.notifyError(new RuntimeException(message, t)); } } } diff --git a/src/java/org/apache/cassandra/repair/AsymmetricRemoteSyncTask.java b/src/java/org/apache/cassandra/repair/AsymmetricRemoteSyncTask.java index c050f2c1b0..475c389678 100644 --- a/src/java/org/apache/cassandra/repair/AsymmetricRemoteSyncTask.java +++ b/src/java/org/apache/cassandra/repair/AsymmetricRemoteSyncTask.java @@ -29,7 +29,6 @@ import org.apache.cassandra.repair.messages.SyncRequest; import org.apache.cassandra.streaming.PreviewKind; import org.apache.cassandra.streaming.SessionSummary; import org.apache.cassandra.tracing.Tracing; -import org.apache.cassandra.utils.FBUtilities; /** * AsymmetricRemoteSyncTask sends {@link SyncRequest} to target node to repair(stream) @@ -39,14 +38,14 @@ import org.apache.cassandra.utils.FBUtilities; */ public class AsymmetricRemoteSyncTask extends SyncTask implements CompletableRemoteSyncTask { - public AsymmetricRemoteSyncTask(RepairJobDesc desc, InetAddressAndPort to, InetAddressAndPort from, List> differences, PreviewKind previewKind) + public AsymmetricRemoteSyncTask(SharedContext ctx, RepairJobDesc desc, InetAddressAndPort to, InetAddressAndPort from, List> differences, PreviewKind previewKind) { - super(desc, to, from, differences, previewKind); + super(ctx, desc, to, from, differences, previewKind); } public void startSync() { - InetAddressAndPort local = FBUtilities.getBroadcastAddressAndPort(); + InetAddressAndPort local = ctx.broadcastAddressAndPort(); SyncRequest request = new SyncRequest(desc, local, nodePair.coordinator, nodePair.peer, rangesToSync, previewKind, true); String message = String.format("Forwarding streaming repair of %d ranges to %s (to be streamed with %s)", request.ranges.size(), request.src, request.dst); Tracing.traceRepair(message); diff --git a/src/java/org/apache/cassandra/repair/IValidationManager.java b/src/java/org/apache/cassandra/repair/IValidationManager.java new file mode 100644 index 0000000000..fce4b62ad5 --- /dev/null +++ b/src/java/org/apache/cassandra/repair/IValidationManager.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.repair; + +import java.util.concurrent.Future; + +import org.apache.cassandra.db.ColumnFamilyStore; + +public interface IValidationManager +{ + Future submitValidation(ColumnFamilyStore cfs, Validator validator); +} diff --git a/src/java/org/apache/cassandra/repair/IncrementalRepairTask.java b/src/java/org/apache/cassandra/repair/IncrementalRepairTask.java index af1a234d34..7f5e5630cc 100644 --- a/src/java/org/apache/cassandra/repair/IncrementalRepairTask.java +++ b/src/java/org/apache/cassandra/repair/IncrementalRepairTask.java @@ -25,26 +25,21 @@ import com.google.common.collect.ImmutableSet; import org.apache.cassandra.concurrent.ExecutorPlus; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.repair.consistent.CoordinatorSession; -import org.apache.cassandra.repair.messages.RepairOption; -import org.apache.cassandra.service.ActiveRepairService; -import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.TimeUUID; import org.apache.cassandra.utils.concurrent.Future; public class IncrementalRepairTask extends AbstractRepairTask { private final TimeUUID parentSession; - private final RepairRunnable.NeighborsAndRanges neighborsAndRanges; + private final RepairCoordinator.NeighborsAndRanges neighborsAndRanges; private final String[] cfnames; - protected IncrementalRepairTask(RepairOption options, - String keyspace, - RepairNotifier notifier, + protected IncrementalRepairTask(RepairCoordinator coordinator, TimeUUID parentSession, - RepairRunnable.NeighborsAndRanges neighborsAndRanges, + RepairCoordinator.NeighborsAndRanges neighborsAndRanges, String[] cfnames) { - super(options, keyspace, notifier); + super(coordinator); this.parentSession = parentSession; this.neighborsAndRanges = neighborsAndRanges; this.cfnames = cfnames; @@ -62,12 +57,12 @@ public class IncrementalRepairTask extends AbstractRepairTask // the local node also needs to be included in the set of participants, since coordinator sessions aren't persisted Set allParticipants = ImmutableSet.builder() .addAll(neighborsAndRanges.participants) - .add(FBUtilities.getBroadcastAddressAndPort()) + .add(broadcastAddressAndPort) .build(); // Not necessary to include self for filtering. The common ranges only contains neighbhor node endpoints. List allRanges = neighborsAndRanges.filterCommonRanges(keyspace, cfnames); - CoordinatorSession coordinatorSession = ActiveRepairService.instance.consistent.coordinated.registerSession(parentSession, allParticipants, neighborsAndRanges.shouldExcludeDeadParticipants); + CoordinatorSession coordinatorSession = coordinator.ctx.repair().consistent.coordinated.registerSession(parentSession, allParticipants, neighborsAndRanges.shouldExcludeDeadParticipants); return coordinatorSession.execute(() -> runRepair(parentSession, true, executor, allRanges, cfnames)); diff --git a/src/java/org/apache/cassandra/repair/LocalSyncTask.java b/src/java/org/apache/cassandra/repair/LocalSyncTask.java index 5df05c79c2..379ba4b2a1 100644 --- a/src/java/org/apache/cassandra/repair/LocalSyncTask.java +++ b/src/java/org/apache/cassandra/repair/LocalSyncTask.java @@ -64,13 +64,13 @@ public class LocalSyncTask extends SyncTask implements StreamEventHandler private final AtomicBoolean active = new AtomicBoolean(true); private final Promise planPromise = new AsyncPromise<>(); - public LocalSyncTask(RepairJobDesc desc, InetAddressAndPort local, InetAddressAndPort remote, + public LocalSyncTask(SharedContext ctx, RepairJobDesc desc, InetAddressAndPort local, InetAddressAndPort remote, List> diff, TimeUUID pendingRepair, boolean requestRanges, boolean transferRanges, PreviewKind previewKind) { - super(desc, local, remote, diff, previewKind); + super(ctx, desc, local, remote, diff, previewKind); Preconditions.checkArgument(requestRanges || transferRanges, "Nothing to do in a sync job"); - Preconditions.checkArgument(local.equals(FBUtilities.getBroadcastAddressAndPort())); + Preconditions.checkArgument(local.equals(ctx.broadcastAddressAndPort())); this.pendingRepair = pendingRepair; this.requestRanges = requestRanges; @@ -119,7 +119,7 @@ public class LocalSyncTask extends SyncTask implements StreamEventHandler Tracing.traceRepair(message); StreamPlan plan = createStreamPlan(); - plan.execute(); + ctx.streamExecutor().execute(plan); planPromise.setSuccess(plan); } } @@ -130,6 +130,7 @@ public class LocalSyncTask extends SyncTask implements StreamEventHandler return true; } + @Override public void handleStreamEvent(StreamEvent event) { if (state == null) @@ -193,8 +194,9 @@ public class LocalSyncTask extends SyncTask implements StreamEventHandler } @Override - public void abort() + public void abort(Throwable reason) { + super.abort(reason); planPromise.addCallback((plan, cause) -> { assert plan != null : "StreamPlan future should never be completed exceptionally"; diff --git a/src/java/org/apache/cassandra/repair/NormalRepairTask.java b/src/java/org/apache/cassandra/repair/NormalRepairTask.java index 56a03f8816..bdc15cce53 100644 --- a/src/java/org/apache/cassandra/repair/NormalRepairTask.java +++ b/src/java/org/apache/cassandra/repair/NormalRepairTask.java @@ -20,7 +20,6 @@ package org.apache.cassandra.repair; import java.util.List; import org.apache.cassandra.concurrent.ExecutorPlus; -import org.apache.cassandra.repair.messages.RepairOption; import org.apache.cassandra.utils.TimeUUID; import org.apache.cassandra.utils.concurrent.Future; @@ -30,14 +29,12 @@ public class NormalRepairTask extends AbstractRepairTask private final List commonRanges; private final String[] cfnames; - protected NormalRepairTask(RepairOption options, - String keyspace, - RepairNotifier notifier, + protected NormalRepairTask(RepairCoordinator coordinator, TimeUUID parentSession, List commonRanges, String[] cfnames) { - super(options, keyspace, notifier); + super(coordinator); this.parentSession = parentSession; this.commonRanges = commonRanges; this.cfnames = cfnames; diff --git a/src/java/org/apache/cassandra/repair/PreviewRepairTask.java b/src/java/org/apache/cassandra/repair/PreviewRepairTask.java index 728a813a2a..5b861f71b3 100644 --- a/src/java/org/apache/cassandra/repair/PreviewRepairTask.java +++ b/src/java/org/apache/cassandra/repair/PreviewRepairTask.java @@ -32,7 +32,6 @@ import org.apache.cassandra.dht.Token; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.metrics.RepairMetrics; import org.apache.cassandra.repair.consistent.SyncStatSummary; -import org.apache.cassandra.repair.messages.RepairOption; import org.apache.cassandra.streaming.PreviewKind; import org.apache.cassandra.utils.DiagnosticSnapshotService; import org.apache.cassandra.utils.TimeUUID; @@ -45,9 +44,9 @@ public class PreviewRepairTask extends AbstractRepairTask private final String[] cfnames; private volatile String successMessage = name() + " completed successfully"; - protected PreviewRepairTask(RepairOption options, String keyspace, RepairNotifier notifier, TimeUUID parentSession, List commonRanges, String[] cfnames) + protected PreviewRepairTask(RepairCoordinator coordinator, TimeUUID parentSession, List commonRanges, String[] cfnames) { - super(options, keyspace, notifier); + super(coordinator); this.parentSession = parentSession; this.commonRanges = commonRanges; this.cfnames = cfnames; @@ -91,7 +90,7 @@ public class PreviewRepairTask extends AbstractRepairTask maybeSnapshotReplicas(parentSession, keyspace, result.results.get()); // we know its present as summary used it } successMessage += "; " + message; - notifier.notification(message); + coordinator.notification(message); return result; }); diff --git a/src/java/org/apache/cassandra/repair/RepairRunnable.java b/src/java/org/apache/cassandra/repair/RepairCoordinator.java similarity index 82% rename from src/java/org/apache/cassandra/repair/RepairRunnable.java rename to src/java/org/apache/cassandra/repair/RepairCoordinator.java index c56601010f..3d1eab3e9e 100644 --- a/src/java/org/apache/cassandra/repair/RepairRunnable.java +++ b/src/java/org/apache/cassandra/repair/RepairCoordinator.java @@ -17,7 +17,6 @@ */ package org.apache.cassandra.repair; -import java.io.IOException; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Arrays; @@ -29,6 +28,9 @@ import java.util.UUID; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; +import java.util.function.BiFunction; +import java.util.function.Function; +import java.util.function.Supplier; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; @@ -38,7 +40,8 @@ import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import com.google.common.collect.Sets; -import org.apache.cassandra.utils.TimeUUID; +import org.apache.cassandra.locator.RangesAtEndpoint; +import org.apache.cassandra.utils.*; import org.apache.cassandra.utils.concurrent.Future; import org.apache.commons.lang3.time.DurationFormatUtils; import org.slf4j.Logger; @@ -57,7 +60,6 @@ import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; import org.apache.cassandra.exceptions.RepairException; -import org.apache.cassandra.gms.FailureDetector; import org.apache.cassandra.locator.EndpointsForRange; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.locator.Replica; @@ -74,44 +76,51 @@ import org.apache.cassandra.tracing.TraceKeyspace; import org.apache.cassandra.tracing.TraceState; import org.apache.cassandra.tracing.Tracing; import org.apache.cassandra.transport.messages.ResultMessage; -import org.apache.cassandra.utils.FBUtilities; -import org.apache.cassandra.utils.Throwables; -import org.apache.cassandra.utils.WrappedRunnable; import org.apache.cassandra.utils.progress.ProgressEvent; import org.apache.cassandra.utils.progress.ProgressEventNotifier; import org.apache.cassandra.utils.progress.ProgressEventType; import org.apache.cassandra.utils.progress.ProgressListener; -import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory; import static org.apache.cassandra.repair.state.AbstractState.COMPLETE; import static org.apache.cassandra.repair.state.AbstractState.INIT; import static org.apache.cassandra.service.QueryState.forInternalCalls; -import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis; -import static org.apache.cassandra.utils.Clock.Global.nanoTime; -public class RepairRunnable implements Runnable, ProgressEventNotifier, RepairNotifier +public class RepairCoordinator implements Runnable, ProgressEventNotifier, RepairNotifier { - private static final Logger logger = LoggerFactory.getLogger(RepairRunnable.class); + private static final Logger logger = LoggerFactory.getLogger(RepairCoordinator.class); private static final AtomicInteger THREAD_COUNTER = new AtomicInteger(1); public final CoordinatorState state; - private final StorageService storageService; - private final String tag; + private final BiFunction> validColumnFamilies; + private final Function getLocalReplicas; private final List listeners = new ArrayList<>(); private final AtomicReference firstError = new AtomicReference<>(null); + final SharedContext ctx; private TraceState traceState; - public RepairRunnable(StorageService storageService, int cmd, RepairOption options, String keyspace) + public RepairCoordinator(StorageService storageService, int cmd, RepairOption options, String keyspace) { - this.state = new CoordinatorState(cmd, keyspace, options); - this.storageService = storageService; + this(SharedContext.Global.instance, + (ks, tables) -> storageService.getValidColumnFamilies(false, false, ks, tables), + storageService::getLocalReplicas, + cmd, options, keyspace); + } + RepairCoordinator(SharedContext ctx, + BiFunction> validColumnFamilies, + Function getLocalReplicas, + int cmd, RepairOption options, String keyspace) + { + this.ctx = ctx; + this.state = new CoordinatorState(ctx.clock(), cmd, keyspace, options); this.tag = "repair:" + cmd; - ActiveRepairService.instance.register(state); + this.validColumnFamilies = validColumnFamilies; + this.getLocalReplicas = getLocalReplicas; + ctx.repair().register(state); } @Override @@ -187,7 +196,7 @@ public class RepairRunnable implements Runnable, ProgressEventNotifier, RepairNo { state.phase.success(msg); fireProgressEvent(jmxEvent(ProgressEventType.SUCCESS, msg)); - ActiveRepairService.instance.recordRepairStatus(state.cmd, ActiveRepairService.ParentRepairStatus.COMPLETED, + ctx.repair().recordRepairStatus(state.cmd, ActiveRepairService.ParentRepairStatus.COMPLETED, ImmutableList.of(msg)); complete(null); } @@ -197,14 +206,14 @@ public class RepairRunnable implements Runnable, ProgressEventNotifier, RepairNo if (reason == null) { Throwable error = firstError.get(); - reason = error != null ? error.getMessage() : "Some repair failed"; + reason = error != null ? error.toString() : "Some repair failed"; } state.phase.fail(reason); String completionMessage = String.format("Repair command #%d finished with error", state.cmd); // Note we rely on the first message being the reason for the failure // when inspecting this state from RepairRunner.queryForCompletedRepair - ActiveRepairService.instance.recordRepairStatus(state.cmd, ParentRepairStatus.FAILED, + ctx.repair().recordRepairStatus(state.cmd, ParentRepairStatus.FAILED, ImmutableList.of(reason, completionMessage)); complete(completionMessage); @@ -222,7 +231,7 @@ public class RepairRunnable implements Runnable, ProgressEventNotifier, RepairNo fireProgressEvent(jmxEvent(ProgressEventType.COMPLETE, msg)); logger.info(state.options.getPreviewKind().logPrefix(state.id) + msg); - ActiveRepairService.instance.removeParentRepairSession(state.id); + ctx.repair().removeParentRepairSession(state.id); TraceState localState = traceState; if (state.options.isTraced() && localState != null) { @@ -251,17 +260,17 @@ public class RepairRunnable implements Runnable, ProgressEventNotifier, RepairNo { skip(e.getMessage()); } - catch (Exception | Error e) + catch (Throwable e) { notifyError(e); fail(e.getMessage()); } } - private void runMayThrow() throws Exception + private void runMayThrow() throws Throwable { state.phase.setup(); - ActiveRepairService.instance.recordRepairStatus(state.cmd, ParentRepairStatus.IN_PROGRESS, ImmutableList.of()); + ctx.repair().recordRepairStatus(state.cmd, ParentRepairStatus.IN_PROGRESS, ImmutableList.of()); List columnFamilies = getColumnFamilies(); String[] cfnames = columnFamilies.stream().map(cfs -> cfs.name).toArray(String[]::new); @@ -276,15 +285,36 @@ public class RepairRunnable implements Runnable, ProgressEventNotifier, RepairNo maybeStoreParentRepairStart(cfnames); - prepare(columnFamilies, neighborsAndRanges.participants, neighborsAndRanges.shouldExcludeDeadParticipants); - - repair(cfnames, neighborsAndRanges); + prepare(columnFamilies, neighborsAndRanges.participants, neighborsAndRanges.shouldExcludeDeadParticipants) + .flatMap(ignore -> repair(cfnames, neighborsAndRanges)) + .addCallback((pair, failure) -> { + if (failure != null) + { + notifyError(failure); + fail(failure.getMessage()); + } + else + { + state.phase.repairCompleted(); + CoordinatedRepairResult result = pair.left; + maybeStoreParentRepairSuccess(result.successfulRanges); + if (result.hasFailed()) + { + fail(null); + } + else + { + success(pair.right.get()); + ctx.repair().cleanUp(state.id, neighborsAndRanges.participants); + } + } + }); } - private List getColumnFamilies() throws IOException + private List getColumnFamilies() { String[] columnFamilies = state.options.getColumnFamilies().toArray(new String[state.options.getColumnFamilies().size()]); - Iterable validColumnFamilies = storageService.getValidColumnFamilies(false, false, state.keyspace, columnFamilies); + Iterable validColumnFamilies = this.validColumnFamilies.apply(state.keyspace, columnFamilies); if (Iterables.isEmpty(validColumnFamilies)) throw new SkipRepairException(String.format("%s Empty keyspace, skipping repair: %s", state.id, state.keyspace)); @@ -327,11 +357,11 @@ public class RepairRunnable implements Runnable, ProgressEventNotifier, RepairNo //pre-calculate output of getLocalReplicas and pass it to getNeighbors to increase performance and prevent //calculation multiple times - Iterable> keyspaceLocalRanges = storageService.getLocalReplicas(state.keyspace).ranges(); + Iterable> keyspaceLocalRanges = getLocalReplicas.apply(state.keyspace).ranges(); for (Range range : state.options.getRanges()) { - EndpointsForRange neighbors = ActiveRepairService.getNeighbors(state.keyspace, keyspaceLocalRanges, range, + EndpointsForRange neighbors = ctx.repair().getNeighbors(state.keyspace, keyspaceLocalRanges, range, state.options.getDataCenters(), state.options.getHosts()); if (neighbors.isEmpty()) @@ -361,7 +391,7 @@ public class RepairRunnable implements Runnable, ProgressEventNotifier, RepairNo if (shouldExcludeDeadParticipants) { - Set actualNeighbors = Sets.newHashSet(Iterables.filter(allNeighbors, FailureDetector.instance::isAlive)); + Set actualNeighbors = Sets.newHashSet(Iterables.filter(allNeighbors, ctx.failureDetector()::isAlive)); shouldExcludeDeadParticipants = !allNeighbors.equals(actualNeighbors); allNeighbors = actualNeighbors; } @@ -392,68 +422,46 @@ public class RepairRunnable implements Runnable, ProgressEventNotifier, RepairNo } } - private void prepare(List columnFamilies, Set allNeighbors, boolean force) + private Future prepare(List columnFamilies, Set allNeighbors, boolean force) { state.phase.prepareStart(); - try (Timer.Context ignore = Keyspace.open(state.keyspace).metric.repairPrepareTime.time()) - { - ActiveRepairService.instance.prepareForRepair(state.id, FBUtilities.getBroadcastAddressAndPort(), allNeighbors, state.options, force, columnFamilies); - } - state.phase.prepareComplete(); + Timer timer = Keyspace.open(state.keyspace).metric.repairPrepareTime; + long startNanos = ctx.clock().nanoTime(); + return ctx.repair().prepareForRepair(state.id, ctx.broadcastAddressAndPort(), allNeighbors, state.options, force, columnFamilies) + .map(ignore -> { + timer.update(ctx.clock().nanoTime() - startNanos, TimeUnit.NANOSECONDS); + state.phase.prepareComplete(); + return null; + }); } - private void repair(String[] cfnames, NeighborsAndRanges neighborsAndRanges) + private Future>> repair(String[] cfnames, NeighborsAndRanges neighborsAndRanges) { RepairTask task; if (state.options.isPreview()) { - task = new PreviewRepairTask(state.options, state.keyspace, this, state.id, neighborsAndRanges.filterCommonRanges(state.keyspace, cfnames), cfnames); + task = new PreviewRepairTask(this, state.id, neighborsAndRanges.filterCommonRanges(state.keyspace, cfnames), cfnames); } else if (state.options.isIncremental()) { - task = new IncrementalRepairTask(state.options, state.keyspace, this, state.id, neighborsAndRanges, cfnames); + task = new IncrementalRepairTask(this, state.id, neighborsAndRanges, cfnames); } else { - task = new NormalRepairTask(state.options, state.keyspace, this, state.id, neighborsAndRanges.filterCommonRanges(state.keyspace, cfnames), cfnames); + task = new NormalRepairTask(this, state.id, neighborsAndRanges.filterCommonRanges(state.keyspace, cfnames), cfnames); } ExecutorPlus executor = createExecutor(); state.phase.repairSubmitted(); - Future f = task.perform(executor); - f.addCallback((result, failure) -> { - state.phase.repairCompleted(); - try - { - if (failure != null) - { - notifyError(failure); - fail(failure.getMessage()); - } - else - { - maybeStoreParentRepairSuccess(result.successfulRanges); - if (result.hasFailed()) - { - fail(null); - } - else - { - success(task.successMessage()); - ActiveRepairService.instance.cleanUp(state.id, neighborsAndRanges.participants); - } - } - } - finally - { - executor.shutdown(); - } - }); + return task.perform(executor) + // after adding the callback java could no longer infer the type... + .>>map(r -> Pair.create(r, task::successMessage)) + .addCallback((s, f) -> executor.shutdown()); } private ExecutorPlus createExecutor() { - return executorFactory() + return ctx.executorFactory() .localAware() .withJmxInternal() .pooled("Repair#" + state.cmd, state.options.getJobThreads()); @@ -480,7 +488,7 @@ public class RepairRunnable implements Runnable, ProgressEventNotifier, RepairNo private Thread createQueryThread(final TimeUUID sessionId) { - return executorFactory().startThread("Repair-Runnable-" + THREAD_COUNTER.incrementAndGet(), new WrappedRunnable() + return ctx.executorFactory().startThread("Repair-Runnable-" + THREAD_COUNTER.incrementAndGet(), new WrappedRunnable() { // Query events within a time interval that overlaps the last by one second. Ignore duplicates. Ignore local traces. // Wake up upon local trace activity. Query when notified of trace activity with a timeout that doubles every two timeouts. @@ -495,13 +503,13 @@ public class RepairRunnable implements Runnable, ProgressEventNotifier, RepairNo SelectStatement statement = (SelectStatement) QueryProcessor.parseStatement(query).prepare(ClientState.forInternalCalls()); ByteBuffer sessionIdBytes = sessionId.toBytes(); - InetAddressAndPort source = FBUtilities.getBroadcastAddressAndPort(); + InetAddressAndPort source = ctx.broadcastAddressAndPort(); HashSet[] seen = new HashSet[]{ new HashSet<>(), new HashSet<>() }; int si = 0; UUID uuid; - long tlast = currentTimeMillis(), tcur; + long tlast = ctx.clock().currentTimeMillis(), tcur; TraceState.Status status; long minWaitMillis = 125; @@ -522,11 +530,11 @@ public class RepairRunnable implements Runnable, ProgressEventNotifier, RepairNo shouldDouble = false; } ByteBuffer tminBytes = TimeUUID.minAtUnixMillis(tlast - 1000).toBytes(); - ByteBuffer tmaxBytes = TimeUUID.maxAtUnixMillis(tcur = currentTimeMillis()).toBytes(); + ByteBuffer tmaxBytes = TimeUUID.maxAtUnixMillis(tcur = ctx.clock().currentTimeMillis()).toBytes(); QueryOptions options = QueryOptions.forInternalCalls(ConsistencyLevel.ONE, Lists.newArrayList(sessionIdBytes, tminBytes, tmaxBytes)); - ResultMessage.Rows rows = statement.execute(forInternalCalls(), options, nanoTime()); + ResultMessage.Rows rows = statement.execute(forInternalCalls(), options, ctx.clock().nanoTime()); UntypedResultSet result = UntypedResultSet.create(rows.result); for (UntypedResultSet.Row r : result) diff --git a/src/java/org/apache/cassandra/repair/RepairJob.java b/src/java/org/apache/cassandra/repair/RepairJob.java index 7d11b0597c..8fec3e9d54 100644 --- a/src/java/org/apache/cassandra/repair/RepairJob.java +++ b/src/java/org/apache/cassandra/repair/RepairJob.java @@ -19,16 +19,19 @@ package org.apache.cassandra.repair; import java.util.*; import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.Executor; import java.util.function.Predicate; import java.util.function.Function; import java.util.stream.Collectors; +import javax.annotation.Nullable; + import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; import com.google.common.util.concurrent.*; -import org.apache.cassandra.concurrent.ExecutorPlus; + import org.apache.cassandra.schema.Schema; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.repair.state.JobState; @@ -45,7 +48,6 @@ import org.apache.cassandra.repair.asymmetric.PreferedNodeFilter; import org.apache.cassandra.repair.asymmetric.ReduceHelper; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.schema.SystemDistributedKeyspace; -import org.apache.cassandra.service.ActiveRepairService; import org.apache.cassandra.streaming.PreviewKind; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; @@ -60,7 +62,6 @@ import org.apache.cassandra.utils.concurrent.ImmediateFuture; import static org.apache.cassandra.config.DatabaseDescriptor.paxosRepairEnabled; import static org.apache.cassandra.service.paxos.Paxos.useV2; -import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis; /** * RepairJob runs repair on given ColumnFamily. @@ -69,11 +70,12 @@ public class RepairJob extends AsyncFuture implements Runnable { private static final Logger logger = LoggerFactory.getLogger(RepairJob.class); + private final SharedContext ctx; public final JobState state; private final RepairJobDesc desc; private final RepairSession session; private final RepairParallelism parallelismDegree; - private final ExecutorPlus taskExecutor; + private final Executor taskExecutor; @VisibleForTesting final List validationTasks = new CopyOnWriteArrayList<>(); @@ -88,16 +90,17 @@ public class RepairJob extends AsyncFuture implements Runnable */ public RepairJob(RepairSession session, String columnFamily) { + this.ctx = session.ctx; this.session = session; this.taskExecutor = session.taskExecutor; this.parallelismDegree = session.parallelismDegree; this.desc = new RepairJobDesc(session.state.parentRepairSession, session.getId(), session.state.keyspace, columnFamily, session.state.commonRange.ranges); - this.state = new JobState(desc, session.state.commonRange.endpoints); + this.state = new JobState(ctx.clock(), desc, session.state.commonRange.endpoints); } public long getNowInSeconds() { - long nowInSeconds = FBUtilities.nowInSeconds(); + long nowInSeconds = ctx.clock().nowInSeconds(); if (session.previewKind == PreviewKind.REPAIRED) { return nowInSeconds + DatabaseDescriptor.getValidationPreviewPurgeHeadStartInSec(); @@ -121,7 +124,7 @@ public class RepairJob extends AsyncFuture implements Runnable ColumnFamilyStore cfs = ks.getColumnFamilyStore(desc.columnFamily); cfs.metric.repairsStarted.inc(); List allEndpoints = new ArrayList<>(session.state.commonRange.endpoints); - allEndpoints.add(FBUtilities.getBroadcastAddressAndPort()); + allEndpoints.add(ctx.broadcastAddressAndPort()); Future> treeResponses; Future paxosRepair; @@ -176,7 +179,7 @@ public class RepairJob extends AsyncFuture implements Runnable state.phase.snapshotsSubmitted(); for (InetAddressAndPort endpoint : allEndpoints) { - SnapshotTask snapshotTask = new SnapshotTask(desc, endpoint); + SnapshotTask snapshotTask = new SnapshotTask(ctx, desc, endpoint); snapshotTasks.add(snapshotTask); taskExecutor.execute(snapshotTask); } @@ -231,9 +234,7 @@ public class RepairJob extends AsyncFuture implements Runnable public void onFailure(Throwable t) { state.phase.fail(t); - // Make sure all validation tasks have cleaned up the off-heap Merkle trees they might contain. - validationTasks.forEach(ValidationTask::abort); - syncTasks.forEach(SyncTask::abort); + abort(t); if (!session.previewKind.isPreview()) { @@ -248,6 +249,17 @@ public class RepairJob extends AsyncFuture implements Runnable }, taskExecutor); } + public synchronized void abort(@Nullable Throwable reason) + { + if (reason == null) + reason = new RuntimeException("Abort"); + // Make sure all validation tasks have cleaned up the off-heap Merkle trees they might contain. + for (ValidationTask v : validationTasks) + v.abort(reason); + for (SyncTask s : syncTasks) + s.abort(reason); + } + private boolean isTransient(InetAddressAndPort ep) { return session.state.commonRange.transEndpoints.contains(ep); @@ -255,10 +267,9 @@ public class RepairJob extends AsyncFuture implements Runnable private Future> standardSyncing(List trees) { - state.phase.streamSubmitted(); - List syncTasks = createStandardSyncTasks(desc, + List syncTasks = createStandardSyncTasks(ctx, desc, trees, - FBUtilities.getLocalAddressAndPort(), + ctx.broadcastAddressAndPort(), this::isTransient, session.isIncremental, session.pullRepair, @@ -266,7 +277,8 @@ public class RepairJob extends AsyncFuture implements Runnable return executeTasks(syncTasks); } - static List createStandardSyncTasks(RepairJobDesc desc, + static List createStandardSyncTasks(SharedContext ctx, + RepairJobDesc desc, List trees, InetAddressAndPort local, Predicate isTransient, @@ -274,7 +286,7 @@ public class RepairJob extends AsyncFuture implements Runnable boolean pullRepair, PreviewKind previewKind) { - long startedAt = currentTimeMillis(); + long startedAt = ctx.clock().currentTimeMillis(); List syncTasks = new ArrayList<>(); // We need to difference all trees one against another for (int i = 0; i < trees.size() - 1; ++i) @@ -309,7 +321,7 @@ public class RepairJob extends AsyncFuture implements Runnable if (!requestRanges && !transferRanges) continue; - task = new LocalSyncTask(desc, self.endpoint, remote.endpoint, differences, isIncremental ? desc.parentSessionId : null, + task = new LocalSyncTask(ctx, desc, self.endpoint, remote.endpoint, differences, isIncremental ? desc.parentSessionId : null, requestRanges, transferRanges, previewKind); } else if (isTransient.test(r1.endpoint) || isTransient.test(r2.endpoint)) @@ -317,11 +329,11 @@ public class RepairJob extends AsyncFuture implements Runnable // Stream only from transient replica TreeResponse streamFrom = isTransient.test(r1.endpoint) ? r1 : r2; TreeResponse streamTo = isTransient.test(r1.endpoint) ? r2 : r1; - task = new AsymmetricRemoteSyncTask(desc, streamTo.endpoint, streamFrom.endpoint, differences, previewKind); + task = new AsymmetricRemoteSyncTask(ctx, desc, streamTo.endpoint, streamFrom.endpoint, differences, previewKind); } else { - task = new SymmetricRemoteSyncTask(desc, r1.endpoint, r2.endpoint, differences, previewKind); + task = new SymmetricRemoteSyncTask(ctx, desc, r1.endpoint, r2.endpoint, differences, previewKind); } syncTasks.add(task); } @@ -329,14 +341,14 @@ public class RepairJob extends AsyncFuture implements Runnable } trees.get(trees.size() - 1).trees.release(); logger.info("Created {} sync tasks based on {} merkle tree responses for {} (took: {}ms)", - syncTasks.size(), trees.size(), desc.parentSessionId, currentTimeMillis() - startedAt); + syncTasks.size(), trees.size(), desc.parentSessionId, ctx.clock().currentTimeMillis() - startedAt); return syncTasks; } private Future> optimisedSyncing(List trees) { - state.phase.streamSubmitted(); - List syncTasks = createOptimisedSyncingSyncTasks(desc, + List syncTasks = createOptimisedSyncingSyncTasks(ctx, + desc, trees, FBUtilities.getLocalAddressAndPort(), this::isTransient, @@ -352,9 +364,12 @@ public class RepairJob extends AsyncFuture implements Runnable { try { - ActiveRepairService.instance.getParentRepairSession(desc.parentSessionId); + ctx.repair().getParentRepairSession(desc.parentSessionId); syncTasks.addAll(tasks); + if (!tasks.isEmpty()) + state.phase.streamSubmitted(); + for (SyncTask task : tasks) { if (!task.isLocal()) @@ -385,7 +400,8 @@ public class RepairJob extends AsyncFuture implements Runnable } } - static List createOptimisedSyncingSyncTasks(RepairJobDesc desc, + static List createOptimisedSyncingSyncTasks(SharedContext ctx, + RepairJobDesc desc, List trees, InetAddressAndPort local, Predicate isTransient, @@ -393,7 +409,7 @@ public class RepairJob extends AsyncFuture implements Runnable boolean isIncremental, PreviewKind previewKind) { - long startedAt = currentTimeMillis(); + long startedAt = ctx.clock().currentTimeMillis(); List syncTasks = new ArrayList<>(); // We need to difference all trees one against another DifferenceHolder diffHolder = new DifferenceHolder(trees); @@ -427,12 +443,12 @@ public class RepairJob extends AsyncFuture implements Runnable SyncTask task; if (address.equals(local)) { - task = new LocalSyncTask(desc, address, fetchFrom, toFetch, isIncremental ? desc.parentSessionId : null, + task = new LocalSyncTask(ctx, desc, address, fetchFrom, toFetch, isIncremental ? desc.parentSessionId : null, true, false, previewKind); } else { - task = new AsymmetricRemoteSyncTask(desc, address, fetchFrom, toFetch, previewKind); + task = new AsymmetricRemoteSyncTask(ctx, desc, address, fetchFrom, toFetch, previewKind); } syncTasks.add(task); @@ -444,14 +460,14 @@ public class RepairJob extends AsyncFuture implements Runnable } } logger.info("Created {} optimised sync tasks based on {} merkle tree responses for {} (took: {}ms)", - syncTasks.size(), trees.size(), desc.parentSessionId, currentTimeMillis() - startedAt); + syncTasks.size(), trees.size(), desc.parentSessionId, ctx.clock().currentTimeMillis() - startedAt); logger.trace("Optimised sync tasks for {}: {}", desc.parentSessionId, syncTasks); return syncTasks; } private String getDC(InetAddressAndPort address) { - return DatabaseDescriptor.getEndpointSnitch().getDatacenter(address); + return ctx.snitch().getDatacenter(address); } /** @@ -582,7 +598,7 @@ public class RepairJob extends AsyncFuture implements Runnable private ValidationTask newValidationTask(InetAddressAndPort endpoint, long nowInSec) { - ValidationTask task = new ValidationTask(desc, endpoint, nowInSec, session.previewKind); + ValidationTask task = new ValidationTask(session.ctx, desc, endpoint, nowInSec, session.previewKind); validationTasks.add(task); return task; } diff --git a/src/java/org/apache/cassandra/repair/RepairJobDesc.java b/src/java/org/apache/cassandra/repair/RepairJobDesc.java index c6b1898f79..dba336b5e7 100644 --- a/src/java/org/apache/cassandra/repair/RepairJobDesc.java +++ b/src/java/org/apache/cassandra/repair/RepairJobDesc.java @@ -21,10 +21,9 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Collection; +import java.util.Objects; import java.util.UUID; -import com.google.common.base.Objects; - import org.apache.commons.lang3.ArrayUtils; import org.apache.cassandra.db.TypeSizes; @@ -96,11 +95,11 @@ public class RepairJobDesc RepairJobDesc that = (RepairJobDesc) o; - if (!columnFamily.equals(that.columnFamily)) return false; - if (!keyspace.equals(that.keyspace)) return false; - if (ranges != null ? that.ranges == null || (ranges.size() != that.ranges.size()) || (ranges.size() == that.ranges.size() && !ranges.containsAll(that.ranges)) : that.ranges != null) return false; + if (!Objects.equals(parentSessionId, that.parentSessionId)) return false; if (!sessionId.equals(that.sessionId)) return false; - if (parentSessionId != null ? !parentSessionId.equals(that.parentSessionId) : that.parentSessionId != null) return false; + if (!keyspace.equals(that.keyspace)) return false; + if (!columnFamily.equals(that.columnFamily)) return false; + if (ranges != null ? that.ranges == null || (ranges.size() != that.ranges.size()) || (ranges.size() == that.ranges.size() && !ranges.containsAll(that.ranges)) : that.ranges != null) return false; return true; } @@ -108,7 +107,7 @@ public class RepairJobDesc @Override public int hashCode() { - return Objects.hashCode(sessionId, keyspace, columnFamily, ranges); + return Objects.hash(parentSessionId, sessionId, keyspace, columnFamily, ranges); } private static class RepairJobDescSerializer implements IVersionedSerializer diff --git a/src/java/org/apache/cassandra/repair/RepairMessageVerbHandler.java b/src/java/org/apache/cassandra/repair/RepairMessageVerbHandler.java index 58612f778a..988159860d 100644 --- a/src/java/org/apache/cassandra/repair/RepairMessageVerbHandler.java +++ b/src/java/org/apache/cassandra/repair/RepairMessageVerbHandler.java @@ -18,6 +18,8 @@ package org.apache.cassandra.repair; import java.util.*; +import java.util.function.BiFunction; +import java.util.function.Function; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -26,17 +28,19 @@ import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.exceptions.RequestFailureReason; import org.apache.cassandra.net.IVerbHandler; import org.apache.cassandra.net.Message; -import org.apache.cassandra.net.MessagingService; import org.apache.cassandra.repair.messages.*; +import org.apache.cassandra.repair.state.AbstractCompletable; +import org.apache.cassandra.repair.state.AbstractState; +import org.apache.cassandra.repair.state.Completable; import org.apache.cassandra.repair.state.ParticipateState; +import org.apache.cassandra.repair.state.SyncState; import org.apache.cassandra.repair.state.ValidationState; import org.apache.cassandra.schema.TableId; import org.apache.cassandra.service.ActiveRepairService; import org.apache.cassandra.streaming.PreviewKind; +import org.apache.cassandra.utils.JVMStabilityInspector; import org.apache.cassandra.utils.TimeUUID; -import static org.apache.cassandra.net.Verb.VALIDATION_RSP; - /** * Handles all repair related message. * @@ -44,18 +48,38 @@ import static org.apache.cassandra.net.Verb.VALIDATION_RSP; */ public class RepairMessageVerbHandler implements IVerbHandler { - public static RepairMessageVerbHandler instance = new RepairMessageVerbHandler(); + private static class Holder + { + private static final RepairMessageVerbHandler instance = new RepairMessageVerbHandler(); + } + + public static RepairMessageVerbHandler instance() + { + return Holder.instance; + } + + private final SharedContext ctx; + + private RepairMessageVerbHandler() + { + this(SharedContext.Global.instance); + } + + public RepairMessageVerbHandler(SharedContext ctx) + { + this.ctx = ctx; + } private static final Logger logger = LoggerFactory.getLogger(RepairMessageVerbHandler.class); private boolean isIncremental(TimeUUID sessionID) { - return ActiveRepairService.instance.consistent.local.isSessionInProgress(sessionID); + return ctx.repair().consistent.local.isSessionInProgress(sessionID); } private PreviewKind previewKind(TimeUUID sessionID) throws NoSuchRepairSessionException { - ActiveRepairService.ParentRepairSession prs = ActiveRepairService.instance.getParentRepairSession(sessionID); + ActiveRepairService.ParentRepairSession prs = ctx.repair().getParentRepairSession(sessionID); return prs != null ? prs.previewKind : PreviewKind.NONE; } @@ -71,16 +95,17 @@ public class RepairMessageVerbHandler implements IVerbHandler { PrepareMessage prepareMessage = (PrepareMessage) message.payload; logger.debug("Preparing, {}", prepareMessage); - ParticipateState state = new ParticipateState(message.from(), prepareMessage); - if (!ActiveRepairService.instance.register(state)) + ParticipateState state = new ParticipateState(ctx.clock(), message.from(), prepareMessage); + if (!ctx.repair().register(state)) { - logger.debug("Duplicate prepare message found for {}", state.id); + replyDedup(ctx.repair().participate(state.id), message); return; } - if (!ActiveRepairService.verifyCompactionsPendingThreshold(prepareMessage.parentRepairSession, prepareMessage.previewKind)) + if (!ctx.repair().verifyCompactionsPendingThreshold(prepareMessage.parentRepairSession, prepareMessage.previewKind)) { // error is logged in verifyCompactionsPendingThreshold state.phase.fail("Too many pending compactions"); + sendFailureResponse(message); return; } @@ -99,22 +124,23 @@ public class RepairMessageVerbHandler implements IVerbHandler } columnFamilyStores.add(columnFamilyStore); } - ActiveRepairService.instance.registerParentRepairSession(prepareMessage.parentRepairSession, - message.from(), - columnFamilyStores, - prepareMessage.ranges, - prepareMessage.isIncremental, - prepareMessage.repairedAt, - prepareMessage.isGlobal, - prepareMessage.previewKind); - MessagingService.instance().send(message.emptyResponse(), message.from()); + state.phase.accept(); + ctx.repair().registerParentRepairSession(prepareMessage.parentRepairSession, + message.from(), + columnFamilyStores, + prepareMessage.ranges, + prepareMessage.isIncremental, + prepareMessage.repairedAt, + prepareMessage.isGlobal, + prepareMessage.previewKind); + sendAck(message); } break; case SNAPSHOT_MSG: { logger.debug("Snapshotting {}", desc); - ParticipateState state = ActiveRepairService.instance.participate(desc.parentSessionId); + ParticipateState state = ctx.repair().participate(desc.parentSessionId); if (state == null) { logErrorAndSendFailureResponse("Unknown repair " + desc.parentSessionId, message); @@ -130,56 +156,65 @@ public class RepairMessageVerbHandler implements IVerbHandler return; } - ActiveRepairService.ParentRepairSession prs = ActiveRepairService.instance.getParentRepairSession(desc.parentSessionId); - prs.setHasSnapshots(); - TableRepairManager repairManager = cfs.getRepairManager(); - if (prs.isGlobal) + ActiveRepairService.ParentRepairSession prs = ctx.repair().getParentRepairSession(desc.parentSessionId); + if (prs.setHasSnapshots()) { - repairManager.snapshot(desc.parentSessionId.toString(), prs.getRanges(), false); + state.getOrCreateJob(desc).snapshot(); + TableRepairManager repairManager = cfs.getRepairManager(); + if (prs.isGlobal) + { + repairManager.snapshot(desc.parentSessionId.toString(), prs.getRanges(), false); + } + else + { + repairManager.snapshot(desc.parentSessionId.toString(), desc.ranges, true); + } + logger.debug("Enqueuing response to snapshot request {} to {}", desc.sessionId, message.from()); } - else - { - repairManager.snapshot(desc.parentSessionId.toString(), desc.ranges, true); - } - logger.debug("Enqueuing response to snapshot request {} to {}", desc.sessionId, message.from()); - MessagingService.instance().send(message.emptyResponse(), message.from()); + sendAck(message); } break; case VALIDATION_REQ: { - // notify initiator that the message has been received, allowing this method to take as long as it needs to - MessagingService.instance().send(message.emptyResponse(), message.from()); ValidationRequest validationRequest = (ValidationRequest) message.payload; logger.debug("Validating {}", validationRequest); - ParticipateState participate = ActiveRepairService.instance.participate(desc.parentSessionId); + ParticipateState participate = ctx.repair().participate(desc.parentSessionId); if (participate == null) { logErrorAndSendFailureResponse("Unknown repair " + desc.parentSessionId, message); return; } - ValidationState vState = new ValidationState(desc, message.from()); - if (!participate.register(vState)) - { - logger.debug("Duplicate validation message found for parent={}, validation={}", participate.id, vState.id); + ValidationState vState = new ValidationState(ctx.clock(), desc, message.from()); + if (!register(message, participate, vState, + participate::register, + (d, i) -> participate.validation(d))) return; - } try { // trigger read-only compaction ColumnFamilyStore store = ColumnFamilyStore.getIfExists(desc.keyspace, desc.columnFamily); if (store == null) { - logger.error("Table {}.{} was dropped during validation phase of repair {}", - desc.keyspace, desc.columnFamily, desc.parentSessionId); - vState.phase.fail(String.format("Table %s.%s was dropped", desc.keyspace, desc.columnFamily)); - MessagingService.instance().send(Message.out(VALIDATION_RSP, new ValidationResponse(desc)), message.from()); + String msg = String.format("Table %s.%s was dropped during validation phase of repair %s", desc.keyspace, desc.columnFamily, desc.parentSessionId); + vState.phase.fail(msg); + logErrorAndSendFailureResponse(msg, message); return; } - ActiveRepairService.instance.consistent.local.maybeSetRepairing(desc.parentSessionId); + try + { + ctx.repair().consistent.local.maybeSetRepairing(desc.parentSessionId); + } + catch (Throwable t) + { + JVMStabilityInspector.inspectThrowable(t); + vState.phase.fail(t.toString()); + logErrorAndSendFailureResponse(t.toString(), message); + return; + } PreviewKind previewKind; try { @@ -189,13 +224,15 @@ public class RepairMessageVerbHandler implements IVerbHandler { logger.warn("Parent repair session {} has been removed, failing repair", desc.parentSessionId); vState.phase.fail(e); - MessagingService.instance().send(Message.out(VALIDATION_RSP, new ValidationResponse(desc)), message.from()); + sendFailureResponse(message); return; } + vState.phase.accept(); + sendAck(message); - Validator validator = new Validator(vState, validationRequest.nowInSec, + Validator validator = new Validator(ctx, vState, validationRequest.nowInSec, isIncremental(desc.parentSessionId), previewKind); - ValidationManager.instance.submitValidation(store, validator); + ctx.validationManager().submitValidation(store, validator); } catch (Throwable t) { @@ -207,12 +244,23 @@ public class RepairMessageVerbHandler implements IVerbHandler case SYNC_REQ: { - // notify initiator that the message has been received, allowing this method to take as long as it needs to - MessagingService.instance().send(message.emptyResponse(), message.from()); // forwarded sync request SyncRequest request = (SyncRequest) message.payload; logger.debug("Syncing {}", request); - StreamingRepairTask task = new StreamingRepairTask(desc, + + ParticipateState participate = ctx.repair().participate(desc.parentSessionId); + if (participate == null) + { + logErrorAndSendFailureResponse("Unknown repair " + desc.parentSessionId, message); + return; + } + SyncState state = new SyncState(ctx.clock(), desc, request.initiator, request.src, request.dst); + if (!register(message, participate, state, + participate::register, + participate::sync)) + return; + state.phase.accept(); + StreamingRepairTask task = new StreamingRepairTask(ctx, state, desc, request.initiator, request.src, request.dst, @@ -221,6 +269,7 @@ public class RepairMessageVerbHandler implements IVerbHandler request.previewKind, request.asymmetric); task.run(); + sendAck(message); } break; @@ -228,50 +277,50 @@ public class RepairMessageVerbHandler implements IVerbHandler { logger.debug("cleaning up repair"); CleanupMessage cleanup = (CleanupMessage) message.payload; - ParticipateState state = ActiveRepairService.instance.participate(cleanup.parentRepairSession); + ParticipateState state = ctx.repair().participate(cleanup.parentRepairSession); if (state != null) state.phase.success("Cleanup message recieved"); - ActiveRepairService.instance.removeParentRepairSession(cleanup.parentRepairSession); - MessagingService.instance().send(message.emptyResponse(), message.from()); + ctx.repair().removeParentRepairSession(cleanup.parentRepairSession); + sendAck(message); } break; case PREPARE_CONSISTENT_REQ: - ActiveRepairService.instance.consistent.local.handlePrepareMessage(message.from(), (PrepareConsistentRequest) message.payload); + ctx.repair().consistent.local.handlePrepareMessage(message.from(), (PrepareConsistentRequest) message.payload); break; case PREPARE_CONSISTENT_RSP: - ActiveRepairService.instance.consistent.coordinated.handlePrepareResponse((PrepareConsistentResponse) message.payload); + ctx.repair().consistent.coordinated.handlePrepareResponse((PrepareConsistentResponse) message.payload); break; case FINALIZE_PROPOSE_MSG: - ActiveRepairService.instance.consistent.local.handleFinalizeProposeMessage(message.from(), (FinalizePropose) message.payload); + ctx.repair().consistent.local.handleFinalizeProposeMessage(message.from(), (FinalizePropose) message.payload); break; case FINALIZE_PROMISE_MSG: - ActiveRepairService.instance.consistent.coordinated.handleFinalizePromiseMessage((FinalizePromise) message.payload); + ctx.repair().consistent.coordinated.handleFinalizePromiseMessage((FinalizePromise) message.payload); break; case FINALIZE_COMMIT_MSG: - ActiveRepairService.instance.consistent.local.handleFinalizeCommitMessage(message.from(), (FinalizeCommit) message.payload); + ctx.repair().consistent.local.handleFinalizeCommitMessage(message.from(), (FinalizeCommit) message.payload); break; case FAILED_SESSION_MSG: FailSession failure = (FailSession) message.payload; - ActiveRepairService.instance.consistent.coordinated.handleFailSessionMessage(failure); - ActiveRepairService.instance.consistent.local.handleFailSessionMessage(message.from(), failure); + ctx.repair().consistent.coordinated.handleFailSessionMessage(failure); + ctx.repair().consistent.local.handleFailSessionMessage(message.from(), failure); break; case STATUS_REQ: - ActiveRepairService.instance.consistent.local.handleStatusRequest(message.from(), (StatusRequest) message.payload); + ctx.repair().consistent.local.handleStatusRequest(message.from(), (StatusRequest) message.payload); break; case STATUS_RSP: - ActiveRepairService.instance.consistent.local.handleStatusResponse(message.from(), (StatusResponse) message.payload); + ctx.repair().consistent.local.handleStatusResponse(message.from(), (StatusResponse) message.payload); break; default: - ActiveRepairService.instance.handleMessage(message); + ctx.repair().handleMessage(message); break; } } @@ -280,15 +329,79 @@ public class RepairMessageVerbHandler implements IVerbHandler logger.error("Got error, removing parent repair session"); if (desc != null && desc.parentSessionId != null) { - ParticipateState parcipate = ActiveRepairService.instance.participate(desc.parentSessionId); + ParticipateState parcipate = ctx.repair().participate(desc.parentSessionId); if (parcipate != null) parcipate.phase.fail(e); - ActiveRepairService.instance.removeParentRepairSession(desc.parentSessionId); + ctx.repair().removeParentRepairSession(desc.parentSessionId); } throw new RuntimeException(e); } } + private > boolean register(Message message, + ParticipateState participate, + T vState, + Function register, + BiFunction getter) + { + ParticipateState.RegisterStatus registerStatus = register.apply(vState); + switch (registerStatus) + { + case ACCEPTED: + return true; + case EXISTS: + logger.debug("Duplicate validation message found for parent={}, validation={}", participate.id, vState.id); + replyDedup(getter.apply(message.payload.desc, vState.id), message); + return false; + case ALREADY_COMPLETED: + case STATUS_REJECTED: + // the repair is complete (most likely failed as we don't know success always), or is at a later phase such as sync + // so send a nack saying that the validation could not be accepted + sendFailureResponse(message); + return false; + default: + throw new IllegalStateException("Unexpected status: " + registerStatus); + } + } + + private enum DedupResult { UNKNOWN, ACCEPT, REJECT } + + private static DedupResult dedupResult(AbstractCompletable state) + { + AbstractCompletable.Status status = state.getCompletionStatus(); + switch (status) + { + case INIT: + return DedupResult.UNKNOWN; + case ACCEPTED: + return DedupResult.ACCEPT; + case COMPLETED: + return state.getResult().kind == Completable.Result.Kind.FAILURE ? DedupResult.REJECT: DedupResult.ACCEPT; + default: + throw new IllegalStateException("Unknown status: " + state); + } + } + + private void replyDedup(AbstractCompletable state, Message message) + { + if (state == null) + throw new IllegalStateException("State is null"); + DedupResult result = dedupResult(state); + switch (result) + { + case ACCEPT: + sendAck(message); + break; + case REJECT: + sendFailureResponse(message); + break; + case UNKNOWN: + break; + default: + throw new IllegalStateException("Unknown result: " + result); + } + } + private void logErrorAndSendFailureResponse(String errorMessage, Message respondTo) { logger.error(errorMessage); @@ -298,6 +411,11 @@ public class RepairMessageVerbHandler implements IVerbHandler private void sendFailureResponse(Message respondTo) { Message reply = respondTo.failureResponse(RequestFailureReason.UNKNOWN); - MessagingService.instance().send(reply, respondTo.from()); + ctx.messaging().send(reply, respondTo.from()); + } + + private void sendAck(Message message) + { + ctx.messaging().send(message.emptyResponse(), message.from()); } } diff --git a/src/java/org/apache/cassandra/repair/RepairSession.java b/src/java/org/apache/cassandra/repair/RepairSession.java index 6fb455b47a..e0e104ff8e 100644 --- a/src/java/org/apache/cassandra/repair/RepairSession.java +++ b/src/java/org/apache/cassandra/repair/RepairSession.java @@ -27,20 +27,21 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.TimeUnit; +import java.util.concurrent.Executor; +import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.atomic.AtomicBoolean; +import javax.annotation.Nullable; + import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Lists; import com.google.common.util.concurrent.*; -import org.apache.cassandra.concurrent.ExecutorFactory; -import org.apache.cassandra.concurrent.ExecutorPlus; -import org.apache.cassandra.utils.TimeUUID; -import org.apache.cassandra.repair.state.SessionState; -import org.apache.cassandra.utils.concurrent.AsyncFuture; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.cassandra.concurrent.ExecutorPlus; +import org.apache.cassandra.concurrent.Stage; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.Keyspace; @@ -49,19 +50,23 @@ import org.apache.cassandra.dht.Token; import org.apache.cassandra.exceptions.RepairException; import org.apache.cassandra.gms.*; import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.net.Message; import org.apache.cassandra.repair.consistent.ConsistentSession; import org.apache.cassandra.repair.consistent.LocalSession; import org.apache.cassandra.repair.consistent.LocalSessions; +import org.apache.cassandra.repair.messages.SyncResponse; +import org.apache.cassandra.repair.messages.ValidationResponse; +import org.apache.cassandra.repair.state.SessionState; import org.apache.cassandra.schema.SystemDistributedKeyspace; import org.apache.cassandra.schema.TableId; import org.apache.cassandra.streaming.PreviewKind; -import org.apache.cassandra.streaming.SessionSummary; import org.apache.cassandra.tracing.Tracing; import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.MerkleTrees; import org.apache.cassandra.utils.Pair; import org.apache.cassandra.utils.Throwables; -import org.apache.cassandra.utils.concurrent.Future; +import org.apache.cassandra.utils.TimeUUID; +import org.apache.cassandra.utils.concurrent.AsyncFuture; /** * Coordinates the (active) repair of a list of non overlapping token ranges. @@ -125,8 +130,10 @@ public class RepairSession extends AsyncFuture implements I private final ConcurrentMap, CompletableRemoteSyncTask> syncingTasks = new ConcurrentHashMap<>(); // Tasks(snapshot, validate request, differencing, ...) are run on taskExecutor - public final ExecutorPlus taskExecutor; + public final SafeExecutor taskExecutor; public final boolean optimiseStreams; + public final SharedContext ctx; + private volatile List jobs = Collections.emptyList(); private volatile boolean terminated = false; @@ -141,7 +148,8 @@ public class RepairSession extends AsyncFuture implements I * @param paxosOnly true if we should only complete paxos operations, not run a normal repair * @param cfnames names of columnfamilies */ - public RepairSession(TimeUUID parentRepairSession, + public RepairSession(SharedContext ctx, + TimeUUID parentRepairSession, CommonRange commonRange, String keyspace, RepairParallelism parallelismDegree, @@ -153,21 +161,23 @@ public class RepairSession extends AsyncFuture implements I boolean paxosOnly, String... cfnames) { + this.ctx = ctx; this.repairPaxos = repairPaxos; this.paxosOnly = paxosOnly; assert cfnames.length > 0 : "Repairing no column families seems pointless, doesn't it"; - this.state = new SessionState(parentRepairSession, keyspace, cfnames, commonRange); + this.state = new SessionState(ctx.clock(), parentRepairSession, keyspace, cfnames, commonRange); this.parallelismDegree = parallelismDegree; this.isIncremental = isIncremental; this.previewKind = previewKind; this.pullRepair = pullRepair; this.optimiseStreams = optimiseStreams; - this.taskExecutor = createExecutor(); + this.taskExecutor = new SafeExecutor(createExecutor(ctx)); } - protected ExecutorPlus createExecutor() + @VisibleForTesting + protected ExecutorPlus createExecutor(SharedContext ctx) { - return ExecutorFactory.Global.executorFactory().pooled("RepairJobTask", Integer.MAX_VALUE); + return ctx.executorFactory().pooled("RepairJobTask", Integer.MAX_VALUE); } public TimeUUID getId() @@ -185,13 +195,20 @@ public class RepairSession extends AsyncFuture implements I return state.commonRange.endpoints; } - public void trackValidationCompletion(Pair key, ValidationTask task) + public synchronized void trackValidationCompletion(Pair key, ValidationTask task) { + if (terminated) + { + task.abort(new RuntimeException("Session terminated")); + return; + } validating.put(key, task); } - public void trackSyncCompletion(Pair key, CompletableRemoteSyncTask task) + public synchronized void trackSyncCompletion(Pair key, CompletableRemoteSyncTask task) { + if (terminated) + return; syncingTasks.put(key, task); } @@ -199,26 +216,28 @@ public class RepairSession extends AsyncFuture implements I * Receive merkle tree response or failed response from {@code endpoint} for current repair job. * * @param desc repair job description - * @param endpoint endpoint that sent merkle tree - * @param trees calculated merkle trees, or null if validation failed + * @param message containing the merkle trees or an error */ - public void validationComplete(RepairJobDesc desc, InetAddressAndPort endpoint, MerkleTrees trees) + public void validationComplete(RepairJobDesc desc, Message message) { + InetAddressAndPort endpoint = message.from(); + MerkleTrees trees = message.payload.trees; ValidationTask task = validating.remove(Pair.create(desc, endpoint)); + // replies without a callback get dropped, so if in mixed mode this should be ignored + ctx.messaging().send(message.emptyResponse(), message.from()); if (task == null) { - assert terminated : "The repair session should be terminated if the validation we're completing no longer exists."; - // The trees may be off-heap, and will therefore need to be released. if (trees != null) trees.release(); - + + // either the session completed so the validation is no longer needed, or this is a retry; in both cases there is nothing to do return; } - String message = String.format("Received merkle tree for %s from %s", desc.columnFamily, endpoint); - logger.info("{} {}", previewKind.logPrefix(getId()), message); - Tracing.traceRepair(message); + String msg = String.format("Received merkle tree for %s from %s", desc.columnFamily, endpoint); + logger.info("{} {}", previewKind.logPrefix(getId()), msg); + Tracing.traceRepair(msg); task.treesReceived(trees); } @@ -226,21 +245,20 @@ public class RepairSession extends AsyncFuture implements I * Notify this session that sync completed/failed with given {@code SyncNodePair}. * * @param desc synced repair job - * @param nodes nodes that completed sync - * @param success true if sync succeeded + * @param message nodes that completed sync and if they were successful */ - public void syncComplete(RepairJobDesc desc, SyncNodePair nodes, boolean success, List summaries) + public void syncComplete(RepairJobDesc desc, Message message) { + SyncNodePair nodes = message.payload.nodes; CompletableRemoteSyncTask task = syncingTasks.remove(Pair.create(desc, nodes)); + // replies without a callback get dropped, so if in mixed mode this should be ignored + ctx.messaging().send(message.emptyResponse(), message.from()); if (task == null) - { - assert terminated : "The repair session should be terminated if the sync task we're completing no longer exists."; return; - } if (logger.isDebugEnabled()) logger.debug("{} Repair completed between {} and {} on {}", previewKind.logPrefix(getId()), nodes.coordinator, nodes.peer, desc.columnFamily); - task.syncComplete(success, summaries); + task.syncComplete(message.payload.success, message.payload.summaries); } @VisibleForTesting @@ -297,7 +315,7 @@ public class RepairSession extends AsyncFuture implements I // Checking all nodes are live for (InetAddressAndPort endpoint : state.commonRange.endpoints) { - if (!FailureDetector.instance.isAlive(endpoint) && !state.commonRange.hasSkippedReplicas) + if (!ctx.failureDetector().isAlive(endpoint) && !state.commonRange.hasSkippedReplicas) { message = String.format("Cannot proceed on repair because a neighbor (%s) is dead: session failed", endpoint); state.phase.fail(message); @@ -314,7 +332,7 @@ public class RepairSession extends AsyncFuture implements I // Create and submit RepairJob for each ColumnFamily state.phase.jobsSubmitted(); - List> jobs = new ArrayList<>(state.cfnames.length); + List jobs = new ArrayList<>(state.cfnames.length); for (String cfname : state.cfnames) { RepairJob job = new RepairJob(this, cfname); @@ -322,6 +340,7 @@ public class RepairSession extends AsyncFuture implements I executor.execute(job); jobs.add(job); } + this.jobs = jobs; // When all RepairJobs are done without error, cleanup and set the final result FBUtilities.allOf(jobs).addCallback(new FutureCallback>() @@ -334,10 +353,9 @@ public class RepairSession extends AsyncFuture implements I Tracing.traceRepair("Completed sync of range {}", state.commonRange); trySuccess(new RepairSessionResult(state.id, state.keyspace, state.commonRange.ranges, results, state.commonRange.hasSkippedReplicas)); - taskExecutor.shutdown(); // mark this session as terminated - terminate(); - awaitTaskExecutorTermination(); + terminate(null); + taskExecutor.shutdown(); } public void onFailure(Throwable t) @@ -351,12 +369,19 @@ public class RepairSession extends AsyncFuture implements I Tracing.traceRepair("Session completed with the following error: {}", t); forceShutdown(t); } - }, executor); + }, taskExecutor); } - public void terminate() + public synchronized void terminate(@Nullable Throwable reason) { terminated = true; + List jobs = this.jobs; + if (jobs != null) + { + for (RepairJob job : jobs) + job.abort(reason); + } + this.jobs = null; validating.clear(); syncingTasks.clear(); } @@ -369,26 +394,9 @@ public class RepairSession extends AsyncFuture implements I public void forceShutdown(Throwable reason) { tryFailure(reason); + terminate(reason); taskExecutor.shutdown(); - terminate(); - awaitTaskExecutorTermination(); } - - private void awaitTaskExecutorTermination() - { - try - { - if (taskExecutor.awaitTermination(30, TimeUnit.SECONDS)) - logger.debug("{} session task executor shut down gracefully", previewKind.logPrefix(getId())); - else - logger.warn("{} session task executor unable to shut down gracefully", previewKind.logPrefix(getId())); - } - catch (InterruptedException e) - { - Thread.currentThread().interrupt(); - } - } - public void onRemove(InetAddressAndPort endpoint) { convict(endpoint, Double.MAX_VALUE); @@ -452,4 +460,33 @@ public class RepairSession extends AsyncFuture implements I } return false; } + + private static class SafeExecutor implements Executor + { + private final ExecutorPlus delegate; + + private SafeExecutor(ExecutorPlus delegate) + { + this.delegate = delegate; + } + + @Override + public void execute(Runnable command) + { + try + { + delegate.execute(command); + } + catch (RejectedExecutionException e) + { + // task executor was shutdown, so fall back to a known good executor to finish callbacks + Stage.INTERNAL_RESPONSE.execute(command); + } + } + + public void shutdown() + { + delegate.shutdown(); + } + } } diff --git a/src/java/org/apache/cassandra/repair/SharedContext.java b/src/java/org/apache/cassandra/repair/SharedContext.java new file mode 100644 index 0000000000..1dc0607cfc --- /dev/null +++ b/src/java/org/apache/cassandra/repair/SharedContext.java @@ -0,0 +1,165 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.repair; + +import java.util.Random; +import java.util.function.Supplier; + +import org.apache.cassandra.concurrent.ExecutorFactory; +import org.apache.cassandra.concurrent.ScheduledExecutorPlus; +import org.apache.cassandra.concurrent.ScheduledExecutors; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.compaction.CompactionManager; +import org.apache.cassandra.db.compaction.ICompactionManager; +import org.apache.cassandra.gms.FailureDetector; +import org.apache.cassandra.gms.Gossiper; +import org.apache.cassandra.gms.IFailureDetector; +import org.apache.cassandra.gms.IGossiper; +import org.apache.cassandra.locator.IEndpointSnitch; +import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.net.MessageDelivery; +import org.apache.cassandra.net.MessagingService; +import org.apache.cassandra.service.ActiveRepairService; +import org.apache.cassandra.streaming.StreamPlan; +import org.apache.cassandra.utils.Clock; +import org.apache.cassandra.utils.FBUtilities; +import org.apache.cassandra.utils.MBeanWrapper; + +/** + * Access methods to shared resources and services. + *

+ * In many parts of the code base we reach into the global space to pull out singletons, but this makes testing much harder; the main goals for this type is to make users easier to test. + * + * @see {@link Global#instance} for the main production path + */ +public interface SharedContext +{ + InetAddressAndPort broadcastAddressAndPort(); + Supplier random(); + Clock clock(); + ExecutorFactory executorFactory(); + MBeanWrapper mbean(); + ScheduledExecutorPlus optionalTasks(); + + MessageDelivery messaging(); + IFailureDetector failureDetector(); + IEndpointSnitch snitch(); + IGossiper gossiper(); + ICompactionManager compactionManager(); + ActiveRepairService repair(); + IValidationManager validationManager(); + TableRepairManager repairManager(ColumnFamilyStore store); + StreamExecutor streamExecutor(); + + class Global implements SharedContext + { + public static final Global instance = new Global(); + + @Override + public InetAddressAndPort broadcastAddressAndPort() + { + return FBUtilities.getBroadcastAddressAndPort(); + } + + @Override + public Supplier random() + { + return Random::new; + } + + @Override + public Clock clock() + { + return Clock.Global.clock(); + } + + @Override + public ExecutorFactory executorFactory() + { + return ExecutorFactory.Global.executorFactory(); + } + + @Override + public MBeanWrapper mbean() + { + return MBeanWrapper.instance; + } + + @Override + public ScheduledExecutorPlus optionalTasks() + { + return ScheduledExecutors.optionalTasks; + } + + @Override + public MessageDelivery messaging() + { + return MessagingService.instance(); + } + + @Override + public IFailureDetector failureDetector() + { + return FailureDetector.instance; + } + + @Override + public IEndpointSnitch snitch() + { + return DatabaseDescriptor.getEndpointSnitch(); + } + + @Override + public IGossiper gossiper() + { + return Gossiper.instance; + } + + @Override + public ICompactionManager compactionManager() + { + return CompactionManager.instance; + } + + @Override + public ActiveRepairService repair() + { + return ActiveRepairService.instance(); + } + + @Override + public IValidationManager validationManager() + { + return ValidationManager.instance; + } + + @Override + public TableRepairManager repairManager(ColumnFamilyStore store) + { + return store.getRepairManager(); + } + + @Override + public StreamExecutor streamExecutor() + { + return StreamPlan::execute; + } + } +} diff --git a/src/java/org/apache/cassandra/repair/SnapshotTask.java b/src/java/org/apache/cassandra/repair/SnapshotTask.java index 7f5dcd7fae..ad45070cfb 100644 --- a/src/java/org/apache/cassandra/repair/SnapshotTask.java +++ b/src/java/org/apache/cassandra/repair/SnapshotTask.java @@ -23,11 +23,12 @@ import org.apache.cassandra.exceptions.RequestFailureReason; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.net.RequestCallback; import org.apache.cassandra.net.Message; -import org.apache.cassandra.net.MessagingService; +import org.apache.cassandra.repair.messages.RepairMessage; import org.apache.cassandra.repair.messages.SnapshotMessage; import org.apache.cassandra.utils.concurrent.AsyncFuture; import static org.apache.cassandra.net.Verb.SNAPSHOT_MSG; +import static org.apache.cassandra.repair.messages.RepairMessage.notDone; /** * SnapshotTask is a task that sends snapshot request. @@ -36,18 +37,18 @@ public class SnapshotTask extends AsyncFuture implements Run { private final RepairJobDesc desc; private final InetAddressAndPort endpoint; + private final SharedContext ctx; - SnapshotTask(RepairJobDesc desc, InetAddressAndPort endpoint) + SnapshotTask(SharedContext ctx, RepairJobDesc desc, InetAddressAndPort endpoint) { + this.ctx = ctx; this.desc = desc; this.endpoint = endpoint; } public void run() { - MessagingService.instance().sendWithCallback(Message.out(SNAPSHOT_MSG, new SnapshotMessage(desc)), - endpoint, - new SnapshotCallback(this)); + RepairMessage.sendMessageWithRetries(ctx, notDone(this), new SnapshotMessage(desc), SNAPSHOT_MSG, endpoint, new SnapshotCallback(this)); } /** diff --git a/src/java/org/apache/cassandra/repair/StreamExecutor.java b/src/java/org/apache/cassandra/repair/StreamExecutor.java new file mode 100644 index 0000000000..cb70d1ab75 --- /dev/null +++ b/src/java/org/apache/cassandra/repair/StreamExecutor.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.repair; + +import org.apache.cassandra.streaming.StreamPlan; +import org.apache.cassandra.streaming.StreamResultFuture; + +public interface StreamExecutor +{ + StreamResultFuture execute(StreamPlan plan); +} diff --git a/src/java/org/apache/cassandra/repair/StreamingRepairTask.java b/src/java/org/apache/cassandra/repair/StreamingRepairTask.java index ea50a50068..0f84d66893 100644 --- a/src/java/org/apache/cassandra/repair/StreamingRepairTask.java +++ b/src/java/org/apache/cassandra/repair/StreamingRepairTask.java @@ -21,6 +21,7 @@ import java.util.Collections; import java.util.Collection; import com.google.common.annotations.VisibleForTesting; + import org.apache.cassandra.locator.RangesAtEndpoint; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -28,9 +29,9 @@ import org.slf4j.LoggerFactory; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; import org.apache.cassandra.locator.InetAddressAndPort; -import org.apache.cassandra.net.Message; -import org.apache.cassandra.net.MessagingService; +import org.apache.cassandra.repair.messages.RepairMessage; import org.apache.cassandra.repair.messages.SyncResponse; +import org.apache.cassandra.repair.state.SyncState; import org.apache.cassandra.streaming.PreviewKind; import org.apache.cassandra.streaming.StreamEvent; import org.apache.cassandra.streaming.StreamEventHandler; @@ -53,6 +54,8 @@ public class StreamingRepairTask implements Runnable, StreamEventHandler { private static final Logger logger = LoggerFactory.getLogger(StreamingRepairTask.class); + private final SharedContext ctx; + private final SyncState state; private final RepairJobDesc desc; private final boolean asymmetric; private final InetAddressAndPort initiator; @@ -62,8 +65,10 @@ public class StreamingRepairTask implements Runnable, StreamEventHandler private final TimeUUID pendingRepair; private final PreviewKind previewKind; - public StreamingRepairTask(RepairJobDesc desc, InetAddressAndPort initiator, InetAddressAndPort src, InetAddressAndPort dst, Collection> ranges, TimeUUID pendingRepair, PreviewKind previewKind, boolean asymmetric) + public StreamingRepairTask(SharedContext ctx, SyncState state, RepairJobDesc desc, InetAddressAndPort initiator, InetAddressAndPort src, InetAddressAndPort dst, Collection> ranges, TimeUUID pendingRepair, PreviewKind previewKind, boolean asymmetric) { + this.ctx = ctx; + this.state = state; this.desc = desc; this.initiator = initiator; this.src = src; @@ -80,12 +85,14 @@ public class StreamingRepairTask implements Runnable, StreamEventHandler long start = approxTime.now(); StreamPlan streamPlan = createStreamPlan(dst); logger.info("[streaming task #{}] Stream plan created in {}ms", desc.sessionId, MILLISECONDS.convert(approxTime.now() - start, NANOSECONDS)); - streamPlan.execute(); + state.phase.start(); + ctx.streamExecutor().execute(streamPlan); } @VisibleForTesting StreamPlan createStreamPlan(InetAddressAndPort dest) { + state.phase.planning(); StreamPlan sp = new StreamPlan(StreamOperation.REPAIR, 1, false, pendingRepair, previewKind) .listeners(this) .flushBeforeTransfer(pendingRepair == null) // sstables are isolated at the beginning of an incremental repair session, so flushing isn't neccessary @@ -98,6 +105,7 @@ public class StreamingRepairTask implements Runnable, StreamEventHandler return sp; } + @Override public void handleStreamEvent(StreamEvent event) { // Nothing to do here, all we care about is the final success or failure and that's handled by @@ -107,17 +115,21 @@ public class StreamingRepairTask implements Runnable, StreamEventHandler /** * If we succeeded on both stream in and out, respond back to coordinator */ + @Override public void onSuccess(StreamState state) { logger.info("[repair #{}] streaming task succeed, returning response to {}", desc.sessionId, initiator); - MessagingService.instance().send(Message.out(SYNC_RSP, new SyncResponse(desc, src, dst, true, state.createSummaries())), initiator); + this.state.phase.success(); + RepairMessage.sendMessageWithRetries(ctx, new SyncResponse(desc, src, dst, true, state.createSummaries()), SYNC_RSP, initiator); } /** * If we failed on either stream in or out, respond fail to coordinator */ + @Override public void onFailure(Throwable t) { - MessagingService.instance().send(Message.out(SYNC_RSP, new SyncResponse(desc, src, dst, false, Collections.emptyList())), initiator); + this.state.phase.fail(t); + RepairMessage.sendMessageWithRetries(ctx, new SyncResponse(desc, src, dst, false, Collections.emptyList()), SYNC_RSP, initiator); } } diff --git a/src/java/org/apache/cassandra/repair/SymmetricRemoteSyncTask.java b/src/java/org/apache/cassandra/repair/SymmetricRemoteSyncTask.java index 96f07f3fb8..95b8959407 100644 --- a/src/java/org/apache/cassandra/repair/SymmetricRemoteSyncTask.java +++ b/src/java/org/apache/cassandra/repair/SymmetricRemoteSyncTask.java @@ -31,7 +31,6 @@ import org.apache.cassandra.repair.messages.SyncRequest; import org.apache.cassandra.streaming.PreviewKind; import org.apache.cassandra.streaming.SessionSummary; import org.apache.cassandra.tracing.Tracing; -import org.apache.cassandra.utils.FBUtilities; /** * SymmetricRemoteSyncTask sends {@link SyncRequest} to remote(non-coordinator) node @@ -43,15 +42,15 @@ public class SymmetricRemoteSyncTask extends SyncTask implements CompletableRemo { private static final Logger logger = LoggerFactory.getLogger(SymmetricRemoteSyncTask.class); - public SymmetricRemoteSyncTask(RepairJobDesc desc, InetAddressAndPort r1, InetAddressAndPort r2, List> differences, PreviewKind previewKind) + public SymmetricRemoteSyncTask(SharedContext ctx, RepairJobDesc desc, InetAddressAndPort r1, InetAddressAndPort r2, List> differences, PreviewKind previewKind) { - super(desc, r1, r2, differences, previewKind); + super(ctx, desc, r1, r2, differences, previewKind); } @Override protected void startSync() { - InetAddressAndPort local = FBUtilities.getBroadcastAddressAndPort(); + InetAddressAndPort local = ctx.broadcastAddressAndPort(); SyncRequest request = new SyncRequest(desc, local, nodePair.coordinator, nodePair.peer, rangesToSync, previewKind, false); Preconditions.checkArgument(nodePair.coordinator.equals(request.src)); String message = String.format("Forwarding streaming repair of %d ranges to %s (to be streamed with %s)", request.ranges.size(), request.src, request.dst); diff --git a/src/java/org/apache/cassandra/repair/SyncTask.java b/src/java/org/apache/cassandra/repair/SyncTask.java index 7393effd2d..a3b1a57493 100644 --- a/src/java/org/apache/cassandra/repair/SyncTask.java +++ b/src/java/org/apache/cassandra/repair/SyncTask.java @@ -37,13 +37,14 @@ import org.apache.cassandra.repair.messages.SyncRequest; import org.apache.cassandra.streaming.PreviewKind; import org.apache.cassandra.tracing.Tracing; -import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis; import static org.apache.cassandra.net.Verb.SYNC_REQ; +import static org.apache.cassandra.repair.messages.RepairMessage.notDone; public abstract class SyncTask extends AsyncFuture implements Runnable { private static final Logger logger = LoggerFactory.getLogger(SyncTask.class); + protected final SharedContext ctx; protected final RepairJobDesc desc; @VisibleForTesting public final List> rangesToSync; @@ -53,9 +54,10 @@ public abstract class SyncTask extends AsyncFuture implements Runnable protected volatile long startTime = Long.MIN_VALUE; protected final SyncStat stat; - protected SyncTask(RepairJobDesc desc, InetAddressAndPort primaryEndpoint, InetAddressAndPort peer, List> rangesToSync, PreviewKind previewKind) + protected SyncTask(SharedContext ctx, RepairJobDesc desc, InetAddressAndPort primaryEndpoint, InetAddressAndPort peer, List> rangesToSync, PreviewKind previewKind) { Preconditions.checkArgument(!peer.equals(primaryEndpoint), "Sending and receiving node are the same: %s", peer); + this.ctx = ctx; this.desc = desc; this.rangesToSync = rangesToSync; this.nodePair = new SyncNodePair(primaryEndpoint, peer); @@ -75,8 +77,7 @@ public abstract class SyncTask extends AsyncFuture implements Runnable */ public final void run() { - startTime = currentTimeMillis(); - + startTime = ctx.clock().currentTimeMillis(); // choose a repair method based on the significance of the difference String format = String.format("%s Endpoints %s and %s %%s for %s", previewKind.logPrefix(desc.sessionId), nodePair.coordinator, nodePair.peer, desc.columnFamily); @@ -102,14 +103,17 @@ public abstract class SyncTask extends AsyncFuture implements Runnable protected void finished() { if (startTime != Long.MIN_VALUE) - Keyspace.open(desc.keyspace).getColumnFamilyStore(desc.columnFamily).metric.repairSyncTime.update(currentTimeMillis() - startTime, TimeUnit.MILLISECONDS); + Keyspace.open(desc.keyspace).getColumnFamilyStore(desc.columnFamily).metric.repairSyncTime.update(ctx.clock().currentTimeMillis() - startTime, TimeUnit.MILLISECONDS); } - public void abort() {} + public void abort(Throwable reason) + { + tryFailure(reason); + } void sendRequest(SyncRequest request, InetAddressAndPort to) { - RepairMessage.sendMessageWithFailureCB(request, + RepairMessage.sendMessageWithFailureCB(ctx, notDone(this), request, SYNC_REQ, to, this::tryFailure); diff --git a/src/java/org/apache/cassandra/repair/ValidationManager.java b/src/java/org/apache/cassandra/repair/ValidationManager.java index a6c9fa9ef2..cc796c5f5b 100644 --- a/src/java/org/apache/cassandra/repair/ValidationManager.java +++ b/src/java/org/apache/cassandra/repair/ValidationManager.java @@ -37,13 +37,12 @@ import org.apache.cassandra.dht.Token; import org.apache.cassandra.metrics.TableMetrics; import org.apache.cassandra.metrics.TopPartitionTracker; import org.apache.cassandra.repair.state.ValidationState; +import org.apache.cassandra.utils.Clock; import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.MerkleTree; import org.apache.cassandra.utils.MerkleTrees; -import static org.apache.cassandra.utils.Clock.Global.nanoTime; - -public class ValidationManager +public class ValidationManager implements IValidationManager { private static final Logger logger = LoggerFactory.getLogger(ValidationManager.class); @@ -99,8 +98,10 @@ public class ValidationManager * but without writing the merge result */ @SuppressWarnings("resource") - private void doValidation(ColumnFamilyStore cfs, Validator validator) throws IOException, NoSuchRepairSessionException + public static void doValidation(ColumnFamilyStore cfs, Validator validator) throws IOException, NoSuchRepairSessionException { + SharedContext ctx = validator.ctx; + Clock clock = ctx.clock(); // this isn't meant to be race-proof, because it's not -- it won't cause bugs for a CFS to be dropped // mid-validation, or to attempt to validate a droped CFS. this is just a best effort to avoid useless work, // particularly in the scenario where a validation is submitted before the drop, and there are compactions @@ -119,8 +120,8 @@ public class ValidationManager // Create Merkle trees suitable to hold estimated partitions for the given ranges. // We blindly assume that a partition is evenly distributed on all sstables for now. - long start = nanoTime(); - try (ValidationPartitionIterator vi = getValidationIterator(cfs.getRepairManager(), validator, topPartitionCollector)) + long start = clock.nanoTime(); + try (ValidationPartitionIterator vi = getValidationIterator(ctx.repairManager(cfs), validator, topPartitionCollector)) { state.phase.start(vi.estimatedPartitions(), vi.getEstimatedBytes()); MerkleTrees trees = createMerkleTrees(vi, validator.desc.ranges, cfs); @@ -148,7 +149,7 @@ public class ValidationManager } if (logger.isDebugEnabled()) { - long duration = TimeUnit.NANOSECONDS.toMillis(nanoTime() - start); + long duration = TimeUnit.NANOSECONDS.toMillis(clock.nanoTime() - start); logger.debug("Validation of {} partitions (~{}) finished in {} msec, for {}", state.partitionsProcessed, FBUtilities.prettyPrintMemory(state.estimatedTotalBytes), @@ -177,6 +178,7 @@ public class ValidationManager /** * Does not mutate data, so is not scheduled. */ + @Override public Future submitValidation(ColumnFamilyStore cfs, Validator validator) { Callable validation = new Callable() diff --git a/src/java/org/apache/cassandra/repair/ValidationTask.java b/src/java/org/apache/cassandra/repair/ValidationTask.java index 003e736b80..322e07cd2d 100644 --- a/src/java/org/apache/cassandra/repair/ValidationTask.java +++ b/src/java/org/apache/cassandra/repair/ValidationTask.java @@ -28,6 +28,7 @@ import org.apache.cassandra.utils.MerkleTrees; import org.apache.cassandra.utils.concurrent.AsyncFuture; import static org.apache.cassandra.net.Verb.VALIDATION_REQ; +import static org.apache.cassandra.repair.messages.RepairMessage.notDone; /** * ValidationTask sends {@link ValidationRequest} to a replica. @@ -39,11 +40,11 @@ public class ValidationTask extends AsyncFuture implements Runnabl private final InetAddressAndPort endpoint; private final long nowInSec; private final PreviewKind previewKind; - - private boolean active = true; + private final SharedContext ctx; - public ValidationTask(RepairJobDesc desc, InetAddressAndPort endpoint, long nowInSec, PreviewKind previewKind) + public ValidationTask(SharedContext ctx, RepairJobDesc desc, InetAddressAndPort endpoint, long nowInSec, PreviewKind previewKind) { + this.ctx = ctx; this.desc = desc; this.endpoint = endpoint; this.nowInSec = nowInSec; @@ -55,7 +56,8 @@ public class ValidationTask extends AsyncFuture implements Runnabl */ public void run() { - RepairMessage.sendMessageWithFailureCB(new ValidationRequest(desc, nowInSec), + RepairMessage.sendMessageWithFailureCB(ctx, notDone(this), + new ValidationRequest(desc, nowInSec), VALIDATION_REQ, endpoint, this::tryFailure); @@ -70,18 +72,12 @@ public class ValidationTask extends AsyncFuture implements Runnabl { if (trees == null) { - active = false; tryFailure(RepairException.warn(desc, previewKind, "Validation failed in " + endpoint)); } - else if (active) + else if (!trySuccess(new TreeResponse(endpoint, trees))) { - trySuccess(new TreeResponse(endpoint, trees)); - } - else - { - // If the task has already been aborted, just release the possibly off-heap trees and move along. + // If the task is done, just release the possibly off-heap trees and move along. trees.release(); - trySuccess(null); } } @@ -89,37 +85,32 @@ public class ValidationTask extends AsyncFuture implements Runnabl * Release any trees already received by this task, and place it a state where any trees * received subsequently will be properly discarded. */ - public synchronized void abort() + public synchronized void abort(Throwable reason) { - if (active) + if (!tryFailure(reason) && isSuccess()) { - if (isDone()) + try { - try - { - // If we're done, this should return immediately. - TreeResponse response = get(); - - if (response.trees != null) - response.trees.release(); - } - catch (InterruptedException e) - { - // Restore the interrupt. - Thread.currentThread().interrupt(); - } - catch (ExecutionException e) - { - // Do nothing here. If an exception was set, there were no trees to release. - } + // If we're done, this should return immediately. + TreeResponse response = get(); + + if (response.trees != null) + response.trees.release(); + } + catch (InterruptedException e) + { + // Restore the interrupt. + Thread.currentThread().interrupt(); + } + catch (ExecutionException e) + { + // Do nothing here. If an exception was set, there were no trees to release. } - - active = false; } } public synchronized boolean isActive() { - return active; + return !isDone(); } } diff --git a/src/java/org/apache/cassandra/repair/Validator.java b/src/java/org/apache/cassandra/repair/Validator.java index 4912fa93a2..d8ba929de6 100644 --- a/src/java/org/apache/cassandra/repair/Validator.java +++ b/src/java/org/apache/cassandra/repair/Validator.java @@ -22,6 +22,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Random; +import com.google.common.annotations.VisibleForTesting; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -37,10 +38,10 @@ import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.metrics.TopPartitionTracker; import org.apache.cassandra.net.Message; import org.apache.cassandra.net.MessagingService; +import org.apache.cassandra.repair.messages.RepairMessage; import org.apache.cassandra.repair.messages.ValidationResponse; import org.apache.cassandra.repair.state.ValidationState; import org.apache.cassandra.streaming.PreviewKind; -import org.apache.cassandra.service.ActiveRepairService; import org.apache.cassandra.tracing.Tracing; import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.MerkleTree; @@ -66,6 +67,7 @@ public class Validator implements Runnable public final long nowInSec; private final boolean evenTreeDistribution; public final boolean isIncremental; + public final SharedContext ctx; // null when all rows with the min token have been consumed private long validated; @@ -83,16 +85,22 @@ public class Validator implements Runnable public Validator(ValidationState state, long nowInSec, PreviewKind previewKind) { - this(state, nowInSec, false, false, previewKind); + this(SharedContext.Global.instance, state, nowInSec, false, false, previewKind); + } + + public Validator(SharedContext ctx, ValidationState state, long nowInSec, boolean isIncremental, PreviewKind previewKind) + { + this(ctx, state, nowInSec, false, isIncremental, previewKind); } public Validator(ValidationState state, long nowInSec, boolean isIncremental, PreviewKind previewKind) { - this(state, nowInSec, false, isIncremental, previewKind); + this(SharedContext.Global.instance, state, nowInSec, false, isIncremental, previewKind); } - public Validator(ValidationState state, long nowInSec, boolean evenTreeDistribution, boolean isIncremental, PreviewKind previewKind) + public Validator(SharedContext ctx, ValidationState state, long nowInSec, boolean evenTreeDistribution, boolean isIncremental, PreviewKind previewKind) { + this.ctx = ctx; this.state = state; this.desc = state.desc; this.initiator = state.initiator; @@ -118,7 +126,7 @@ public class Validator implements Runnable else { List keys = new ArrayList<>(); - Random random = new Random(); + Random random = ctx.random().get(); for (Range range : trees.ranges()) { @@ -269,11 +277,12 @@ public class Validator implements Runnable return !FBUtilities.getBroadcastAddressAndPort().equals(initiator); } - private void respond(ValidationResponse response) + @VisibleForTesting + void respond(ValidationResponse response) { if (initiatorIsRemote()) { - MessagingService.instance().send(Message.out(VALIDATION_RSP, response), initiator); + RepairMessage.sendMessageWithRetries(ctx, response, VALIDATION_RSP, initiator); return; } @@ -294,7 +303,7 @@ public class Validator implements Runnable { logger.error("Failed to move local merkle tree for {} off heap", desc, e); } - ActiveRepairService.instance.handleMessage(Message.out(VALIDATION_RSP, movedResponse)); + ctx.repair().handleMessage(Message.out(VALIDATION_RSP, movedResponse)); }); } } diff --git a/src/java/org/apache/cassandra/repair/consistent/ConsistentSession.java b/src/java/org/apache/cassandra/repair/consistent/ConsistentSession.java index 86ecfb76f0..76645f1adc 100644 --- a/src/java/org/apache/cassandra/repair/consistent/ConsistentSession.java +++ b/src/java/org/apache/cassandra/repair/consistent/ConsistentSession.java @@ -44,6 +44,7 @@ import org.apache.cassandra.repair.messages.RepairOption; import org.apache.cassandra.repair.messages.StatusRequest; import org.apache.cassandra.repair.messages.StatusResponse; import org.apache.cassandra.repair.messages.ValidationRequest; +import org.apache.cassandra.repair.SharedContext; import org.apache.cassandra.schema.TableId; import org.apache.cassandra.service.ActiveRepairService; import org.apache.cassandra.tools.nodetool.RepairAdmin; @@ -187,6 +188,7 @@ public abstract class ConsistentSession } private volatile State state; + public final SharedContext ctx; public final TimeUUID sessionID; public final InetAddressAndPort coordinator; public final ImmutableSet tableIds; @@ -197,6 +199,7 @@ public abstract class ConsistentSession ConsistentSession(AbstractBuilder builder) { builder.validate(); + this.ctx = builder.ctx; this.state = builder.state; this.sessionID = builder.sessionID; this.coordinator = builder.coordinator; @@ -264,6 +267,7 @@ public abstract class ConsistentSession abstract static class AbstractBuilder { + private final SharedContext ctx; private State state; private TimeUUID sessionID; private InetAddressAndPort coordinator; @@ -272,6 +276,11 @@ public abstract class ConsistentSession private Collection> ranges; private Set participants; + protected AbstractBuilder(SharedContext ctx) + { + this.ctx = ctx; + } + void withState(State state) { this.state = state; diff --git a/src/java/org/apache/cassandra/repair/consistent/CoordinatorSession.java b/src/java/org/apache/cassandra/repair/consistent/CoordinatorSession.java index 88f30029c3..0ca1576df1 100644 --- a/src/java/org/apache/cassandra/repair/consistent/CoordinatorSession.java +++ b/src/java/org/apache/cassandra/repair/consistent/CoordinatorSession.java @@ -29,6 +29,7 @@ import com.google.common.base.Preconditions; import com.google.common.collect.Iterables; import org.apache.cassandra.concurrent.ImmediateExecutor; +import org.apache.cassandra.repair.SharedContext; import org.apache.cassandra.repair.CoordinatedRepairResult; import org.apache.cassandra.utils.concurrent.AsyncPromise; import org.apache.cassandra.utils.concurrent.Future; @@ -40,7 +41,6 @@ import org.slf4j.LoggerFactory; import org.apache.cassandra.exceptions.RepairException; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.net.Message; -import org.apache.cassandra.net.MessagingService; import org.apache.cassandra.net.Verb; import org.apache.cassandra.repair.SomeRepairFailedException; import org.apache.cassandra.repair.messages.FailSession; @@ -51,8 +51,6 @@ import org.apache.cassandra.repair.messages.RepairMessage; import org.apache.cassandra.service.ActiveRepairService; import org.apache.cassandra.utils.concurrent.ImmediateFuture; -import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis; - /** * Coordinator side logic and state of a consistent repair session. Like {@link ActiveRepairService.ParentRepairSession}, * there is only one {@code CoordinatorSession} per user repair command, regardless of the number of tables and token @@ -62,6 +60,7 @@ public class CoordinatorSession extends ConsistentSession { private static final Logger logger = LoggerFactory.getLogger(CoordinatorSession.class); + private final SharedContext ctx; private final Map participantStates = new HashMap<>(); private final AsyncPromise prepareFuture = AsyncPromise.uncancellable(); private final AsyncPromise finalizeProposeFuture = AsyncPromise.uncancellable(); @@ -73,6 +72,7 @@ public class CoordinatorSession extends ConsistentSession public CoordinatorSession(Builder builder) { super(builder); + ctx = builder.ctx == null ? SharedContext.Global.instance : builder.ctx; for (InetAddressAndPort participant : participants) { participantStates.put(participant, State.PREPARING); @@ -81,6 +81,19 @@ public class CoordinatorSession extends ConsistentSession public static class Builder extends AbstractBuilder { + private SharedContext ctx; + + public Builder(SharedContext ctx) + { + super(ctx); + } + + public Builder withContext(SharedContext ctx) + { + this.ctx = ctx; + return this; + } + public CoordinatorSession build() { validate(); @@ -88,9 +101,9 @@ public class CoordinatorSession extends ConsistentSession } } - public static Builder builder() + public static Builder builder(SharedContext ctx) { - return new Builder(); + return new Builder(ctx); } public void setState(State state) @@ -144,7 +157,8 @@ public class CoordinatorSession extends ConsistentSession protected void sendMessage(InetAddressAndPort destination, Message message) { logger.trace("Sending {} to {}", message.payload, destination); - MessagingService.instance().send(message, destination); + + ctx.messaging().send(message, destination); } public Future prepare() @@ -297,12 +311,12 @@ public class CoordinatorSession extends ConsistentSession { logger.info("Beginning coordination of incremental repair session {}", sessionID); - sessionStart = currentTimeMillis(); + sessionStart = ctx.clock().currentTimeMillis(); Future prepareResult = prepare(); // run repair sessions normally Future repairSessionResults = prepareResult.flatMap(ignore -> { - repairStart = currentTimeMillis(); + repairStart = ctx.clock().currentTimeMillis(); if (logger.isDebugEnabled()) logger.debug("Incremental repair {} prepare phase completed in {}", sessionID, formatDuration(sessionStart, repairStart)); setRepairing(); @@ -311,7 +325,7 @@ public class CoordinatorSession extends ConsistentSession // if any session failed, then fail the future Future onlySuccessSessionResults = repairSessionResults.flatMap(result -> { - finalizeStart = currentTimeMillis(); + finalizeStart = ctx.clock().currentTimeMillis(); if (result.hasFailed()) { if (logger.isDebugEnabled()) @@ -324,10 +338,10 @@ public class CoordinatorSession extends ConsistentSession // mark propose finalization and commit Future proposeFuture = onlySuccessSessionResults.flatMap(results -> finalizePropose().map(ignore -> { if (logger.isDebugEnabled()) - logger.debug("Incremental repair {} finalization phase completed in {}", sessionID, formatDuration(finalizeStart, currentTimeMillis())); + logger.debug("Incremental repair {} finalization phase completed in {}", sessionID, formatDuration(finalizeStart, ctx.clock().currentTimeMillis())); finalizeCommit(); if (logger.isDebugEnabled()) - logger.debug("Incremental repair {} phase completed in {}", sessionID, formatDuration(sessionStart, currentTimeMillis())); + logger.debug("Incremental repair {} phase completed in {}", sessionID, formatDuration(sessionStart, ctx.clock().currentTimeMillis())); return results; })); @@ -335,7 +349,7 @@ public class CoordinatorSession extends ConsistentSession if (failure != null) { if (logger.isDebugEnabled()) - logger.debug("Incremental repair {} phase failed in {}", sessionID, formatDuration(sessionStart, currentTimeMillis())); + logger.debug("Incremental repair {} phase failed in {}", sessionID, formatDuration(sessionStart, ctx.clock().currentTimeMillis())); fail(); } }, ImmediateExecutor.INSTANCE); diff --git a/src/java/org/apache/cassandra/repair/consistent/CoordinatorSessions.java b/src/java/org/apache/cassandra/repair/consistent/CoordinatorSessions.java index 328c16e645..1abab8c476 100644 --- a/src/java/org/apache/cassandra/repair/consistent/CoordinatorSessions.java +++ b/src/java/org/apache/cassandra/repair/consistent/CoordinatorSessions.java @@ -24,6 +24,7 @@ import java.util.Set; import com.google.common.base.Preconditions; +import org.apache.cassandra.repair.SharedContext; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.repair.messages.FailSession; import org.apache.cassandra.repair.messages.FinalizePromise; @@ -37,8 +38,15 @@ import org.apache.cassandra.utils.TimeUUID; */ public class CoordinatorSessions { + private final SharedContext ctx; + private final Map sessions = new HashMap<>(); + public CoordinatorSessions(SharedContext ctx) + { + this.ctx = ctx; + } + protected CoordinatorSession buildSession(CoordinatorSession.Builder builder) { return new CoordinatorSession(builder); @@ -46,14 +54,14 @@ public class CoordinatorSessions public synchronized CoordinatorSession registerSession(TimeUUID sessionId, Set participants, boolean isForced) throws NoSuchRepairSessionException { - ActiveRepairService.ParentRepairSession prs = ActiveRepairService.instance.getParentRepairSession(sessionId); + ActiveRepairService.ParentRepairSession prs = ctx.repair().getParentRepairSession(sessionId); Preconditions.checkArgument(!sessions.containsKey(sessionId), "A coordinator already exists for session %s", sessionId); Preconditions.checkArgument(!isForced || prs.repairedAt == ActiveRepairService.UNREPAIRED_SSTABLE, "cannot promote data for forced incremental repairs"); - CoordinatorSession.Builder builder = CoordinatorSession.builder(); + CoordinatorSession.Builder builder = CoordinatorSession.builder(ctx); builder.withState(ConsistentSession.State.PREPARING); builder.withSessionID(sessionId); builder.withCoordinator(prs.coordinator); @@ -62,6 +70,7 @@ public class CoordinatorSessions builder.withRepairedAt(prs.repairedAt); builder.withRanges(prs.getRanges()); builder.withParticipants(participants); + builder.withContext(ctx); CoordinatorSession session = buildSession(builder); sessions.put(session.sessionID, session); return session; diff --git a/src/java/org/apache/cassandra/repair/consistent/LocalSession.java b/src/java/org/apache/cassandra/repair/consistent/LocalSession.java index a0d1ac05c8..06c6755562 100644 --- a/src/java/org/apache/cassandra/repair/consistent/LocalSession.java +++ b/src/java/org/apache/cassandra/repair/consistent/LocalSession.java @@ -22,7 +22,7 @@ import java.util.Objects; import com.google.common.base.Preconditions; -import org.apache.cassandra.utils.FBUtilities; +import org.apache.cassandra.repair.SharedContext; /** * Basically just a record of a local session. All of the local session logic is implemented in {@link LocalSessions} @@ -57,7 +57,7 @@ public class LocalSession extends ConsistentSession public void setLastUpdate() { - lastUpdate = FBUtilities.nowInSeconds(); + lastUpdate = ctx.clock().nowInSeconds(); } public boolean equals(Object o) @@ -97,6 +97,11 @@ public class LocalSession extends ConsistentSession private long startedAt; private long lastUpdate; + public Builder(SharedContext ctx) + { + super(ctx); + } + public Builder withStartedAt(long startedAt) { this.startedAt = startedAt; @@ -123,8 +128,8 @@ public class LocalSession extends ConsistentSession } } - public static Builder builder() + public static Builder builder(SharedContext ctx) { - return new Builder(); + return new Builder(ctx); } } diff --git a/src/java/org/apache/cassandra/repair/consistent/LocalSessions.java b/src/java/org/apache/cassandra/repair/consistent/LocalSessions.java index 31df2e69bb..85bdf15923 100644 --- a/src/java/org/apache/cassandra/repair/consistent/LocalSessions.java +++ b/src/java/org/apache/cassandra/repair/consistent/LocalSessions.java @@ -74,11 +74,9 @@ import org.apache.cassandra.db.marshal.UUIDType; import org.apache.cassandra.dht.IPartitioner; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; -import org.apache.cassandra.gms.FailureDetector; import org.apache.cassandra.io.util.DataInputBuffer; import org.apache.cassandra.io.util.DataOutputBuffer; import org.apache.cassandra.net.Message; -import org.apache.cassandra.net.MessagingService; import org.apache.cassandra.repair.messages.FailSession; import org.apache.cassandra.repair.messages.FinalizeCommit; import org.apache.cassandra.repair.messages.FinalizePromise; @@ -88,16 +86,15 @@ import org.apache.cassandra.repair.messages.PrepareConsistentResponse; import org.apache.cassandra.repair.messages.RepairMessage; import org.apache.cassandra.repair.messages.StatusRequest; import org.apache.cassandra.repair.messages.StatusResponse; +import org.apache.cassandra.repair.SharedContext; import org.apache.cassandra.schema.TableId; import org.apache.cassandra.service.ActiveRepairService; import org.apache.cassandra.repair.NoSuchRepairSessionException; import org.apache.cassandra.service.StorageService; -import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.Throwables; import org.apache.cassandra.utils.TimeUUID; import org.apache.cassandra.utils.concurrent.Future; -import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory; import static org.apache.cassandra.config.CassandraRelevantProperties.REPAIR_CLEANUP_INTERVAL_SECONDS; import static org.apache.cassandra.config.CassandraRelevantProperties.REPAIR_DELETE_TIMEOUT_SECONDS; import static org.apache.cassandra.config.CassandraRelevantProperties.REPAIR_FAIL_TIMEOUT_SECONDS; @@ -154,10 +151,16 @@ public class LocalSessions private final String keyspace = SchemaConstants.SYSTEM_KEYSPACE_NAME; private final String table = SystemKeyspace.REPAIRS; + private final SharedContext ctx; private boolean started = false; private volatile ImmutableMap sessions = ImmutableMap.of(); private volatile ImmutableMap repairedStates = ImmutableMap.of(); + public LocalSessions(SharedContext ctx) + { + this.ctx = ctx; + } + @VisibleForTesting int getNumSessions() { @@ -167,13 +170,13 @@ public class LocalSessions @VisibleForTesting protected InetAddressAndPort getBroadcastAddressAndPort() { - return FBUtilities.getBroadcastAddressAndPort(); + return ctx.broadcastAddressAndPort(); } @VisibleForTesting protected boolean isAlive(InetAddressAndPort address) { - return FailureDetector.instance.isAlive(address); + return ctx.failureDetector().isAlive(address); } @VisibleForTesting @@ -443,7 +446,7 @@ public class LocalSessions { synchronized (session) { - long now = FBUtilities.nowInSeconds(); + long now = ctx.clock().nowInSeconds(); if (shouldFail(session, now)) { logger.warn("Auto failing timed out repair session {}", session); @@ -564,7 +567,7 @@ public class LocalSessions private LocalSession load(UntypedResultSet.Row row) { - LocalSession.Builder builder = LocalSession.builder(); + LocalSession.Builder builder = LocalSession.builder(ctx); builder.withState(ConsistentSession.State.valueOf(row.getInt("state"))); builder.withSessionID(row.getTimeUUID("parent_id")); InetAddressAndPort coordinator = InetAddressAndPort.getByAddressOverrideDefaults( @@ -662,7 +665,7 @@ public class LocalSessions @VisibleForTesting LocalSession createSessionUnsafe(TimeUUID sessionId, ActiveRepairService.ParentRepairSession prs, Set peers) { - LocalSession.Builder builder = LocalSession.builder(); + LocalSession.Builder builder = LocalSession.builder(ctx); builder.withState(ConsistentSession.State.PREPARING); builder.withSessionID(sessionId); builder.withCoordinator(prs.coordinator); @@ -672,7 +675,7 @@ public class LocalSessions builder.withRanges(prs.getRanges()); builder.withParticipants(peers); - long now = FBUtilities.nowInSeconds(); + long now = ctx.clock().nowInSeconds(); builder.withStartedAt(now); builder.withLastUpdate(now); @@ -681,13 +684,14 @@ public class LocalSessions protected ActiveRepairService.ParentRepairSession getParentRepairSession(TimeUUID sessionID) throws NoSuchRepairSessionException { - return ActiveRepairService.instance.getParentRepairSession(sessionID); + + return ctx.repair().getParentRepairSession(sessionID); } protected void sendMessage(InetAddressAndPort destination, Message message) { logger.trace("sending {} to {}", message.payload, destination); - MessagingService.instance().send(message, destination); + ctx.messaging().send(message, destination); } @VisibleForTesting @@ -821,7 +825,7 @@ public class LocalSessions putSessionUnsafe(session); logger.info("Beginning local incremental repair session {}", session); - ExecutorService executor = executorFactory().pooled("Repair-" + sessionID, parentSession.getColumnFamilyStores().size()); + ExecutorService executor = ctx.executorFactory().pooled("Repair-" + sessionID, parentSession.getColumnFamilyStores().size()); KeyspaceRepairManager repairManager = parentSession.getKeyspace().getRepairManager(); RangesAtEndpoint tokenRanges = filterLocalRanges(parentSession.getKeyspace().getName(), parentSession.getRanges()); @@ -1099,6 +1103,12 @@ public class LocalSessions listeners.remove(listener); } + @VisibleForTesting + public static void unsafeClearListeners() + { + listeners.clear(); + } + public interface Listener { void onIRStateChange(LocalSession session); diff --git a/src/java/org/apache/cassandra/repair/messages/CleanupMessage.java b/src/java/org/apache/cassandra/repair/messages/CleanupMessage.java index 0e1a74f3d7..add59060a7 100644 --- a/src/java/org/apache/cassandra/repair/messages/CleanupMessage.java +++ b/src/java/org/apache/cassandra/repair/messages/CleanupMessage.java @@ -40,6 +40,12 @@ public class CleanupMessage extends RepairMessage this.parentRepairSession = parentRepairSession; } + @Override + public TimeUUID parentRepairSession() + { + return parentRepairSession; + } + @Override public boolean equals(Object o) { diff --git a/src/java/org/apache/cassandra/repair/messages/PrepareMessage.java b/src/java/org/apache/cassandra/repair/messages/PrepareMessage.java index 7064227e11..bf76190fc3 100644 --- a/src/java/org/apache/cassandra/repair/messages/PrepareMessage.java +++ b/src/java/org/apache/cassandra/repair/messages/PrepareMessage.java @@ -61,6 +61,12 @@ public class PrepareMessage extends RepairMessage this.previewKind = previewKind; } + @Override + public TimeUUID parentRepairSession() + { + return parentRepairSession; + } + @Override public boolean equals(Object o) { diff --git a/src/java/org/apache/cassandra/repair/messages/RepairMessage.java b/src/java/org/apache/cassandra/repair/messages/RepairMessage.java index 00ce888696..d6d4d9b536 100644 --- a/src/java/org/apache/cassandra/repair/messages/RepairMessage.java +++ b/src/java/org/apache/cassandra/repair/messages/RepairMessage.java @@ -17,21 +17,34 @@ */ package org.apache.cassandra.repair.messages; +import java.util.Collections; +import java.util.EnumMap; +import java.util.EnumSet; +import java.util.Map; +import java.util.Set; +import java.util.function.Supplier; + +import javax.annotation.Nullable; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.config.RepairRetrySpec; +import org.apache.cassandra.config.RetrySpec; +import org.apache.cassandra.repair.SharedContext; import org.apache.cassandra.exceptions.RepairException; import org.apache.cassandra.exceptions.RequestFailureReason; -import org.apache.cassandra.gms.Gossiper; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.net.Message; -import org.apache.cassandra.net.MessagingService; import org.apache.cassandra.net.RequestCallback; import org.apache.cassandra.net.Verb; import org.apache.cassandra.repair.RepairJobDesc; import org.apache.cassandra.streaming.PreviewKind; +import org.apache.cassandra.utils.Backoff; import org.apache.cassandra.utils.CassandraVersion; import org.apache.cassandra.utils.TimeUUID; +import org.apache.cassandra.utils.concurrent.Future; import static org.apache.cassandra.net.MessageFlag.CALL_BACK_ON_FAILURE; @@ -42,29 +55,121 @@ import static org.apache.cassandra.net.MessageFlag.CALL_BACK_ON_FAILURE; */ public abstract class RepairMessage { - private static final CassandraVersion SUPPORTS_TIMEOUTS = new CassandraVersion("4.0.7-SNAPSHOT"); + private enum ErrorHandling { NONE, TIMEOUT, RETRY } + private static final CassandraVersion SUPPORTS_RETRY = new CassandraVersion("5.0.0-alpha2.SNAPSHOT"); + private static final Map VERB_TIMEOUT_VERSIONS; + + static + { + CassandraVersion timeoutVersion = new CassandraVersion("4.0.7-SNAPSHOT"); + EnumMap map = new EnumMap<>(Verb.class); + map.put(Verb.VALIDATION_REQ, timeoutVersion); + map.put(Verb.SYNC_REQ, timeoutVersion); + map.put(Verb.VALIDATION_RSP, SUPPORTS_RETRY); + map.put(Verb.SYNC_RSP, SUPPORTS_RETRY); + VERB_TIMEOUT_VERSIONS = Collections.unmodifiableMap(map); + } + private static final Set SUPPORTS_RETRY_WITHOUT_VERSION_CHECK = Collections.unmodifiableSet(EnumSet.of(Verb.CLEANUP_MSG)); + private static final Logger logger = LoggerFactory.getLogger(RepairMessage.class); + @Nullable public final RepairJobDesc desc; - protected RepairMessage(RepairJobDesc desc) + protected RepairMessage(@Nullable RepairJobDesc desc) { this.desc = desc; } + public TimeUUID parentRepairSession() + { + return desc.parentSessionId; + } + public interface RepairFailureCallback { void onFailure(Exception e); } - public static void sendMessageWithFailureCB(RepairMessage request, Verb verb, InetAddressAndPort endpoint, RepairFailureCallback failureCallback) + private static Backoff backoff(SharedContext ctx, Verb verb) { - RequestCallback callback = new RequestCallback() + RepairRetrySpec retrySpec = DatabaseDescriptor.getRepairRetrySpec(); + RetrySpec spec = verb == Verb.VALIDATION_RSP ? retrySpec.getMerkleTreeResponseSpec() : retrySpec; + if (!spec.isEnabled()) + return Backoff.None.INSTANCE; + return new Backoff.ExponentialBackoff(spec.maxAttempts.value, spec.baseSleepTime.toMilliseconds(), spec.maxSleepTime.toMilliseconds(), ctx.random().get()::nextDouble); + } + + public static Supplier notDone(Future f) + { + return () -> !f.isDone(); + } + + private static Supplier always() + { + return () -> true; + } + + public static void sendMessageWithRetries(SharedContext ctx, Supplier allowRetry, RepairMessage request, Verb verb, InetAddressAndPort endpoint, RequestCallback finalCallback) + { + sendMessageWithRetries(ctx, backoff(ctx, verb), allowRetry, request, verb, endpoint, finalCallback, 0); + } + + public static void sendMessageWithRetries(SharedContext ctx, RepairMessage request, Verb verb, InetAddressAndPort endpoint, RequestCallback finalCallback) + { + sendMessageWithRetries(ctx, backoff(ctx, verb), always(), request, verb, endpoint, finalCallback, 0); + } + + public static void sendMessageWithRetries(SharedContext ctx, RepairMessage request, Verb verb, InetAddressAndPort endpoint) + { + sendMessageWithRetries(ctx, backoff(ctx, verb), always(), request, verb, endpoint, new RequestCallback<>() { @Override public void onResponse(Message msg) { - logger.info("[#{}] {} received by {}", request.desc.parentSessionId, verb, endpoint); - // todo: at some point we should make repair messages follow the normal path, actually using this + } + + @Override + public void onFailure(InetAddressAndPort from, RequestFailureReason failureReason) + { + } + }, 0); + } + + private static void sendMessageWithRetries(SharedContext ctx, Backoff backoff, Supplier allowRetry, RepairMessage request, Verb verb, InetAddressAndPort endpoint, RequestCallback finalCallback, int attempt) + { + RequestCallback callback = new RequestCallback<>() + { + @Override + public void onResponse(Message msg) + { + finalCallback.onResponse(msg); + } + + @Override + public void onFailure(InetAddressAndPort from, RequestFailureReason failureReason) + { + ErrorHandling allowed = errorHandlingSupported(ctx, endpoint, verb, request.parentRepairSession()); + switch (allowed) + { + case NONE: + logger.error("[#{}] {} failed on {}: {}", request.parentRepairSession(), verb, from, failureReason); + return; + case TIMEOUT: + finalCallback.onFailure(from, failureReason); + return; + case RETRY: + int maxAttempts = backoff.maxAttempts(); + if (failureReason == RequestFailureReason.TIMEOUT && attempt < maxAttempts && allowRetry.get()) + { + ctx.optionalTasks().schedule(() -> sendMessageWithRetries(ctx, backoff, allowRetry, request, verb, endpoint, finalCallback, attempt + 1), + backoff.computeWaitTime(attempt), backoff.unit()); + return; + } + finalCallback.onFailure(from, failureReason); + return; + default: + throw new AssertionError("Unknown error handler: " + allowed); + } } @Override @@ -72,27 +177,59 @@ public abstract class RepairMessage { return true; } + }; + ctx.messaging().sendWithCallback(Message.outWithFlag(verb, request, CALL_BACK_ON_FAILURE), + endpoint, + callback); + } + public static void sendMessageWithFailureCB(SharedContext ctx, Supplier allowRetry, RepairMessage request, Verb verb, InetAddressAndPort endpoint, RepairFailureCallback failureCallback) + { + RequestCallback callback = new RequestCallback<>() + { + @Override + public void onResponse(Message msg) + { + logger.info("[#{}] {} received by {}", request.parentRepairSession(), verb, endpoint); + // todo: at some point we should make repair messages follow the normal path, actually using this + } + + @Override public void onFailure(InetAddressAndPort from, RequestFailureReason failureReason) { - logger.error("[#{}] {} failed on {}: {}", request.desc.parentSessionId, verb, from, failureReason); + failureCallback.onFailure(RepairException.error(request.desc, PreviewKind.NONE, String.format("Got %s failure from %s: %s", verb, from, failureReason))); + } - if (supportsTimeouts(from, request.desc.parentSessionId)) - failureCallback.onFailure(RepairException.error(request.desc, PreviewKind.NONE, String.format("Got %s failure from %s: %s", verb, from, failureReason))); + @Override + public boolean invokeOnFailure() + { + return true; } }; - - MessagingService.instance().sendWithCallback(Message.outWithFlag(verb, request, CALL_BACK_ON_FAILURE), - endpoint, - callback); + sendMessageWithRetries(ctx, allowRetry, request, verb, endpoint, callback); } - private static boolean supportsTimeouts(InetAddressAndPort from, TimeUUID parentSessionId) + private static ErrorHandling errorHandlingSupported(SharedContext ctx, InetAddressAndPort from, Verb verb, TimeUUID parentSessionId) { - CassandraVersion remoteVersion = Gossiper.instance.getReleaseVersion(from); - if (remoteVersion != null && remoteVersion.compareTo(SUPPORTS_TIMEOUTS) >= 0) - return true; - logger.warn("[#{}] Not failing repair due to remote host {} not supporting repair message timeouts (version = {})", parentSessionId, from, remoteVersion); - return false; + if (SUPPORTS_RETRY_WITHOUT_VERSION_CHECK.contains(verb)) + return ErrorHandling.RETRY; + // Repair in mixed mode isn't fully supported, but also not activally blocked... so in the common case all participants + // will be on the same version as this instance, so can avoid the lookup from gossip + CassandraVersion remoteVersion = ctx.gossiper().getReleaseVersion(from); + if (remoteVersion == null) + { + if (VERB_TIMEOUT_VERSIONS.containsKey(verb)) + { + logger.warn("[#{}] Not failing repair due to remote host {} not supporting repair message timeouts (version is unknown)", parentSessionId, from); + return ErrorHandling.NONE; + } + return ErrorHandling.TIMEOUT; + } + if (remoteVersion.compareTo(SUPPORTS_RETRY) >= 0) + return ErrorHandling.RETRY; + CassandraVersion timeoutVersion = VERB_TIMEOUT_VERSIONS.get(verb); + if (timeoutVersion == null || remoteVersion.compareTo(timeoutVersion) >= 0) + return ErrorHandling.TIMEOUT; + return ErrorHandling.NONE; } } diff --git a/src/java/org/apache/cassandra/repair/state/AbstractCompletable.java b/src/java/org/apache/cassandra/repair/state/AbstractCompletable.java index 20e59d65d5..e919dc314b 100644 --- a/src/java/org/apache/cassandra/repair/state/AbstractCompletable.java +++ b/src/java/org/apache/cassandra/repair/state/AbstractCompletable.java @@ -24,19 +24,35 @@ import com.google.common.base.Throwables; import org.apache.cassandra.utils.Clock; -public class AbstractCompletable implements Completable +public abstract class AbstractCompletable implements Completable { - private final long creationTimeMillis = Clock.Global.currentTimeMillis(); // used to convert from nanos to millis - private final long creationTimeNanos = Clock.Global.nanoTime(); + public enum Status { INIT, ACCEPTED, COMPLETED } + + private final long creationTimeMillis; // used to convert from nanos to millis + private final long creationTimeNanos; + protected final Clock clock; private final AtomicReference result = new AtomicReference<>(null); public final I id; protected volatile long lastUpdatedAtNs; - public AbstractCompletable(I id) + public AbstractCompletable(Clock clock, I id) { + this.creationTimeMillis = clock.currentTimeMillis(); + this.creationTimeNanos = clock.nanoTime(); + this.clock = clock; this.id = id; } + public abstract boolean isAccepted(); + + public Status getCompletionStatus() + { + Result result = getResult(); + if (result != null) + return Status.COMPLETED; + return isAccepted() ? Status.ACCEPTED : Status.INIT; + } + @Override public I getId() { @@ -75,7 +91,7 @@ public class AbstractCompletable implements Completable public void updated() { - lastUpdatedAtNs = Clock.Global.nanoTime(); + lastUpdatedAtNs = clock.nanoTime(); } protected boolean tryResult(Result result) @@ -83,7 +99,7 @@ public class AbstractCompletable implements Completable if (!this.result.compareAndSet(null, result)) return false; onComplete(); - lastUpdatedAtNs = Clock.Global.nanoTime(); + lastUpdatedAtNs = clock.nanoTime(); return true; } diff --git a/src/java/org/apache/cassandra/repair/state/AbstractState.java b/src/java/org/apache/cassandra/repair/state/AbstractState.java index f9b0a93645..08e172ce95 100644 --- a/src/java/org/apache/cassandra/repair/state/AbstractState.java +++ b/src/java/org/apache/cassandra/repair/state/AbstractState.java @@ -23,6 +23,27 @@ import org.apache.cassandra.utils.Clock; public abstract class AbstractState, I> extends AbstractCompletable implements State { + protected enum UpdateType + { + NO_CHANGE, ACCEPTED, + LARGER_STATE_SEEN, ALREADY_COMPLETED; + + protected boolean isRejected() + { + switch (this) + { + case NO_CHANGE: + case ACCEPTED: + return false; + case LARGER_STATE_SEEN: + case ALREADY_COMPLETED: + return true; + default: + throw new IllegalStateException("Unknown type: " + this); + } + } + } + public static final int INIT = -1; public static final int COMPLETE = -2; @@ -30,13 +51,19 @@ public abstract class AbstractState, I> extends AbstractComple protected final long[] stateTimesNanos; protected int currentState = INIT; - public AbstractState(I id, Class klass) + public AbstractState(Clock clock, I id, Class klass) { - super(id); + super(clock, id); this.klass = klass; this.stateTimesNanos = new long[klass.getEnumConstants().length]; } + @Override + public boolean isAccepted() + { + return currentState == INIT ? false : true; + } + @Override public T getStatus() { @@ -46,6 +73,27 @@ public abstract class AbstractState, I> extends AbstractComple return klass.getEnumConstants()[current]; } + public String status() + { + T state = getStatus(); + Result result = getResult(); + if (result != null) + return result.kind.name(); + if (state == null) + return "init"; + return state.name(); + } + + @Override + public String toString() + { + return getClass().getSimpleName() + "{" + + "id=" + id + + ", status=" + status() + + ", lastUpdatedAtNs=" + lastUpdatedAtNs + + '}'; + } + public int getCurrentState() { return currentState; @@ -85,11 +133,22 @@ public abstract class AbstractState, I> extends AbstractComple protected void updateState(T state) { - int currentState = this.currentState; - if (currentState >= state.ordinal()) + if (maybeUpdateState(state).isRejected()) throw new IllegalStateException("State went backwards; current=" + klass.getEnumConstants()[currentState] + ", desired=" + state); - long now = Clock.Global.nanoTime(); + } + + protected UpdateType maybeUpdateState(T state) + { + int currentState = this.currentState; + if (currentState == COMPLETE) + return UpdateType.ALREADY_COMPLETED; + if (currentState == state.ordinal()) + return UpdateType.NO_CHANGE; + if (currentState > state.ordinal()) + return UpdateType.LARGER_STATE_SEEN; + long now = clock.nanoTime(); stateTimesNanos[this.currentState = state.ordinal()] = now; lastUpdatedAtNs = now; + return UpdateType.ACCEPTED; } } diff --git a/src/java/org/apache/cassandra/repair/state/Completable.java b/src/java/org/apache/cassandra/repair/state/Completable.java index 61813c6974..1cf496d124 100644 --- a/src/java/org/apache/cassandra/repair/state/Completable.java +++ b/src/java/org/apache/cassandra/repair/state/Completable.java @@ -17,6 +17,7 @@ */ package org.apache.cassandra.repair.state; +import java.util.Objects; import java.util.concurrent.TimeUnit; import org.apache.cassandra.utils.Clock; @@ -78,24 +79,48 @@ public interface Completable this.message = message; } - protected static Result success() + public static Result success() { return new Result(Result.Kind.SUCCESS, null); } - protected static Result success(String msg) + public static Result success(String msg) { return new Result(Result.Kind.SUCCESS, msg); } - protected static Result skip(String msg) + public static Result skip(String msg) { return new Result(Result.Kind.SKIPPED, msg); } - protected static Result fail(String msg) + public static Result fail(String msg) { return new Result(Result.Kind.FAILURE, msg); } + + @Override + public boolean equals(Object o) + { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Result result = (Result) o; + return kind == result.kind && Objects.equals(message, result.message); + } + + @Override + public int hashCode() + { + return Objects.hash(kind, message); + } + + @Override + public String toString() + { + return "Result{" + + "kind=" + kind + + ", message='" + message + '\'' + + '}'; + } } } diff --git a/src/java/org/apache/cassandra/repair/state/CoordinatorState.java b/src/java/org/apache/cassandra/repair/state/CoordinatorState.java index 74ca3e393e..5bc8a9e5d8 100644 --- a/src/java/org/apache/cassandra/repair/state/CoordinatorState.java +++ b/src/java/org/apache/cassandra/repair/state/CoordinatorState.java @@ -17,18 +17,21 @@ */ package org.apache.cassandra.repair.state; +import java.util.Arrays; import java.util.Collection; import java.util.List; import java.util.Objects; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import java.util.stream.Collectors; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.repair.CommonRange; -import org.apache.cassandra.repair.RepairRunnable; +import org.apache.cassandra.repair.RepairCoordinator; import org.apache.cassandra.repair.messages.RepairOption; +import org.apache.cassandra.utils.Clock; import org.apache.cassandra.utils.TimeUUID; import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID; @@ -48,14 +51,14 @@ public class CoordinatorState extends AbstractState sessions = new ConcurrentHashMap<>(); private List columnFamilies = null; - private RepairRunnable.NeighborsAndRanges neighborsAndRanges = null; + private RepairCoordinator.NeighborsAndRanges neighborsAndRanges = null; // API to split function calls for phase changes from getting the state public final Phase phase = new Phase(); - public CoordinatorState(int cmd, String keyspace, RepairOption options) + public CoordinatorState(Clock clock, int cmd, String keyspace, RepairOption options) { - super(nextTimeUUID(), State.class); + super(clock, nextTimeUUID(), State.class); this.cmd = cmd; this.keyspace = Objects.requireNonNull(keyspace); this.options = Objects.requireNonNull(options); @@ -88,7 +91,7 @@ public class CoordinatorState extends AbstractState e.getKey() + " -> " + e.getValue().status()).collect(Collectors.toList()); + else + return currentState.name(); + } + + @Override + public String toString() + { + return "CoordinatorState{" + + "id=" + id + + ", stateTimesNanos=" + Arrays.toString(stateTimesNanos) + + ", status=" + status() + + ", lastUpdatedAtNs=" + lastUpdatedAtNs + + '}'; + } + public final class Phase extends BaseSkipPhase { public void setup() @@ -121,7 +150,7 @@ public class CoordinatorState extends AbstractState columnFamilies, RepairRunnable.NeighborsAndRanges neighborsAndRanges) + public void start(List columnFamilies, RepairCoordinator.NeighborsAndRanges neighborsAndRanges) { CoordinatorState.this.columnFamilies = Objects.requireNonNull(columnFamilies); CoordinatorState.this.neighborsAndRanges = Objects.requireNonNull(neighborsAndRanges); diff --git a/src/java/org/apache/cassandra/repair/state/JobState.java b/src/java/org/apache/cassandra/repair/state/JobState.java index f8d42c350e..6c02b6d73d 100644 --- a/src/java/org/apache/cassandra/repair/state/JobState.java +++ b/src/java/org/apache/cassandra/repair/state/JobState.java @@ -24,6 +24,7 @@ import com.google.common.collect.ImmutableSet; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.repair.RepairJobDesc; +import org.apache.cassandra.utils.Clock; public class JobState extends AbstractState { @@ -40,9 +41,9 @@ public class JobState extends AbstractState public final Phase phase = new Phase(); - public JobState(RepairJobDesc desc, ImmutableSet endpoints) + public JobState(Clock clock, RepairJobDesc desc, ImmutableSet endpoints) { - super(desc.determanisticId(), State.class); + super(clock, desc.determanisticId(), State.class); this.desc = desc; this.endpoints = endpoints; } diff --git a/src/java/org/apache/cassandra/repair/state/ParticipateState.java b/src/java/org/apache/cassandra/repair/state/ParticipateState.java index c1faa4e939..ede9cc2024 100644 --- a/src/java/org/apache/cassandra/repair/state/ParticipateState.java +++ b/src/java/org/apache/cassandra/repair/state/ParticipateState.java @@ -22,17 +22,26 @@ import java.util.List; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Function; +import java.util.stream.Collectors; + +import javax.annotation.Nullable; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.repair.RepairJobDesc; import org.apache.cassandra.repair.messages.PrepareMessage; import org.apache.cassandra.schema.TableId; import org.apache.cassandra.streaming.PreviewKind; +import org.apache.cassandra.utils.Clock; import org.apache.cassandra.utils.TimeUUID; public class ParticipateState extends AbstractCompletable { + public enum RegisterStatus + { ACCEPTED, EXISTS, STATUS_REJECTED, ALREADY_COMPLETED } public final InetAddressAndPort initiator; public final List tableIds; public final Collection> ranges; @@ -40,14 +49,21 @@ public class ParticipateState extends AbstractCompletable public final long repairedAt; public final boolean global; public final PreviewKind previewKind; - - private final ConcurrentMap validations = new ConcurrentHashMap<>(); + private volatile boolean accepted = false; public final Phase phase = new Phase(); - public ParticipateState(InetAddressAndPort initiator, PrepareMessage msg) + @Override + public boolean isAccepted() { - super(msg.parentRepairSession); + return accepted; + } + + public final ConcurrentMap jobs = new ConcurrentHashMap<>(); + + public ParticipateState(Clock clock, InetAddressAndPort initiator, PrepareMessage msg) + { + super(clock, msg.parentRepairSession); this.initiator = initiator; this.tableIds = msg.tableIds; this.ranges = msg.ranges; @@ -57,24 +73,150 @@ public class ParticipateState extends AbstractCompletable this.previewKind = msg.previewKind; } - public boolean register(ValidationState state) + @Nullable + public Job job(RepairJobDesc desc) { - ValidationState current = validations.putIfAbsent(state.id, state); - return current == null; + return jobs.get(desc); + } + + public Job getOrCreateJob(RepairJobDesc desc) + { + return jobs.computeIfAbsent(desc, d -> new Job(clock, d)); + } + + @Nullable + public ValidationState validation(RepairJobDesc desc) + { + Job job = job(desc); + if (job == null) + return null; + return job.validation(); + } + + public RegisterStatus register(ValidationState state) + { + return getOrCreateJob(state.desc).register(state); + } + + @Nullable + public SyncState sync(RepairJobDesc desc, SyncState.Id id) + { + Job job = job(desc); + if (job == null) + return null; + return job.sync(id); + } + + public RegisterStatus register(SyncState state) + { + return getOrCreateJob(state.id.desc).register(state); } public Collection validations() { - return validations.values(); + return jobs.values().stream() + .map(j -> j.validation()) + .filter(f -> f != null) + .collect(Collectors.toList()); } public Collection validationIds() { - return validations.keySet(); + return jobs.values().stream() + .map(j -> j.validation()) + .filter(f -> f != null) + .map(v -> v.id) + .collect(Collectors.toList()); + } + + @Override + public String toString() + { + Result result = getResult(); + return "ParticipateState{" + + "initiator=" + initiator + + ", status=" + (result == null ? "pending" : result.toString()) + + ", jobs=" + jobs.values() + + '}'; } public class Phase extends BasePhase { + public void accept() + { + accepted = true; + } + } + public static class Job extends AbstractState + { + public enum State { ACCEPT, SNAPSHOT, VALIDATION, SYNC } + + private final AtomicReference validation = new AtomicReference<>(null); + private final ConcurrentMap syncs = new ConcurrentHashMap<>(); + + public Job(Clock clock, RepairJobDesc desc) + { + super(clock, desc, State.class); + } + + @Override + protected synchronized UpdateType maybeUpdateState(State state) + { + return super.maybeUpdateState(state); + } + + public void snapshot() + { + updateState(State.SNAPSHOT); + } + + public RegisterStatus register(ValidationState state) + { + return register(s -> validation.compareAndSet(null, s) ? null : validation(), State.VALIDATION, state); + } + + @Nullable + public ValidationState validation() + { + return validation.get(); + } + + public RegisterStatus register(SyncState state) + { + return register(s -> syncs.putIfAbsent(s.id, s), State.SYNC, state); + } + + private > RegisterStatus register(Function putter, State state, S value) + { + UpdateType updateType = maybeUpdateState(state); + switch (updateType) + { + case ALREADY_COMPLETED: + return RegisterStatus.ALREADY_COMPLETED; + case LARGER_STATE_SEEN: + return RegisterStatus.STATUS_REJECTED; + case ACCEPTED: + case NO_CHANGE: + // allow + break; + default: + throw new IllegalStateException("Unknown status: " + updateType); + } + S current = putter.apply(value); + return current == null ? RegisterStatus.ACCEPTED : RegisterStatus.EXISTS; + } + + @Nullable + public SyncState sync(SyncState.Id id) + { + return syncs.get(id); + } + + @Override + public String toString() + { + return super.toString(); + } } } diff --git a/src/java/org/apache/cassandra/repair/state/SessionState.java b/src/java/org/apache/cassandra/repair/state/SessionState.java index 8e8e6485be..32be089350 100644 --- a/src/java/org/apache/cassandra/repair/state/SessionState.java +++ b/src/java/org/apache/cassandra/repair/state/SessionState.java @@ -22,9 +22,11 @@ import java.util.Set; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import java.util.stream.Collectors; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.repair.CommonRange; +import org.apache.cassandra.utils.Clock; import org.apache.cassandra.utils.TimeUUID; import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID; @@ -44,9 +46,9 @@ public class SessionState extends AbstractState public final Phase phase = new Phase(); - public SessionState(TimeUUID parentRepairSession, String keyspace, String[] cfnames, CommonRange commonRange) + public SessionState(Clock clock, TimeUUID parentRepairSession, String keyspace, String[] cfnames, CommonRange commonRange) { - super(nextTimeUUID(), State.class); + super(clock, nextTimeUUID(), State.class); this.parentRepairSession = parentRepairSession; this.keyspace = keyspace; this.cfnames = cfnames; @@ -73,6 +75,21 @@ public class SessionState extends AbstractState return commonRange.endpoints; } + @Override + public String status() + { + State state = getStatus(); + Result result = getResult(); + if (result != null) + return result.kind.name(); + else if (state == null) + return "init"; + else if (state == State.JOBS_START) + return state.name() + " " + jobs.entrySet().stream().map(e -> e.getKey() + " -> " + e.getValue().status()).collect(Collectors.toList()); + else + return state.name(); + } + public void register(JobState state) { jobs.put(state.id, state); diff --git a/src/java/org/apache/cassandra/repair/state/SyncState.java b/src/java/org/apache/cassandra/repair/state/SyncState.java new file mode 100644 index 0000000000..9c88aa1877 --- /dev/null +++ b/src/java/org/apache/cassandra/repair/state/SyncState.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.repair.state; + +import java.util.Objects; + +import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.repair.RepairJobDesc; +import org.apache.cassandra.utils.Clock; + +public class SyncState extends AbstractState +{ + public enum State + { ACCEPT, PLANNING, START } + + public final Phase phase = new Phase(); + + public SyncState(Clock clock, RepairJobDesc desc, InetAddressAndPort initiator, InetAddressAndPort src, InetAddressAndPort dst) + { + super(clock, new Id(desc, initiator, src, dst), State.class); + } + + public final class Phase extends BaseSkipPhase + { + public void accept() + { + updateState(State.ACCEPT); + } + + public void planning() + { + updateState(State.PLANNING); + } + + public void start() + { + updateState(State.START); + } + } + + public static class Id + { + public final RepairJobDesc desc; + public final InetAddressAndPort initiator, src, dst; + + public Id(RepairJobDesc desc, InetAddressAndPort initiator, InetAddressAndPort src, InetAddressAndPort dst) + { + this.desc = desc; + this.initiator = initiator; + this.src = src; + this.dst = dst; + } + + @Override + public boolean equals(Object o) + { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Id id = (Id) o; + return desc.equals(id.desc) && initiator.equals(id.initiator) && src.equals(id.src) && dst.equals(id.dst); + } + + @Override + public int hashCode() + { + return Objects.hash(desc, initiator, src, dst); + } + } +} diff --git a/src/java/org/apache/cassandra/repair/state/ValidationState.java b/src/java/org/apache/cassandra/repair/state/ValidationState.java index ff2ca537df..8dc63ec245 100644 --- a/src/java/org/apache/cassandra/repair/state/ValidationState.java +++ b/src/java/org/apache/cassandra/repair/state/ValidationState.java @@ -21,11 +21,12 @@ import java.util.UUID; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.repair.RepairJobDesc; +import org.apache.cassandra.utils.Clock; public class ValidationState extends AbstractState { public enum State - { START, SENDING_TREES } + { ACCEPT, START, SENDING_TREES } public final Phase phase = new Phase(); public final RepairJobDesc desc; @@ -35,9 +36,10 @@ public class ValidationState extends AbstractState public long partitionsProcessed; public long bytesRead; - public ValidationState(RepairJobDesc desc, InetAddressAndPort initiator) + public ValidationState(Clock clock, RepairJobDesc desc, InetAddressAndPort initiator) { - super(desc.determanisticId(), State.class); + // UUID is used to make the validations table easier for users to lookup by a single key rather than a composite key + super(clock, desc.determanisticId(), State.class); this.desc = desc; this.initiator = initiator; } @@ -56,6 +58,11 @@ public class ValidationState extends AbstractState public final class Phase extends BaseSkipPhase { + public void accept() + { + updateState(State.ACCEPT); + } + public void start(long estimatedPartitions, long estimatedTotalBytes) { updateState(State.START); diff --git a/src/java/org/apache/cassandra/schema/Tables.java b/src/java/org/apache/cassandra/schema/Tables.java index 33ed1b8025..0f8f6b3190 100644 --- a/src/java/org/apache/cassandra/schema/Tables.java +++ b/src/java/org/apache/cassandra/schema/Tables.java @@ -123,7 +123,7 @@ public final class Tables implements Iterable } @Nullable - TableMetadata getNullable(TableId id) + public TableMetadata getNullable(TableId id) { return tablesById.get(id); } diff --git a/src/java/org/apache/cassandra/service/ActiveRepairService.java b/src/java/org/apache/cassandra/service/ActiveRepairService.java index 738551d7c3..755301dcff 100644 --- a/src/java/org/apache/cassandra/service/ActiveRepairService.java +++ b/src/java/org/apache/cassandra/service/ActiveRepairService.java @@ -26,8 +26,8 @@ import java.util.concurrent.ScheduledFuture; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; -import java.util.concurrent.atomic.AtomicBoolean; import javax.management.openmbean.CompositeData; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Predicate; import java.util.stream.Collectors; @@ -44,7 +44,7 @@ import com.google.common.collect.Multimap; import org.apache.cassandra.concurrent.ExecutorPlus; import org.apache.cassandra.config.Config; import org.apache.cassandra.config.DurationSpec; -import org.apache.cassandra.db.compaction.CompactionManager; +import org.apache.cassandra.repair.SharedContext; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.locator.AbstractReplicationStrategy; import org.apache.cassandra.locator.EndpointsByRange; @@ -59,11 +59,10 @@ import org.apache.cassandra.schema.Schema; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.streaming.PreviewKind; import org.apache.cassandra.utils.TimeUUID; -import org.apache.cassandra.utils.concurrent.CountDownLatch; +import org.apache.cassandra.utils.concurrent.AsyncPromise; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.cassandra.concurrent.ScheduledExecutors; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.Keyspace; @@ -73,10 +72,8 @@ import org.apache.cassandra.exceptions.RequestFailureReason; import org.apache.cassandra.gms.ApplicationState; import org.apache.cassandra.gms.EndpointState; import org.apache.cassandra.gms.FailureDetector; -import org.apache.cassandra.gms.Gossiper; import org.apache.cassandra.gms.IEndpointStateChangeSubscriber; import org.apache.cassandra.gms.IFailureDetectionEventListener; -import org.apache.cassandra.gms.IFailureDetector; import org.apache.cassandra.gms.VersionedValue; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.locator.TokenMetadata; @@ -84,7 +81,6 @@ import org.apache.cassandra.metrics.RepairMetrics; import org.apache.cassandra.net.RequestCallback; import org.apache.cassandra.net.Verb; import org.apache.cassandra.net.Message; -import org.apache.cassandra.net.MessagingService; import org.apache.cassandra.repair.CommonRange; import org.apache.cassandra.repair.NoSuchRepairSessionException; import org.apache.cassandra.service.paxos.PaxosRepair; @@ -106,8 +102,6 @@ import org.apache.cassandra.repair.messages.RepairOption; import org.apache.cassandra.repair.messages.SyncResponse; import org.apache.cassandra.repair.messages.ValidationResponse; import org.apache.cassandra.schema.TableId; -import org.apache.cassandra.utils.FBUtilities; -import org.apache.cassandra.utils.MBeanWrapper; import org.apache.cassandra.utils.MerkleTrees; import org.apache.cassandra.utils.Pair; import org.apache.cassandra.utils.concurrent.Future; @@ -126,12 +120,9 @@ import static org.apache.cassandra.config.CassandraRelevantProperties.SKIP_PAXOS import static org.apache.cassandra.config.CassandraRelevantProperties.SKIP_PAXOS_REPAIR_ON_TOPOLOGY_CHANGE_KEYSPACES; import static org.apache.cassandra.config.Config.RepairCommandPoolFullStrategy.reject; import static org.apache.cassandra.config.DatabaseDescriptor.*; -import static org.apache.cassandra.net.Message.out; import static org.apache.cassandra.net.Verb.PREPARE_MSG; -import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis; +import static org.apache.cassandra.repair.messages.RepairMessage.notDone; import static org.apache.cassandra.utils.Simulate.With.MONITORS; -import static org.apache.cassandra.utils.Clock.Global.nanoTime; -import static org.apache.cassandra.utils.concurrent.CountDownLatch.newCountDownLatch; /** * ActiveRepairService is the starting point for manual "active" repairs. @@ -150,6 +141,7 @@ import static org.apache.cassandra.utils.concurrent.CountDownLatch.newCountDownL @Simulate(with = MONITORS) public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFailureDetectionEventListener, ActiveRepairServiceMBean { + public enum ParentRepairStatus { IN_PROGRESS, COMPLETED, FAILED @@ -157,21 +149,35 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai public static class ConsistentSessions { - public final LocalSessions local = new LocalSessions(); - public final CoordinatorSessions coordinated = new CoordinatorSessions(); + public final LocalSessions local; + public final CoordinatorSessions coordinated; + + public ConsistentSessions(SharedContext ctx) + { + local = new LocalSessions(ctx); + coordinated = new CoordinatorSessions(ctx); + } } - public final ConsistentSessions consistent = new ConsistentSessions(); + public final ConsistentSessions consistent; private boolean registeredForEndpointChanges = false; private static final Logger logger = LoggerFactory.getLogger(ActiveRepairService.class); - // singleton enforcement - public static final ActiveRepairService instance = new ActiveRepairService(FailureDetector.instance, Gossiper.instance); public static final long UNREPAIRED_SSTABLE = 0; public static final TimeUUID NO_PENDING_REPAIR = null; + public static ActiveRepairService instance() + { + return Holder.instance; + } + + private static class Holder + { + private static final ActiveRepairService instance = new ActiveRepairService(); + } + /** * A map of active coordinator session. */ @@ -182,6 +188,7 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai private final Cache repairs; // map of top level repair id (parent repair id) -> participate state private final Cache participates; + public final SharedContext ctx; private volatile ScheduledFuture irCleanup; @@ -213,18 +220,22 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai return RepairCommandExecutorHandle.repairCommandExecutor; } - private final IFailureDetector failureDetector; - private final Gossiper gossiper; private final Cache>> repairStatusByCmd; + public final ExecutorPlus snapshotExecutor; - public final ExecutorPlus snapshotExecutor = executorFactory().configurePooled("RepairSnapshotExecutor", 1) - .withKeepAlive(1, TimeUnit.HOURS) - .build(); - - public ActiveRepairService(IFailureDetector failureDetector, Gossiper gossiper) + public ActiveRepairService() { - this.failureDetector = failureDetector; - this.gossiper = gossiper; + this(SharedContext.Global.instance); + } + + @VisibleForTesting + public ActiveRepairService(SharedContext ctx) + { + this.ctx = ctx; + consistent = new ConsistentSessions(ctx); + this.snapshotExecutor = ctx.executorFactory().configurePooled("RepairSnapshotExecutor", 1) + .withKeepAlive(1, TimeUnit.HOURS) + .build(); this.repairStatusByCmd = CacheBuilder.newBuilder() .expireAfterWrite(PARENT_REPAIR_STATUS_EXPIRY_SECONDS.getLong(), TimeUnit.SECONDS) // using weight wouldn't work so well, since it doesn't reflect mutation of cached data @@ -245,15 +256,15 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai .maximumSize(numElements) .build(); - MBeanWrapper.instance.registerMBean(this, MBEAN_NAME); + ctx.mbean().registerMBean(this, MBEAN_NAME); } public void start() { consistent.local.start(); - this.irCleanup = ScheduledExecutors.optionalTasks.scheduleAtFixedRate(consistent.local::cleanup, 0, - LocalSessions.CLEANUP_INTERVAL, - TimeUnit.SECONDS); + this.irCleanup = ctx.optionalTasks().scheduleAtFixedRate(consistent.local::cleanup, 0, + LocalSessions.CLEANUP_INTERVAL, + TimeUnit.SECONDS); } @VisibleForTesting @@ -428,7 +439,7 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai if (cfnames.length == 0) return null; - final RepairSession session = new RepairSession(parentRepairSession, range, keyspace, + final RepairSession session = new RepairSession(ctx, parentRepairSession, range, keyspace, parallelismDegree, isIncremental, pullRepair, previewKind, optimiseStreams, repairPaxos, paxosOnly, cfnames); repairs.getIfPresent(parentRepairSession).register(session.state); @@ -463,8 +474,8 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai IEndpointStateChangeSubscriber & IFailureDetectionEventListener> void registerOnFdAndGossip(final T task) { - gossiper.register(task); - failureDetector.registerFailureDetectionEventListener(task); + ctx.gossiper().register(task); + ctx.failureDetector().registerFailureDetectionEventListener(task); // unregister listeners at completion task.addListener(new Runnable() @@ -474,8 +485,8 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai */ public void run() { - failureDetector.unregisterFailureDetectionEventListener(task); - gossiper.unregister(task); + ctx.failureDetector().unregisterFailureDetectionEventListener(task); + ctx.gossiper().unregister(task); } }); } @@ -511,9 +522,9 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai * @param dataCenters the data centers to involve in the repair * @return neighbors with whom we share the provided range */ - public static EndpointsForRange getNeighbors(String keyspaceName, Iterable> keyspaceLocalRanges, - Range toRepair, Collection dataCenters, - Collection hosts) + public EndpointsForRange getNeighbors(String keyspaceName, Iterable> keyspaceLocalRanges, + Range toRepair, Collection dataCenters, + Collection hosts) { StorageService ss = StorageService.instance; EndpointsByRange replicaSets = ss.getRangeToAddressMap(keyspaceName); @@ -536,7 +547,8 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai if (rangeSuperSet == null || !replicaSets.containsKey(rangeSuperSet)) return EndpointsForRange.empty(toRepair); - EndpointsForRange neighbors = replicaSets.get(rangeSuperSet).withoutSelf(); + // same as withoutSelf(), but done this way for testing + EndpointsForRange neighbors = replicaSets.get(rangeSuperSet).filter(r -> !ctx.broadcastAddressAndPort().equals(r.endpoint())); if (dataCenters != null && !dataCenters.isEmpty()) { @@ -553,7 +565,7 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai try { final InetAddressAndPort endpoint = InetAddressAndPort.getByName(host.trim()); - if (endpoint.equals(FBUtilities.getBroadcastAddressAndPort()) || neighbors.endpoints().contains(endpoint)) + if (endpoint.equals(ctx.broadcastAddressAndPort()) || neighbors.endpoints().contains(endpoint)) specifiedHost.add(endpoint); } catch (UnknownHostException e) @@ -562,7 +574,7 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai } } - if (!specifiedHost.contains(FBUtilities.getBroadcastAddressAndPort())) + if (!specifiedHost.contains(ctx.broadcastAddressAndPort())) throw new IllegalArgumentException("The current host must be part of the repair"); if (specifiedHost.size() <= 1) @@ -573,7 +585,7 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai throw new IllegalArgumentException(String.format(msg, hosts, toRepair, neighbors)); } - specifiedHost.remove(FBUtilities.getBroadcastAddressAndPort()); + specifiedHost.remove(ctx.broadcastAddressAndPort()); return neighbors.keep(specifiedHost); } @@ -585,14 +597,14 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai * incremental repairs, forced incremental repairs, and full repairs, the UNREPAIRED_SSTABLE value will prevent * sstables from being promoted to repaired or preserve the repairedAt/pendingRepair values, respectively. */ - static long getRepairedAt(RepairOption options, boolean force) + long getRepairedAt(RepairOption options, boolean force) { // we only want to set repairedAt for incremental repairs including all replicas for a token range. For non-global incremental repairs, full repairs, the UNREPAIRED_SSTABLE value will prevent // sstables from being promoted to repaired or preserve the repairedAt/pendingRepair values, respectively. For forced repairs, repairedAt time is only set to UNREPAIRED_SSTABLE if we actually // end up skipping replicas if (options.isIncremental() && options.isGlobal() && !force) { - return currentTimeMillis(); + return ctx.clock().currentTimeMillis(); } else { @@ -600,11 +612,11 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai } } - public static boolean verifyCompactionsPendingThreshold(TimeUUID parentRepairSession, PreviewKind previewKind) + public boolean verifyCompactionsPendingThreshold(TimeUUID parentRepairSession, PreviewKind previewKind) { // Snapshot values so failure message is consistent with decision - int pendingCompactions = CompactionManager.instance.getPendingTasks(); - int pendingThreshold = ActiveRepairService.instance.getRepairPendingCompactionRejectThreshold(); + int pendingCompactions = ctx.compactionManager().getPendingTasks(); + int pendingThreshold = getRepairPendingCompactionRejectThreshold(); if (pendingCompactions > pendingThreshold) { logger.error("[{}] Rejecting incoming repair, pending compactions ({}) above threshold ({})", @@ -614,54 +626,28 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai return true; } - public TimeUUID prepareForRepair(TimeUUID parentRepairSession, InetAddressAndPort coordinator, Set endpoints, RepairOption options, boolean isForcedRepair, List columnFamilyStores) + public Future prepareForRepair(TimeUUID parentRepairSession, InetAddressAndPort coordinator, Set endpoints, RepairOption options, boolean isForcedRepair, List columnFamilyStores) { if (!verifyCompactionsPendingThreshold(parentRepairSession, options.getPreviewKind())) failRepair(parentRepairSession, "Rejecting incoming repair, pending compactions above threshold"); // failRepair throws exception long repairedAt = getRepairedAt(options, isForcedRepair); registerParentRepairSession(parentRepairSession, coordinator, columnFamilyStores, options.getRanges(), options.isIncremental(), repairedAt, options.isGlobal(), options.getPreviewKind()); - final CountDownLatch prepareLatch = newCountDownLatch(endpoints.size()); - final AtomicBoolean status = new AtomicBoolean(true); - final Set failedNodes = synchronizedSet(new HashSet()); - final AtomicInteger timeouts = new AtomicInteger(0); - RequestCallback callback = new RequestCallback() - { - @Override - public void onResponse(Message msg) - { - prepareLatch.decrement(); - } - - @Override - public void onFailure(InetAddressAndPort from, RequestFailureReason failureReason) - { - status.set(false); - failedNodes.add(from.toString()); - if (failureReason == RequestFailureReason.TIMEOUT) - timeouts.incrementAndGet(); - prepareLatch.decrement(); - } - - @Override - public boolean invokeOnFailure() - { - return true; - } - }; + AtomicInteger pending = new AtomicInteger(endpoints.size()); + Set failedNodes = synchronizedSet(new HashSet<>()); + AsyncPromise promise = new AsyncPromise<>(); List tableIds = new ArrayList<>(columnFamilyStores.size()); for (ColumnFamilyStore cfs : columnFamilyStores) tableIds.add(cfs.metadata.id); PrepareMessage message = new PrepareMessage(parentRepairSession, tableIds, options.getRanges(), options.isIncremental(), repairedAt, options.isGlobal(), options.getPreviewKind()); - register(new ParticipateState(FBUtilities.getBroadcastAddressAndPort(), message)); + register(new ParticipateState(ctx.clock(), ctx.broadcastAddressAndPort(), message)); for (InetAddressAndPort neighbour : endpoints) { - if (FailureDetector.instance.isAlive(neighbour)) + if (ctx.failureDetector().isAlive(neighbour)) { - Message msg = out(PREPARE_MSG, message); - MessagingService.instance().sendWithCallback(msg, neighbour, callback); + sendPrepareWithRetries(parentRepairSession, pending, failedNodes, promise, neighbour, message); } else { @@ -669,7 +655,7 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai // remaining ones go down, we still want to fail so we don't create repair sessions that can't complete if (isForcedRepair && !options.isIncremental()) { - prepareLatch.decrement(); + pending.decrementAndGet(); } else { @@ -678,22 +664,66 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai } } } - try - { - if (!prepareLatch.await(getRpcTimeout(MILLISECONDS), MILLISECONDS) || timeouts.get() > 0) - failRepair(parentRepairSession, "Did not get replies from all endpoints."); - } - catch (InterruptedException e) - { - failRepair(parentRepairSession, "Interrupted while waiting for prepare repair response."); - } + // implement timeout to bound the runtime of the future + long timeoutMillis = getRepairRetrySpec().isEnabled() ? getRepairRpcTimeout(MILLISECONDS) + : getRpcTimeout(MILLISECONDS); + ctx.optionalTasks().schedule(() -> { + if (promise.isDone()) + return; + String errorMsg = "Did not get replies from all endpoints."; + if (promise.tryFailure(new RuntimeException(errorMsg))) + participateFailed(parentRepairSession, errorMsg); + }, timeoutMillis, MILLISECONDS); - if (!status.get()) - { - failRepair(parentRepairSession, "Got negative replies from endpoints " + failedNodes); - } + return promise; + } + + private void sendPrepareWithRetries(TimeUUID parentRepairSession, + AtomicInteger pending, + Set failedNodes, + AsyncPromise promise, + InetAddressAndPort to, + RepairMessage msg) + { + RepairMessage.sendMessageWithRetries(ctx, notDone(promise), msg, PREPARE_MSG, to, new RequestCallback<>() + { + @Override + public void onResponse(Message msg) + { + ack(); + } + + @Override + public void onFailure(InetAddressAndPort from, RequestFailureReason failureReason) + { + failedNodes.add(from.toString()); + if (failureReason == RequestFailureReason.TIMEOUT) + { + pending.set(-1); + promise.setFailure(failRepairException(parentRepairSession, "Did not get replies from all endpoints.")); + } + else + { + ack(); + } + } + + private void ack() + { + if (pending.decrementAndGet() == 0) + { + if (failedNodes.isEmpty()) + { + promise.setSuccess(null); + } + else + { + promise.setFailure(failRepairException(parentRepairSession, "Got negative replies from endpoints " + failedNodes)); + } + } + } + }); - return parentRepairSession; } /** @@ -707,10 +737,9 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai { try { - if (FailureDetector.instance.isAlive(endpoint)) + if (ctx.failureDetector().isAlive(endpoint)) { CleanupMessage message = new CleanupMessage(parentRepairSession); - Message msg = Message.out(Verb.CLEANUP_MSG, message); RequestCallback loggingCallback = new RequestCallback() { @@ -728,8 +757,7 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai "of messages like this.", parentRepairSession, endpoint); } }; - - MessagingService.instance().sendWithCallback(msg, endpoint, loggingCallback); + RepairMessage.sendMessageWithRetries(ctx, message, Verb.CLEANUP_MSG, endpoint, loggingCallback); } } catch (Exception exc) @@ -743,12 +771,22 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai } private void failRepair(TimeUUID parentRepairSession, String errorMsg) + { + throw failRepairException(parentRepairSession, errorMsg); + } + + private RuntimeException failRepairException(TimeUUID parentRepairSession, String errorMsg) + { + participateFailed(parentRepairSession, errorMsg); + removeParentRepairSession(parentRepairSession); + return new RuntimeException(errorMsg); + } + + private void participateFailed(TimeUUID parentRepairSession, String errorMsg) { ParticipateState state = participate(parentRepairSession); if (state != null) state.phase.fail(errorMsg); - removeParentRepairSession(parentRepairSession); - throw new RuntimeException(errorMsg); } public synchronized void registerParentRepairSession(TimeUUID parentRepairSession, InetAddressAndPort coordinator, List columnFamilyStores, Collection> ranges, boolean isIncremental, long repairedAt, boolean isGlobal, PreviewKind previewKind) @@ -756,8 +794,8 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai assert isIncremental || repairedAt == ActiveRepairService.UNREPAIRED_SSTABLE; if (!registeredForEndpointChanges) { - Gossiper.instance.register(this); - FailureDetector.instance.registerFailureDetectionEventListener(this); + ctx.gossiper().register(this); + ctx.failureDetector().registerFailureDetectionEventListener(this); registeredForEndpointChanges = true; } @@ -796,25 +834,25 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai */ public synchronized ParentRepairSession removeParentRepairSession(TimeUUID parentSessionId) { - String snapshotName = parentSessionId.toString(); ParentRepairSession session = parentRepairSessions.remove(parentSessionId); if (session == null) return null; - if (session.hasSnapshots) + String snapshotName = parentSessionId.toString(); + if (session.hasSnapshots.get()) { snapshotExecutor.submit(() -> { logger.info("[repair #{}] Clearing snapshots for {}", parentSessionId, session.columnFamilyStores.values() .stream() .map(cfs -> cfs.metadata().toString()).collect(Collectors.joining(", "))); - long startNanos = nanoTime(); + long startNanos = ctx.clock().nanoTime(); for (ColumnFamilyStore cfs : session.columnFamilyStores.values()) { if (cfs.snapshotExists(snapshotName)) cfs.clearSnapshot(snapshotName); } - logger.info("[repair #{}] Cleared snapshots in {}ms", parentSessionId, TimeUnit.NANOSECONDS.toMillis(nanoTime() - startNanos)); + logger.info("[repair #{}] Cleared snapshots in {}ms", parentSessionId, TimeUnit.NANOSECONDS.toMillis(ctx.clock().nanoTime() - startNanos)); }); } return session; @@ -828,6 +866,13 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai if (session == null) { + switch (message.verb()) + { + case VALIDATION_RSP: + case SYNC_RSP: + ctx.messaging().send(message.emptyResponse(), message.from()); + break; + } if (payload instanceof ValidationResponse) { // The trees may be off-heap, and will therefore need to be released. @@ -845,13 +890,10 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai switch (message.verb()) { case VALIDATION_RSP: - ValidationResponse validation = (ValidationResponse) payload; - session.validationComplete(desc, message.from(), validation.trees); + session.validationComplete(desc, (Message) message); break; case SYNC_RSP: - // one of replica is synced. - SyncResponse sync = (SyncResponse) payload; - session.syncComplete(desc, sync.nodes, sync.success, sync.summaries); + session.syncComplete(desc, (Message) message); break; default: break; @@ -873,7 +915,7 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai public final long repairedAt; public final InetAddressAndPort coordinator; public final PreviewKind previewKind; - public volatile boolean hasSnapshots = false; + public final AtomicBoolean hasSnapshots = new AtomicBoolean(false); public ParentRepairSession(InetAddressAndPort coordinator, List columnFamilyStores, Collection> ranges, boolean isIncremental, long repairedAt, boolean isGlobal, PreviewKind previewKind) { @@ -930,9 +972,9 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai '}'; } - public void setHasSnapshots() + public boolean setHasSnapshots() { - hasSnapshots = true; + return hasSnapshots.compareAndSet(false, true); } } diff --git a/src/java/org/apache/cassandra/service/CassandraDaemon.java b/src/java/org/apache/cassandra/service/CassandraDaemon.java index 2407dea8d0..332f2a714f 100644 --- a/src/java/org/apache/cassandra/service/CassandraDaemon.java +++ b/src/java/org/apache/cassandra/service/CassandraDaemon.java @@ -367,7 +367,7 @@ public class CassandraDaemon if (sizeRecorderInterval > 0) ScheduledExecutors.optionalTasks.scheduleWithFixedDelay(SizeEstimatesRecorder.instance, 30, sizeRecorderInterval, TimeUnit.SECONDS); - ActiveRepairService.instance.start(); + ActiveRepairService.instance().start(); StreamManager.instance.start(); // Prepared statements diff --git a/src/java/org/apache/cassandra/service/StorageService.java b/src/java/org/apache/cassandra/service/StorageService.java index c119f2d82b..1a79cdeeed 100644 --- a/src/java/org/apache/cassandra/service/StorageService.java +++ b/src/java/org/apache/cassandra/service/StorageService.java @@ -179,7 +179,7 @@ import org.apache.cassandra.metrics.StorageMetrics; import org.apache.cassandra.net.AsyncOneResponse; import org.apache.cassandra.net.Message; import org.apache.cassandra.net.MessagingService; -import org.apache.cassandra.repair.RepairRunnable; +import org.apache.cassandra.repair.RepairCoordinator; import org.apache.cassandra.repair.messages.RepairOption; import org.apache.cassandra.schema.CompactionParams.TombstoneOption; import org.apache.cassandra.schema.KeyspaceMetadata; @@ -361,8 +361,12 @@ public class StorageService extends NotificationBroadcasterSupport implements IE public RangesAtEndpoint getLocalReplicas(String keyspaceName) { - return Keyspace.open(keyspaceName).getReplicationStrategy() - .getAddressReplicas(FBUtilities.getBroadcastAddressAndPort()); + return getReplicas(keyspaceName, FBUtilities.getBroadcastAddressAndPort()); + } + + public RangesAtEndpoint getReplicas(String keyspaceName, InetAddressAndPort endpoint) + { + return Keyspace.open(keyspaceName).getReplicationStrategy().getAddressReplicas(endpoint); } public List> getLocalRanges(String ks) @@ -4593,7 +4597,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE * @param cfNames CFs * @throws java.lang.IllegalArgumentException when given CF name does not exist */ - public Iterable getValidColumnFamilies(boolean allowIndexes, boolean autoAddIndexes, String keyspaceName, String... cfNames) throws IOException + public Iterable getValidColumnFamilies(boolean allowIndexes, boolean autoAddIndexes, String keyspaceName, String... cfNames) { Keyspace keyspace = getValidKeyspace(keyspaceName); return keyspace.getValidColumnFamilies(allowIndexes, autoAddIndexes, cfNames); @@ -4714,7 +4718,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE { if (!options.getDataCenters().isEmpty() && !options.getDataCenters().contains(DatabaseDescriptor.getLocalDataCenter())) { - throw new IllegalArgumentException("the local data center must be part of the repair"); + throw new IllegalArgumentException("the local data center must be part of the repair; requested " + options.getDataCenters() + " but DC is " + DatabaseDescriptor.getLocalDataCenter()); } Set existingDatacenters = tokenMetadata.cloneOnlyTokenMap().getTopology().getDatacenterEndpoints().keys().elementSet(); List datacenters = new ArrayList<>(options.getDataCenters()); @@ -4724,7 +4728,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE throw new IllegalArgumentException("data center(s) " + datacenters.toString() + " not found"); } - RepairRunnable task = new RepairRunnable(this, cmd, options, keyspace); + RepairCoordinator task = new RepairCoordinator(this, cmd, options, keyspace); task.addProgressListener(progressSupport); for (ProgressListener listener : listeners) task.addProgressListener(listener); @@ -4807,7 +4811,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE continue; List> ranges = getLocalAndPendingRanges(ksName); - futures.add(ActiveRepairService.instance.repairPaxosForTopologyChange(ksName, ranges, reason)); + futures.add(ActiveRepairService.instance().repairPaxosForTopologyChange(ksName, ranges, reason)); } return FutureCombiner.allOf(futures); @@ -4827,13 +4831,13 @@ public class StorageService extends NotificationBroadcasterSupport implements IE public void forceTerminateAllRepairSessions() { - ActiveRepairService.instance.terminateSessions(); + ActiveRepairService.instance().terminateSessions(); } @Nullable public List getParentRepairStatus(int cmd) { - Pair> pair = ActiveRepairService.instance.getRepairStatus(cmd); + Pair> pair = ActiveRepairService.instance().getRepairStatus(cmd); return pair == null ? null : ImmutableList.builder().add(pair.left.name()).addAll(pair.right).build(); } @@ -5729,7 +5733,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE shutdownClientServers(); ScheduledExecutors.optionalTasks.shutdown(); Gossiper.instance.stop(); - ActiveRepairService.instance.stop(); + ActiveRepairService.instance().stop(); if (!isFinalShutdown) setMode(Mode.DRAINING, "shutting down MessageService", false); diff --git a/src/java/org/apache/cassandra/service/StorageServiceMBean.java b/src/java/org/apache/cassandra/service/StorageServiceMBean.java index 0ed8e655cf..dc76845cfc 100644 --- a/src/java/org/apache/cassandra/service/StorageServiceMBean.java +++ b/src/java/org/apache/cassandra/service/StorageServiceMBean.java @@ -477,7 +477,7 @@ public interface StorageServiceMBean extends NotificationEmitter /** * Get the status of a given parent repair session. * @param cmd the int reference returned when issuing the repair - * @return status of parent repair from enum {@link org.apache.cassandra.repair.RepairRunnable.Status} + * @return status of parent repair from enum {@link ActiveRepairService.ParentRepairStatus} * followed by final message or messages of the session */ @Nullable diff --git a/src/java/org/apache/cassandra/streaming/PreviewKind.java b/src/java/org/apache/cassandra/streaming/PreviewKind.java index 8d7ad8ab1b..23b010e971 100644 --- a/src/java/org/apache/cassandra/streaming/PreviewKind.java +++ b/src/java/org/apache/cassandra/streaming/PreviewKind.java @@ -86,7 +86,7 @@ public enum PreviewKind StatsMetadata sstableMetadata = sstable.getSSTableMetadata(); if (sstableMetadata.pendingRepair != null) { - LocalSession session = ActiveRepairService.instance.consistent.local.getSession(sstableMetadata.pendingRepair); + LocalSession session = ActiveRepairService.instance().consistent.local.getSession(sstableMetadata.pendingRepair); if (session == null) return false; else if (session.getState() == ConsistentSession.State.FINALIZED) diff --git a/src/java/org/apache/cassandra/streaming/StreamPlan.java b/src/java/org/apache/cassandra/streaming/StreamPlan.java index 9e79a5db4b..47fa9e1463 100644 --- a/src/java/org/apache/cassandra/streaming/StreamPlan.java +++ b/src/java/org/apache/cassandra/streaming/StreamPlan.java @@ -151,6 +151,22 @@ public class StreamPlan return this; } + public TimeUUID planId() + { + return planId; + } + + public StreamOperation streamOperation() + { + return streamOperation; + } + + @VisibleForTesting + public List handlers() + { + return handlers; + } + /** * Set custom StreamConnectionFactory to be used for establishing connection * diff --git a/src/java/org/apache/cassandra/streaming/StreamSession.java b/src/java/org/apache/cassandra/streaming/StreamSession.java index c63e69cd23..d76f1f6d25 100644 --- a/src/java/org/apache/cassandra/streaming/StreamSession.java +++ b/src/java/org/apache/cassandra/streaming/StreamSession.java @@ -542,7 +542,9 @@ public class StreamSession this.failureReason = failureReason; sink.onClose(peer); - streamResult.handleSessionComplete(this); + // closed before init? + if (streamResult != null) + streamResult.handleSessionComplete(this); }}).flatMap(ignore -> { List> futures = new ArrayList<>(); // ensure aborting the tasks do not happen on the network IO thread (read: netty event loop) @@ -959,7 +961,7 @@ public class StreamSession } } Collections.sort(tables); - int pendingThreshold = ActiveRepairService.instance.getRepairPendingCompactionRejectThreshold(); + int pendingThreshold = ActiveRepairService.instance().getRepairPendingCompactionRejectThreshold(); if (pendingCompactionsAfterStreaming > pendingThreshold) { logger.error("[Stream #{}] Rejecting incoming files based on pending compactions calculation " + diff --git a/src/java/org/apache/cassandra/tools/nodetool/Repair.java b/src/java/org/apache/cassandra/tools/nodetool/Repair.java index 8b077628d8..3583240830 100644 --- a/src/java/org/apache/cassandra/tools/nodetool/Repair.java +++ b/src/java/org/apache/cassandra/tools/nodetool/Repair.java @@ -20,6 +20,7 @@ package org.apache.cassandra.tools.nodetool; import static com.google.common.collect.Lists.newArrayList; import static org.apache.commons.lang3.StringUtils.EMPTY; import io.airlift.airline.Arguments; +import io.airlift.airline.Cli; import io.airlift.airline.Command; import io.airlift.airline.Option; @@ -28,6 +29,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.function.Supplier; import com.google.common.collect.Sets; @@ -141,39 +143,7 @@ public class Repair extends NodeToolCmd if ((args == null || args.isEmpty()) && ONLY_EXPLICITLY_REPAIRED.contains(keyspace)) continue; - Map options = new HashMap<>(); - RepairParallelism parallelismDegree = RepairParallelism.PARALLEL; - if (sequential) - parallelismDegree = RepairParallelism.SEQUENTIAL; - else if (dcParallel) - parallelismDegree = RepairParallelism.DATACENTER_AWARE; - options.put(RepairOption.PARALLELISM_KEY, parallelismDegree.getName()); - options.put(RepairOption.PRIMARY_RANGE_KEY, Boolean.toString(primaryRange)); - options.put(RepairOption.INCREMENTAL_KEY, Boolean.toString(!fullRepair && !(paxosOnly && getPreviewKind() == PreviewKind.NONE))); - options.put(RepairOption.JOB_THREADS_KEY, Integer.toString(numJobThreads)); - options.put(RepairOption.TRACE_KEY, Boolean.toString(trace)); - options.put(RepairOption.COLUMNFAMILIES_KEY, StringUtils.join(cfnames, ",")); - options.put(RepairOption.PULL_REPAIR_KEY, Boolean.toString(pullRepair)); - options.put(RepairOption.FORCE_REPAIR_KEY, Boolean.toString(force)); - options.put(RepairOption.PREVIEW, getPreviewKind().toString()); - options.put(RepairOption.OPTIMISE_STREAMS_KEY, Boolean.toString(optimiseStreams)); - options.put(RepairOption.IGNORE_UNREPLICATED_KS, Boolean.toString(ignoreUnreplicatedKeyspaces)); - options.put(RepairOption.REPAIR_PAXOS_KEY, Boolean.toString(!skipPaxos && getPreviewKind() == PreviewKind.NONE)); - options.put(RepairOption.PAXOS_ONLY_KEY, Boolean.toString(paxosOnly && getPreviewKind() == PreviewKind.NONE)); - - if (!startToken.isEmpty() || !endToken.isEmpty()) - { - options.put(RepairOption.RANGES_KEY, startToken + ":" + endToken); - } - if (localDC) - { - options.put(RepairOption.DATACENTERS_KEY, StringUtils.join(newArrayList(probe.getDataCenter()), ",")); - } - else - { - options.put(RepairOption.DATACENTERS_KEY, StringUtils.join(specificDataCenters, ",")); - } - options.put(RepairOption.HOSTS_KEY, StringUtils.join(specificHosts, ",")); + Map options = createOptions(probe::getDataCenter, cfnames); try { probe.repairAsync(probe.output().out, keyspace, options); @@ -183,4 +153,53 @@ public class Repair extends NodeToolCmd } } } + + public static Map parseOptionMap(Supplier localDCOption, List args) + { + List realArgs = new ArrayList<>(args.size() + 1); + realArgs.add("repair"); + realArgs.addAll(args); + Cli parser = Cli.builder("fortesting").withCommand(Repair.class).build(); + Repair repair = (Repair) parser.parse(realArgs); + String[] cfnames = repair.parseOptionalTables(repair.args); + return repair.createOptions(localDCOption, cfnames); + } + + private Map createOptions(Supplier localDCOption, String[] cfnames) + { + Map options = new HashMap<>(); + RepairParallelism parallelismDegree = RepairParallelism.PARALLEL; + if (sequential) + parallelismDegree = RepairParallelism.SEQUENTIAL; + else if (dcParallel) + parallelismDegree = RepairParallelism.DATACENTER_AWARE; + options.put(RepairOption.PARALLELISM_KEY, parallelismDegree.getName()); + options.put(RepairOption.PRIMARY_RANGE_KEY, Boolean.toString(primaryRange)); + options.put(RepairOption.INCREMENTAL_KEY, Boolean.toString(!fullRepair && !(paxosOnly && getPreviewKind() == PreviewKind.NONE))); + options.put(RepairOption.JOB_THREADS_KEY, Integer.toString(numJobThreads)); + options.put(RepairOption.TRACE_KEY, Boolean.toString(trace)); + options.put(RepairOption.COLUMNFAMILIES_KEY, StringUtils.join(cfnames, ",")); + options.put(RepairOption.PULL_REPAIR_KEY, Boolean.toString(pullRepair)); + options.put(RepairOption.FORCE_REPAIR_KEY, Boolean.toString(force)); + options.put(RepairOption.PREVIEW, getPreviewKind().toString()); + options.put(RepairOption.OPTIMISE_STREAMS_KEY, Boolean.toString(optimiseStreams)); + options.put(RepairOption.IGNORE_UNREPLICATED_KS, Boolean.toString(ignoreUnreplicatedKeyspaces)); + options.put(RepairOption.REPAIR_PAXOS_KEY, Boolean.toString(!skipPaxos && getPreviewKind() == PreviewKind.NONE)); + options.put(RepairOption.PAXOS_ONLY_KEY, Boolean.toString(paxosOnly && getPreviewKind() == PreviewKind.NONE)); + + if (!startToken.isEmpty() || !endToken.isEmpty()) + { + options.put(RepairOption.RANGES_KEY, startToken + ":" + endToken); + } + if (localDC) + { + options.put(RepairOption.DATACENTERS_KEY, StringUtils.join(newArrayList(localDCOption.get()), ",")); + } + else + { + options.put(RepairOption.DATACENTERS_KEY, StringUtils.join(specificDataCenters, ",")); + } + options.put(RepairOption.HOSTS_KEY, StringUtils.join(specificHosts, ",")); + return options; + } } diff --git a/src/java/org/apache/cassandra/utils/Backoff.java b/src/java/org/apache/cassandra/utils/Backoff.java new file mode 100644 index 0000000000..2f0b7e2c9c --- /dev/null +++ b/src/java/org/apache/cassandra/utils/Backoff.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.utils; + +import java.util.concurrent.TimeUnit; +import java.util.function.DoubleSupplier; + +public interface Backoff +{ + /** + * @return max attempts allowed, {@code == 0} implies no retries are allowed + */ + int maxAttempts(); + long computeWaitTime(int retryCount); + TimeUnit unit(); + + enum None implements Backoff + { + INSTANCE; + + @Override + public int maxAttempts() + { + return 0; + } + + @Override + public long computeWaitTime(int retryCount) + { + throw new UnsupportedOperationException(); + } + + @Override + public TimeUnit unit() + { + throw new UnsupportedOperationException(); + } + } + + class ExponentialBackoff implements Backoff + { + private final int maxAttempts; + private final long baseSleepTimeMillis; + private final long maxSleepMillis; + private final DoubleSupplier randomSource; + + public ExponentialBackoff(int maxAttempts, long baseSleepTimeMillis, long maxSleepMillis, DoubleSupplier randomSource) + { + this.maxAttempts = maxAttempts; + this.baseSleepTimeMillis = baseSleepTimeMillis; + this.maxSleepMillis = maxSleepMillis; + this.randomSource = randomSource; + } + + @Override + public int maxAttempts() + { + return maxAttempts; + } + + @Override + public long computeWaitTime(int retryCount) + { + long baseTimeMillis = baseSleepTimeMillis * (1L << retryCount); + // it's possible that this overflows, so fall back to max; + if (baseTimeMillis <= 0) + baseTimeMillis = maxSleepMillis; + // now make sure this is capped to target max + baseTimeMillis = Math.min(baseTimeMillis, maxSleepMillis); + + return (long) (baseTimeMillis * (randomSource.getAsDouble() + 0.5)); + } + + @Override + public TimeUnit unit() + { + return TimeUnit.MILLISECONDS; + } + } +} diff --git a/src/java/org/apache/cassandra/utils/Clock.java b/src/java/org/apache/cassandra/utils/Clock.java index 7fffefb587..c8ba785cab 100644 --- a/src/java/org/apache/cassandra/utils/Clock.java +++ b/src/java/org/apache/cassandra/utils/Clock.java @@ -87,6 +87,11 @@ public interface Clock INITIALIZE_MESSAGE = null; } + public static Clock clock() + { + return instance; + } + /** * Semantically equivalent to {@link System#nanoTime()} */ @@ -133,6 +138,11 @@ public interface Clock */ public long currentTimeMillis(); + public default long nowInSeconds() + { + return currentTimeMillis() / 1000L; + } + @Intercept public static void waitUntil(long deadlineNanos) throws InterruptedException { diff --git a/src/java/org/apache/cassandra/utils/FailingBiConsumer.java b/src/java/org/apache/cassandra/utils/FailingBiConsumer.java new file mode 100644 index 0000000000..d7dcbfb294 --- /dev/null +++ b/src/java/org/apache/cassandra/utils/FailingBiConsumer.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.utils; + +import java.util.function.BiConsumer; + +public interface FailingBiConsumer extends BiConsumer +{ + void acceptOrFail(A a, B b) throws Throwable; + + @Override + default void accept(A a, B b) + { + try + { + acceptOrFail(a, b); + } + catch (Throwable e) + { + throw Throwables.throwAsUncheckedException(e); + } + } + + static FailingBiConsumer orFail(FailingBiConsumer fn) + { + return fn; + } +} diff --git a/src/java/org/apache/cassandra/utils/MerkleTree.java b/src/java/org/apache/cassandra/utils/MerkleTree.java index 0cdb74e742..d84eedfced 100644 --- a/src/java/org/apache/cassandra/utils/MerkleTree.java +++ b/src/java/org/apache/cassandra/utils/MerkleTree.java @@ -1015,7 +1015,7 @@ public class MerkleTree default void serialize(DataOutputPlus out, int version) throws IOException { byte[] hash = hash(); - assert hash.length == HASH_SIZE; + assert hash.length == HASH_SIZE: String.format("Expected hash length to be %d, but given %d", HASH_SIZE, hash.length); out.writeByte(Leaf.IDENT); diff --git a/test/distributed/org/apache/cassandra/distributed/impl/Instance.java b/test/distributed/org/apache/cassandra/distributed/impl/Instance.java index a6d838e3fe..7c09d90aa7 100644 --- a/test/distributed/org/apache/cassandra/distributed/impl/Instance.java +++ b/test/distributed/org/apache/cassandra/distributed/impl/Instance.java @@ -747,7 +747,7 @@ public class Instance extends IsolatedExecutor implements IInvokableInstance FBUtilities.getBroadcastAddressAndPort().getPort() != broadcastAddress().getPort()) throw new IllegalStateException(String.format("%s != %s", FBUtilities.getBroadcastAddressAndPort(), broadcastAddress())); - ActiveRepairService.instance.start(); + ActiveRepairService.instance().start(); StreamManager.instance.start(); PaxosState.startAutoRepairs(); @@ -898,7 +898,7 @@ public class Instance extends IsolatedExecutor implements IInvokableInstance () -> DiagnosticSnapshotService.instance.shutdownAndWait(1L, MINUTES), () -> SSTableReader.shutdownBlocking(1L, MINUTES), () -> shutdownAndWait(Collections.singletonList(ActiveRepairService.repairCommandExecutor())), - () -> ActiveRepairService.instance.shutdownNowAndWait(1L, MINUTES), + () -> ActiveRepairService.instance().shutdownNowAndWait(1L, MINUTES), () -> SnapshotManager.shutdownAndWait(1L, MINUTES) ); diff --git a/test/distributed/org/apache/cassandra/distributed/mock/nodetool/InternalNodeProbe.java b/test/distributed/org/apache/cassandra/distributed/mock/nodetool/InternalNodeProbe.java index 381bb09ec4..4de180636c 100644 --- a/test/distributed/org/apache/cassandra/distributed/mock/nodetool/InternalNodeProbe.java +++ b/test/distributed/org/apache/cassandra/distributed/mock/nodetool/InternalNodeProbe.java @@ -79,7 +79,7 @@ public class InternalNodeProbe extends NodeProbe gcProxy = new GCInspector(); gossProxy = Gossiper.instance; bmProxy = BatchlogManager.instance; - arsProxy = ActiveRepairService.instance; + arsProxy = ActiveRepairService.instance(); memProxy = ManagementFactory.getMemoryMXBean(); runtimeProxy = ManagementFactory.getRuntimeMXBean(); } diff --git a/test/distributed/org/apache/cassandra/distributed/test/ClearSnapshotTest.java b/test/distributed/org/apache/cassandra/distributed/test/ClearSnapshotTest.java index 3a1ba4edd5..384e79d468 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/ClearSnapshotTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/ClearSnapshotTest.java @@ -100,7 +100,7 @@ public class ClearSnapshotTest extends TestBaseImpl long activeRepairs; do { - activeRepairs = cluster.get(1).callOnInstance(() -> ActiveRepairService.instance.parentRepairSessionCount()); + activeRepairs = cluster.get(1).callOnInstance(() -> ActiveRepairService.instance().parentRepairSessionCount()); Thread.sleep(50); } while (activeRepairs < 35); diff --git a/test/distributed/org/apache/cassandra/distributed/test/IncRepairAdminTest.java b/test/distributed/org/apache/cassandra/distributed/test/IncRepairAdminTest.java index b902632674..f376c4b4d2 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/IncRepairAdminTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/IncRepairAdminTest.java @@ -149,14 +149,14 @@ public class IncRepairAdminTest extends TestBaseImpl ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore("tbl"); Range range = new Range<>(cfs.metadata().partitioner.getMinimumToken(), cfs.metadata().partitioner.getRandomToken()); - ActiveRepairService.instance.registerParentRepairSession(sessionId, - InetAddressAndPort.getByAddress(coordinator.getAddress()), - Lists.newArrayList(cfs), - Sets.newHashSet(range), - true, - currentTimeMillis(), - true, - PreviewKind.NONE); + ActiveRepairService.instance().registerParentRepairSession(sessionId, + InetAddressAndPort.getByAddress(coordinator.getAddress()), + Lists.newArrayList(cfs), + Sets.newHashSet(range), + true, + currentTimeMillis(), + true, + PreviewKind.NONE); LocalSessionAccessor.prepareUnsafe(sessionId, InetAddressAndPort.getByAddress(coordinator.getAddress()), participants.stream().map(participant -> InetAddressAndPort.getByAddress(participant.getAddress())).collect(Collectors.toSet())); diff --git a/test/distributed/org/apache/cassandra/distributed/test/IncRepairCoordinatorErrorTest.java b/test/distributed/org/apache/cassandra/distributed/test/IncRepairCoordinatorErrorTest.java index 447cd822e2..06ac200be5 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/IncRepairCoordinatorErrorTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/IncRepairCoordinatorErrorTest.java @@ -48,7 +48,7 @@ public class IncRepairCoordinatorErrorTest extends TestBaseImpl cluster.get(1).nodetoolResult("repair", KEYSPACE).asserts().success(); TimeUUID result = (TimeUUID) cluster.get(1).executeInternal("select parent_id from system_distributed.repair_history")[0][0]; cluster.get(3).runOnInstance(() -> { - ActiveRepairService.instance.failSession(result.toString(), true); + ActiveRepairService.instance().failSession(result.toString(), true); }); assertThat(cluster.get(1).logs().watchFor("Can't transition endpoints .* to FAILED").getResult()).isNotEmpty(); } diff --git a/test/distributed/org/apache/cassandra/distributed/test/OptimiseStreamsRepairTest.java b/test/distributed/org/apache/cassandra/distributed/test/OptimiseStreamsRepairTest.java index 14d49dc0ca..1febbdbff7 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/OptimiseStreamsRepairTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/OptimiseStreamsRepairTest.java @@ -39,6 +39,7 @@ import net.bytebuddy.ByteBuddy; import net.bytebuddy.dynamic.loading.ClassLoadingStrategy; import net.bytebuddy.implementation.MethodDelegation; import net.bytebuddy.implementation.bind.annotation.SuperCall; +import org.apache.cassandra.repair.SharedContext; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; import org.apache.cassandra.distributed.Cluster; @@ -114,7 +115,8 @@ public class OptimiseStreamsRepairTest extends TestBaseImpl .load(cl, ClassLoadingStrategy.Default.INJECTION); } - public static List createOptimisedSyncingSyncTasks(RepairJobDesc desc, + public static List createOptimisedSyncingSyncTasks(SharedContext ctx, + RepairJobDesc desc, List trees, InetAddressAndPort local, Predicate isTransient, diff --git a/test/distributed/org/apache/cassandra/distributed/test/PaxosRepair2Test.java b/test/distributed/org/apache/cassandra/distributed/test/PaxosRepair2Test.java index 47932b3c73..a150a33a95 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/PaxosRepair2Test.java +++ b/test/distributed/org/apache/cassandra/distributed/test/PaxosRepair2Test.java @@ -161,7 +161,7 @@ public class PaxosRepair2Test extends TestBaseImpl { throw new AssertionError(e); } - Pair> status = ActiveRepairService.instance.getRepairStatus(cmd); + Pair> status = ActiveRepairService.instance().getRepairStatus(cmd); if (status == null) continue; diff --git a/test/distributed/org/apache/cassandra/distributed/test/PaxosRepairTest.java b/test/distributed/org/apache/cassandra/distributed/test/PaxosRepairTest.java index 3251f84ee5..dd1aa07e20 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/PaxosRepairTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/PaxosRepairTest.java @@ -175,7 +175,7 @@ public class PaxosRepairTest extends TestBaseImpl { throw new AssertionError(e); } - Pair> status = ActiveRepairService.instance.getRepairStatus(cmd); + Pair> status = ActiveRepairService.instance().getRepairStatus(cmd); if (status == null) continue; diff --git a/test/distributed/org/apache/cassandra/distributed/test/RepairErrorsTest.java b/test/distributed/org/apache/cassandra/distributed/test/RepairErrorsTest.java index f34bb2a79f..537599c297 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/RepairErrorsTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/RepairErrorsTest.java @@ -201,7 +201,7 @@ public class RepairErrorsTest extends TestBaseImpl private void assertNoActiveRepairSessions(IInvokableInstance instance) { // Make sure we've cleaned up local sessions: - Integer sessions = instance.callOnInstance(() -> ActiveRepairService.instance.sessionCount()); + Integer sessions = instance.callOnInstance(() -> ActiveRepairService.instance().sessionCount()); assertEquals(0, sessions.intValue()); } diff --git a/test/unit/accord/utils/DefaultRandom.java b/test/unit/accord/utils/DefaultRandom.java new file mode 100644 index 0000000000..b16f1f8bbf --- /dev/null +++ b/test/unit/accord/utils/DefaultRandom.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package accord.utils; + +import java.util.Random; + +public class DefaultRandom implements RandomSource +{ + private final Random delegate; + public DefaultRandom() + { + this.delegate = new Random(); + } + + public DefaultRandom(long seed) + { + this.delegate = new Random(seed); + } + + @Override + public void nextBytes(byte[] bytes) + { + delegate.nextBytes(bytes); + } + + @Override + public boolean nextBoolean() + { + return delegate.nextBoolean(); + } + + @Override + public int nextInt() + { + return delegate.nextInt(); + } + + @Override + public long nextLong() + { + return delegate.nextLong(); + } + + @Override + public float nextFloat() + { + return delegate.nextFloat(); + } + + @Override + public double nextDouble() + { + return delegate.nextDouble(); + } + + @Override + public double nextGaussian() + { + return delegate.nextGaussian(); + } + + @Override + public void setSeed(long seed) + { + delegate.setSeed(seed); + } + + @Override + public DefaultRandom fork() { + return new DefaultRandom(nextLong()); + } + + @Override + public Random asJdkRandom() + { + return delegate; + } +} diff --git a/test/unit/accord/utils/Gen.java b/test/unit/accord/utils/Gen.java new file mode 100644 index 0000000000..7563e8c29b --- /dev/null +++ b/test/unit/accord/utils/Gen.java @@ -0,0 +1,179 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package accord.utils; + +import java.util.function.BiFunction; +import java.util.function.Function; +import java.util.function.IntPredicate; +import java.util.function.IntSupplier; +import java.util.function.LongPredicate; +import java.util.function.LongSupplier; +import java.util.function.Predicate; +import java.util.function.Supplier; +import java.util.function.ToIntFunction; +import java.util.function.ToLongFunction; +import java.util.stream.IntStream; +import java.util.stream.LongStream; +import java.util.stream.Stream; + +public interface Gen { + /** + * For cases where method handles isn't able to detect the proper type, this method acts as a cast + * to inform the compiler of the desired type. + */ + static Gen of(Gen fn) + { + return fn; + } + + A next(RandomSource random); + + default Gen map(Function fn) + { + return r -> fn.apply(this.next(r)); + } + + default Gen map(BiFunction fn) + { + return r -> fn.apply(r, this.next(r)); + } + + default IntGen mapToInt(ToIntFunction fn) + { + return r -> fn.applyAsInt(next(r)); + } + + default LongGen mapToLong(ToLongFunction fn) + { + return r -> fn.applyAsLong(next(r)); + } + + default Gen flatMap(Function> mapper) + { + return rs -> mapper.apply(this.next(rs)).next(rs); + } + + default Gen flatMap(BiFunction> mapper) + { + return rs -> mapper.apply(rs, this.next(rs)).next(rs); + } + + default Gen filter(Predicate fn) + { + Gen self = this; + return r -> { + A value; + do { + value = self.next(r); + } + while (!fn.test(value)); + return value; + }; + } + + default Supplier asSupplier(RandomSource rs) + { + return () -> next(rs); + } + + default Stream asStream(RandomSource rs) + { + return Stream.generate(() -> next(rs)); + } + + interface IntGen extends Gen + { + int nextInt(RandomSource random); + + @Override + default Integer next(RandomSource random) + { + return nextInt(random); + } + + default Gen.IntGen filterInt(IntPredicate fn) + { + return rs -> { + int value; + do + { + value = nextInt(rs); + } + while (!fn.test(value)); + return value; + }; + } + + @Override + default Gen.IntGen filter(Predicate fn) + { + return filterInt(i -> fn.test(i)); + } + + default IntSupplier asIntSupplier(RandomSource rs) + { + return () -> nextInt(rs); + } + + default IntStream asIntStream(RandomSource rs) + { + return IntStream.generate(() -> nextInt(rs)); + } + } + + interface LongGen extends Gen + { + long nextLong(RandomSource random); + + @Override + default Long next(RandomSource random) + { + return nextLong(random); + } + + default Gen.LongGen filterLong(LongPredicate fn) + { + return rs -> { + long value; + do + { + value = nextLong(rs); + } + while (!fn.test(value)); + return value; + }; + } + + @Override + default Gen.LongGen filter(Predicate fn) + { + return filterLong(i -> fn.test(i)); + } + + default LongSupplier asLongSupplier(RandomSource rs) + { + return () -> nextLong(rs); + } + + default LongStream asLongStream(RandomSource rs) + { + return LongStream.generate(() -> nextLong(rs)); + } + } +} diff --git a/test/unit/accord/utils/Gens.java b/test/unit/accord/utils/Gens.java new file mode 100644 index 0000000000..4a80d724af --- /dev/null +++ b/test/unit/accord/utils/Gens.java @@ -0,0 +1,576 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package accord.utils; + +import java.lang.reflect.Array; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; +import java.util.EnumMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.NavigableSet; +import java.util.Objects; +import java.util.Set; +import java.util.function.Function; +import java.util.function.Supplier; +import java.util.stream.Collectors; + + +public class Gens { + private Gens() { + } + + public static Gen constant(T constant) + { + return ignore -> constant; + } + + public static Gen constant(Supplier constant) + { + return ignore -> constant.get(); + } + + public static Gen pick(T... ts) + { + return pick(Arrays.asList(ts)); + } + + public static Gen pick(List ts) + { + Gen.IntGen offset = ints().between(0, ts.size() - 1); + return rs -> ts.get(offset.nextInt(rs)); + } + + public static > Gen pick(Set set) + { + List list = new ArrayList<>(set); + // Non-ordered sets may have different iteration order on different environments, which would make a seed produce different histories! + // To avoid such a problem, make sure to apply a deterministic function (sort). + if (!(set instanceof NavigableSet)) + list.sort(Comparator.naturalOrder()); + return pick(list); + } + + public static Gen pick(Map values) + { + if (values == null || values.isEmpty()) + throw new IllegalArgumentException("values is empty"); + double totalWeight = values.values().stream().mapToDouble(Integer::intValue).sum(); + List> list = values.entrySet().stream().map(e -> new Weight<>(e.getKey(), e.getValue())).collect(Collectors.toList()); + Collections.sort(list); + return rs -> { + double value = rs.nextDouble() * totalWeight; + for (Weight w : list) + { + value -= w.weight; + if (value <= 0) + return w.value; + } + return list.get(list.size() - 1).value; + }; + } + + public static Gen charArray(Gen.IntGen sizes, char[] domain) + { + return charArray(sizes, domain, (a, b) -> true); + } + + public interface IntCharBiPredicate + { + boolean test(int a, char b); + } + + public static Gen charArray(Gen.IntGen sizes, char[] domain, IntCharBiPredicate fn) + { + Gen.IntGen indexGen = ints().between(0, domain.length - 1); + return rs -> { + int size = sizes.nextInt(rs); + char[] is = new char[size]; + for (int i = 0; i != size; i++) + { + char c; + do + { + c = domain[indexGen.nextInt(rs)]; + } + while (!fn.test(i, c)); + is[i] = c; + } + return is; + }; + } + + public static Gen random() { + return r -> r; + } + + public static BooleanDSL bools() + { + return new BooleanDSL(); + } + + public static IntDSL ints() + { + return new IntDSL(); + } + + public static LongDSL longs() { + return new LongDSL(); + } + + public static ListDSL lists(Gen fn) { + return new ListDSL<>(fn); + } + + public static ArrayDSL arrays(Class type, Gen fn) { + return new ArrayDSL<>(type, fn); + } + + public static IntArrayDSL arrays(Gen.IntGen fn) { + return new IntArrayDSL(fn); + } + + public static LongArrayDSL arrays(Gen.LongGen fn) { + return new LongArrayDSL(fn); + } + + public static EnumDSL enums() + { + return new EnumDSL(); + } + + public static StringDSL strings() + { + return new StringDSL(); + } + + public static class BooleanDSL + { + public Gen all() + { + return RandomSource::nextBoolean; + } + + public Gen runs(double ratio, int maxRuns) + { + Invariants.checkArgument(ratio > 0 && ratio <= 1, "Expected %d to be larger than 0 and <= 1", ratio); + double lower = ratio * .8; + double upper = ratio * 1.2; + return new Gen<>() { + // run represents how many consecutaive true values should be returned; -1 implies no active "run" exists + private int run = -1; + private long falseCount = 0, trueCount = 0; + @Override + public Boolean next(RandomSource rs) + { + if (run != -1) + { + run--; + trueCount++; + return true; + } + double currentRatio = trueCount / (double) (falseCount + trueCount); + if (currentRatio < lower) + { + // not enough true + trueCount++; + return true; + } + if (currentRatio > upper) + { + // not enough false + falseCount++; + return false; + } + if (rs.decide(ratio)) + { + run = rs.nextInt(maxRuns); + run--; + trueCount++; + return true; + } + falseCount++; + return false; + } + }; + } + } + + public static class IntDSL + { + public Gen.IntGen of(int value) + { + return r -> value; + } + + public Gen.IntGen all() + { + return RandomSource::nextInt; + } + + public Gen.IntGen between(int min, int max) + { + Invariants.checkArgument(max >= min, "max (%d) < min (%d)", max, min); + if (min == max) + return of(min); + // since bounds is exclusive, if max == max_value unable to do +1 to include... so will return a gen + // that does not include + if (max == Integer.MAX_VALUE) + return r -> r.nextInt(min, max); + return r -> r.nextInt(min, max + 1); + } + } + + public static class LongDSL { + public Gen.LongGen of(long value) + { + return r -> value; + } + + public Gen.LongGen all() { + return RandomSource::nextLong; + } + + public Gen.LongGen between(long min, long max) { + Invariants.checkArgument(max >= min); + if (min == max) + return of(min); + // since bounds is exclusive, if max == max_value unable to do +1 to include... so will return a gen + // that does not include + if (max == Long.MAX_VALUE) + return r -> r.nextLong(min, max); + return r -> r.nextLong(min, max + 1); + } + } + + public static class EnumDSL + { + public > Gen all(Class klass) + { + return pick(klass.getEnumConstants()); + } + + public > Gen allWithWeights(Class klass, int... weights) + { + T[] constants = klass.getEnumConstants(); + if (constants.length != weights.length) + throw new IllegalArgumentException(String.format("Total number of weights (%s) does not match the enum (%s)", Arrays.toString(weights), Arrays.toString(constants))); + Map values = new EnumMap<>(klass); + for (int i = 0; i < constants.length; i++) + values.put(constants[i], weights[i]); + return pick(values); + } + } + + public static class StringDSL + { + public Gen of(Gen.IntGen sizes, char[] domain) + { + // note, map is overloaded so String::new is ambugious to javac, so need a lambda here + return charArray(sizes, domain).map(c -> new String(c)); + } + + public SizeBuilder of(char[] domain) + { + return new SizeBuilder<>(sizes -> of(sizes, domain)); + } + + public Gen of(Gen.IntGen sizes, char[] domain, IntCharBiPredicate fn) + { + // note, map is overloaded so String::new is ambugious to javac, so need a lambda here + return charArray(sizes, domain, fn).map(c -> new String(c)); + } + + public SizeBuilder of(char[] domain, IntCharBiPredicate fn) + { + return new SizeBuilder<>(sizes -> of(sizes, domain, fn)); + } + + public Gen all(Gen.IntGen sizes) + { + return betweenCodePoints(sizes, Character.MIN_CODE_POINT, Character.MAX_CODE_POINT); + } + + public SizeBuilder all() + { + return new SizeBuilder<>(this::all); + } + + public Gen ascii(Gen.IntGen sizes) + { + return betweenCodePoints(sizes, 0, 127); + } + + public SizeBuilder ascii() + { + return new SizeBuilder<>(this::ascii); + } + + public Gen betweenCodePoints(Gen.IntGen sizes, int min, int max) + { + Gen.IntGen codePointGen = ints().between(min, max).filter(Character::isDefined); + return rs -> { + int[] array = new int[sizes.nextInt(rs)]; + for (int i = 0; i < array.length; i++) + array[i] = codePointGen.nextInt(rs); + return new String(array, 0, array.length); + }; + } + + public SizeBuilder betweenCodePoints(int min, int max) + { + return new SizeBuilder<>(sizes -> betweenCodePoints(sizes, min, max)); + } + } + + public static class SizeBuilder + { + private final Function> fn; + + public SizeBuilder(Function> fn) + { + this.fn = fn; + } + + public Gen ofLength(int fixed) + { + return ofLengthBetween(fixed, fixed); + } + + public Gen ofLengthBetween(int min, int max) + { + return fn.apply(ints().between(min, max)); + } + } + + public static class ListDSL implements BaseSequenceDSL, List> { + private final Gen fn; + + public ListDSL(Gen fn) { + this.fn = Objects.requireNonNull(fn); + } + + @Override + public ListDSL unique() + { + return new ListDSL<>(new GenReset<>(fn)); + } + + @Override + public Gen> ofSizeBetween(int minSize, int maxSize) { + Gen.IntGen sizeGen = ints().between(minSize, maxSize); + return r -> + { + Reset.tryReset(fn); + int size = sizeGen.nextInt(r); + List list = new ArrayList<>(size); + for (int i = 0; i < size; i++) + list.add(fn.next(r)); + return list; + }; + } + } + + public static class ArrayDSL implements BaseSequenceDSL, T[]> { + private final Class type; + private final Gen fn; + + public ArrayDSL(Class type, Gen fn) { + this.type = Objects.requireNonNull(type); + this.fn = Objects.requireNonNull(fn); + } + + @Override + public ArrayDSL unique() + { + return new ArrayDSL<>(type, new GenReset<>(fn)); + } + + @Override + public Gen ofSizeBetween(int minSize, int maxSize) { + Gen.IntGen sizeGen = ints().between(minSize, maxSize); + return r -> + { + Reset.tryReset(fn); + int size = sizeGen.nextInt(r); + T[] list = (T[]) Array.newInstance(type, size); + for (int i = 0; i < size; i++) + list[i] = fn.next(r); + return list; + }; + } + } + + public static class IntArrayDSL implements BaseSequenceDSL { + private final Gen.IntGen fn; + + public IntArrayDSL(Gen.IntGen fn) { + this.fn = Objects.requireNonNull(fn); + } + + @Override + public IntArrayDSL unique() + { + return new IntArrayDSL(new IntGenReset(fn)); + } + + @Override + public Gen ofSizeBetween(int minSize, int maxSize) { + Gen.IntGen sizeGen = ints().between(minSize, maxSize); + return r -> + { + int size = sizeGen.nextInt(r); + int[] list = new int[size]; + for (int i = 0; i < size; i++) + list[i] = fn.nextInt(r); + return list; + }; + } + } + + public static class LongArrayDSL implements BaseSequenceDSL { + private final Gen.LongGen fn; + + public LongArrayDSL(Gen.LongGen fn) { + this.fn = Objects.requireNonNull(fn); + } + + @Override + public LongArrayDSL unique() + { + return new LongArrayDSL(new LongGenReset(fn)); + } + + @Override + public Gen ofSizeBetween(int minSize, int maxSize) { + Gen.IntGen sizeGen = ints().between(minSize, maxSize); + return r -> + { + int size = sizeGen.nextInt(r); + long[] list = new long[size]; + for (int i = 0; i < size; i++) + list[i] = fn.nextLong(r); + return list; + }; + } + } + + public interface BaseSequenceDSL, B> + { + A unique(); + + Gen ofSizeBetween(int min, int max); + + default Gen ofSize(int size) { + return ofSizeBetween(size, size); + } + } + + private interface Reset { + static void tryReset(Object o) + { + if (o instanceof Reset) + ((Reset) o).reset(); + } + + void reset(); + } + + private static class GenReset implements Gen, Reset + { + private final Set seen = new HashSet<>(); + private final Gen fn; + + private GenReset(Gen fn) + { + this.fn = fn; + } + + @Override + public T next(RandomSource random) + { + T value; + while (!seen.add((value = fn.next(random)))) {} + return value; + } + + @Override + public void reset() + { + seen.clear(); + } + } + + private static class IntGenReset implements Gen.IntGen, Reset + { + private final GenReset base; + + private IntGenReset(Gen.IntGen fn) + { + this.base = new GenReset<>(fn); + } + @Override + public int nextInt(RandomSource random) { + return base.next(random); + } + + @Override + public void reset() { + base.reset(); + } + } + + private static class LongGenReset implements Gen.LongGen, Reset + { + private final GenReset base; + + private LongGenReset(Gen.LongGen fn) + { + this.base = new GenReset<>(fn); + } + @Override + public long nextLong(RandomSource random) { + return base.next(random); + } + + @Override + public void reset() { + base.reset(); + } + } + + private static class Weight implements Comparable> + { + private final T value; + private final double weight; + + private Weight(T value, double weight) { + this.value = value; + this.weight = weight; + } + + @Override + public int compareTo(Weight o) { + return Double.compare(weight, o.weight); + } + } +} diff --git a/test/unit/accord/utils/Invariants.java b/test/unit/accord/utils/Invariants.java new file mode 100644 index 0000000000..b960d8b268 --- /dev/null +++ b/test/unit/accord/utils/Invariants.java @@ -0,0 +1,327 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package accord.utils; + +import net.nicoulaj.compilecommand.annotations.Inline; + +import javax.annotation.Nullable; +import java.util.function.Predicate; + +import static java.lang.String.format; + +public class Invariants +{ + private static final boolean PARANOID = true; + private static final boolean DEBUG = true; + + public static boolean isParanoid() + { + return PARANOID; + } + public static boolean debug() + { + return DEBUG; + } + + private static void illegalState(String msg) + { + throw new IllegalStateException(msg); + } + + private static void illegalState() + { + illegalState(null); + } + + private static void illegalArgument(String msg) + { + throw new IllegalArgumentException(msg); + } + + + private static void illegalArgument() + { + illegalArgument(null); + } + + public static T2 checkType(T1 cast) + { + return (T2)cast; + } + + public static T2 checkType(Class to, T1 cast) + { + if (cast != null && !to.isInstance(cast)) + illegalState(); + return (T2)cast; + } + + public static T2 checkType(Class to, T1 cast, String msg) + { + if (cast != null && !to.isInstance(cast)) + illegalState(msg); + return (T2)cast; + } + + public static void paranoid(boolean condition) + { + if (PARANOID && !condition) + illegalState(); + } + + public static void checkState(boolean condition) + { + if (!condition) + illegalState(); + } + + public static void checkState(boolean condition, String msg) + { + if (!condition) + illegalState(msg); + } + + public static void checkState(boolean condition, String fmt, int p1) + { + if (!condition) + illegalState(format(fmt, p1)); + } + + public static void checkState(boolean condition, String fmt, int p1, int p2) + { + if (!condition) + illegalState(format(fmt, p1, p2)); + } + + public static void checkState(boolean condition, String fmt, long p1) + { + if (!condition) + illegalState(format(fmt, p1)); + } + + public static void checkState(boolean condition, String fmt, long p1, long p2) + { + if (!condition) + illegalState(format(fmt, p1, p2)); + } + + public static void checkState(boolean condition, String fmt, @Nullable Object p1) + { + if (!condition) + illegalState(format(fmt, p1)); + } + + public static void checkState(boolean condition, String fmt, @Nullable Object p1, @Nullable Object p2) + { + if (!condition) + illegalState(format(fmt, p1, p2)); + } + + public static void checkState(boolean condition, String fmt, Object... args) + { + if (!condition) + illegalState(format(fmt, args)); + } + + public static T nonNull(T param) + { + if (param == null) + throw new NullPointerException(); + return param; + } + + public static T nonNull(T param, String fmt, Object... args) + { + if (param == null) + throw new NullPointerException(format(fmt, args)); + return param; + } + + public static int isNatural(int input) + { + if (input < 0) + illegalState(); + return input; + } + + public static long isNatural(long input) + { + if (input < 0) + illegalState(); + return input; + } + + public static void checkArgument(boolean condition) + { + if (!condition) + illegalArgument(); + } + + public static void checkArgument(boolean condition, String msg) + { + if (!condition) + illegalArgument(msg); + } + + public static void checkArgument(boolean condition, String fmt, int p1) + { + if (!condition) + illegalArgument(format(fmt, p1)); + } + + public static void checkArgument(boolean condition, String fmt, int p1, int p2) + { + if (!condition) + illegalArgument(format(fmt, p1, p2)); + } + + public static void checkArgument(boolean condition, String fmt, long p1) + { + if (!condition) + illegalArgument(format(fmt, p1)); + } + + public static void checkArgument(boolean condition, String fmt, long p1, long p2) + { + if (!condition) + illegalArgument(format(fmt, p1, p2)); + } + + public static void checkArgument(boolean condition, String fmt, @Nullable Object p1) + { + if (!condition) + illegalArgument(format(fmt, p1)); + } + + public static void checkArgument(boolean condition, String fmt, @Nullable Object p1, @Nullable Object p2) + { + if (!condition) + illegalArgument(format(fmt, p1, p2)); + } + + public static void checkArgument(boolean condition, String fmt, Object... args) + { + if (!condition) + illegalArgument(format(fmt, args)); + } + + public static T checkArgument(T param, boolean condition) + { + if (!condition) + illegalArgument(); + return param; + } + + public static T checkArgument(T param, boolean condition, String msg) + { + if (!condition) + illegalArgument(msg); + return param; + } + + public static T checkArgument(T param, boolean condition, String fmt, int p1) + { + if (!condition) + illegalArgument(format(fmt, p1)); + return param; + } + + public static T checkArgument(T param, boolean condition, String fmt, int p1, int p2) + { + if (!condition) + illegalArgument(format(fmt, p1, p2)); + return param; + } + + public static T checkArgument(T param, boolean condition, String fmt, long p1) + { + if (!condition) + illegalArgument(format(fmt, p1)); + return param; + } + + public static T checkArgument(T param, boolean condition, String fmt, long p1, long p2) + { + if (!condition) + illegalArgument(format(fmt, p1, p2)); + return param; + } + + public static T checkArgument(T param, boolean condition, String fmt, @Nullable Object p1) + { + if (!condition) + illegalArgument(format(fmt, p1)); + return param; + } + + public static T checkArgument(T param, boolean condition, String fmt, @Nullable Object p1, @Nullable Object p2) + { + if (!condition) + illegalArgument(format(fmt, p1, p2)); + return param; + } + + public static T checkArgument(T param, boolean condition, String fmt, Object... args) + { + if (!condition) + illegalArgument(format(fmt, args)); + return param; + } + + @Inline + public static T checkArgument(T param, Predicate condition) + { + if (!condition.test(param)) + illegalArgument(); + return param; + } + + @Inline + public static T checkArgument(T param, Predicate condition, String msg) + { + if (!condition.test(param)) + illegalArgument(msg); + return param; + } + + public static O cast(Object o, Class klass) + { + try + { + return klass.cast(o); + } + catch (ClassCastException e) + { + throw new IllegalArgumentException(format("Unable to cast %s to %s", o, klass.getName())); + } + } + + public static void checkIndexInBounds(int realLength, int offset, int length) + { + if (realLength == 0 || length == 0) + throw new IndexOutOfBoundsException("Unable to access offset " + offset + "; empty"); + if (offset < 0) + throw new IndexOutOfBoundsException("Offset " + offset + " must not be negative"); + if (length < 0) + throw new IndexOutOfBoundsException("Length " + length + " must not be negative"); + int endOffset = offset + length; + if (endOffset > realLength) + throw new IndexOutOfBoundsException(String.format("Offset %d, length = %d; real length was %d", offset, length, realLength)); + } +} diff --git a/test/unit/accord/utils/Property.java b/test/unit/accord/utils/Property.java new file mode 100644 index 0000000000..b90472c862 --- /dev/null +++ b/test/unit/accord/utils/Property.java @@ -0,0 +1,382 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package accord.utils; + +import java.time.Duration; +import java.util.Arrays; +import java.util.Objects; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import javax.annotation.Nullable; + +import org.apache.cassandra.transport.ProtocolException; +import org.apache.cassandra.utils.concurrent.AsyncPromise; + +public class Property +{ + public static abstract class Common> + { + protected long seed = ThreadLocalRandom.current().nextLong(); + protected int examples = 1000; + + protected boolean pure = true; + @Nullable + protected Duration timeout = null; + + protected Common() { + } + + protected Common(Common other) { + this.seed = other.seed; + this.examples = other.examples; + this.pure = other.pure; + this.timeout = other.timeout; + } + + public T withSeed(long seed) + { + this.seed = seed; + return (T) this; + } + + public T withExamples(int examples) + { + if (examples <= 0) + throw new IllegalArgumentException("Examples must be positive"); + this.examples = examples; + return (T) this; + } + + public T withPure(boolean pure) + { + this.pure = pure; + return (T) this; + } + + public T withTimeout(Duration timeout) + { + this.timeout = timeout; + return (T) this; + } + + protected void checkWithTimeout(Runnable fn) + { + AsyncPromise promise = new AsyncPromise<>(); + Thread t = new Thread(() -> { + try + { + fn.run(); + promise.setSuccess(null); + } + catch (Throwable e) + { + promise.setFailure(e); + } + }); + t.setName("property with timeout"); + t.setDaemon(true); + try + { + t.start(); + promise.get(timeout.toNanos(), TimeUnit.NANOSECONDS); + } + catch (ExecutionException e) + { + throw new ProtocolException(propertyError(this, e.getCause())); + } + catch (InterruptedException e) + { + t.interrupt(); + throw new PropertyError(propertyError(this, e)); + } + catch (TimeoutException e) + { + t.interrupt(); + TimeoutException override = new TimeoutException("property test did not complete within " + this.timeout); + override.setStackTrace(new StackTraceElement[0]); + throw new PropertyError(propertyError(this, override)); + } + } + } + + public static class ForBuilder extends Common + { + public void check(FailingConsumer fn) + { + forAll(Gens.random()).check(fn); + } + + public SingleBuilder forAll(Gen gen) + { + return new SingleBuilder<>(gen, this); + } + + public DoubleBuilder forAll(Gen a, Gen b) + { + return new DoubleBuilder<>(a, b, this); + } + + public TrippleBuilder forAll(Gen a, Gen b, Gen c) + { + return new TrippleBuilder<>(a, b, c, this); + } + } + + private static Object normalizeValue(Object value) + { + if (value == null) + return null; + // one day java arrays will have a useful toString... one day... + if (value.getClass().isArray()) + { + Class subType = value.getClass().getComponentType(); + if (!subType.isPrimitive()) + return Arrays.asList((Object[]) value); + if (Byte.TYPE == subType) + return Arrays.toString((byte[]) value); + if (Character.TYPE == subType) + return Arrays.toString((char[]) value); + if (Short.TYPE == subType) + return Arrays.toString((short[]) value); + if (Integer.TYPE == subType) + return Arrays.toString((int[]) value); + if (Long.TYPE == subType) + return Arrays.toString((long[]) value); + if (Float.TYPE == subType) + return Arrays.toString((float[]) value); + if (Double.TYPE == subType) + return Arrays.toString((double[]) value); + } + try + { + return value.toString(); + } + catch (Throwable t) + { + return "Object.toString failed: " + t.getClass().getCanonicalName() + ": " + t.getMessage(); + } + } + + private static String propertyError(Common input, Throwable cause, Object... values) + { + StringBuilder sb = new StringBuilder(); + // return "Seed=" + seed + "\nExamples=" + examples; + sb.append("Property error detected:\nSeed = ").append(input.seed).append('\n'); + sb.append("Examples = ").append(input.examples).append('\n'); + sb.append("Pure = ").append(input.pure).append('\n'); + if (cause != null) + { + String msg = cause.getMessage(); + sb.append("Error: "); + // to improve readability, if a newline is detected move the error msg to the next line + if (msg != null && msg.contains("\n")) + msg = "\n\t" + msg.replace("\n", "\n\t"); + if (msg == null) + msg = cause.getClass().getCanonicalName(); + sb.append(msg).append('\n'); + } + if (values != null) + { + sb.append("Values:\n"); + for (int i = 0; i < values.length; i++) + sb.append('\t').append(i).append(" = ").append(normalizeValue(values[i])).append('\n'); + } + return sb.toString(); + } + + public interface FailingConsumer + { + void accept(A value) throws Exception; + } + + public static class SingleBuilder extends Common> + { + private final Gen gen; + + private SingleBuilder(Gen gen, Common other) { + super(other); + this.gen = Objects.requireNonNull(gen); + } + + public void check(FailingConsumer fn) + { + if (timeout != null) + { + checkWithTimeout(() -> checkInternal(fn)); + return; + } + checkInternal(fn); + } + + private void checkInternal(FailingConsumer fn) + { + RandomSource random = new DefaultRandom(seed); + for (int i = 0; i < examples; i++) + { + T value = null; + try + { + checkInterrupted(); + fn.accept(value = gen.next(random)); + } + catch (Throwable t) + { + throw new PropertyError(propertyError(this, t, value), t); + } + if (pure) + { + seed = random.nextLong(); + random.setSeed(seed); + } + } + } + } + + public interface FailingBiConsumer + { + void accept(A a, B b) throws Exception; + } + + public static class DoubleBuilder extends Common> + { + private final Gen aGen; + private final Gen bGen; + + private DoubleBuilder(Gen aGen, Gen bGen, Common other) { + super(other); + this.aGen = Objects.requireNonNull(aGen); + this.bGen = Objects.requireNonNull(bGen); + } + + public void check(FailingBiConsumer fn) + { + if (timeout != null) + { + checkWithTimeout(() -> checkInternal(fn)); + return; + } + checkInternal(fn); + } + + private void checkInternal(FailingBiConsumer fn) + { + RandomSource random = new DefaultRandom(seed); + for (int i = 0; i < examples; i++) + { + A a = null; + B b = null; + try + { + checkInterrupted(); + fn.accept(a = aGen.next(random), b = bGen.next(random)); + } + catch (Throwable t) + { + throw new PropertyError(propertyError(this, t, a, b), t); + } + if (pure) + { + seed = random.nextLong(); + random.setSeed(seed); + } + } + } + } + + public interface FailingTriConsumer + { + void accept(A a, B b, C c) throws Exception; + } + + public static class TrippleBuilder extends Common> + { + private final Gen as; + private final Gen bs; + private final Gen cs; + + public TrippleBuilder(Gen as, Gen bs, Gen cs, Common other) + { + super(other); + this.as = as; + this.bs = bs; + this.cs = cs; + } + + public void check(FailingTriConsumer fn) + { + if (timeout != null) + { + checkWithTimeout(() -> checkInternal(fn)); + return; + } + checkInternal(fn); + } + + private void checkInternal(FailingTriConsumer fn) + { + RandomSource random = new DefaultRandom(seed); + for (int i = 0; i < examples; i++) + { + A a = null; + B b = null; + C c = null; + try + { + checkInterrupted(); + fn.accept(a = as.next(random), b = bs.next(random), c = cs.next(random)); + } + catch (Throwable t) + { + throw new PropertyError(propertyError(this, t, a, b, c), t); + } + if (pure) + { + seed = random.nextLong(); + random.setSeed(seed); + } + } + } + } + + private static void checkInterrupted() throws InterruptedException + { + if (Thread.currentThread().isInterrupted()) + throw new InterruptedException(); + } + + public static class PropertyError extends AssertionError + { + public PropertyError(String message, Throwable cause) + { + super(message, cause); + } + + public PropertyError(String message) + { + super(message); + } + } + + public static ForBuilder qt() + { + return new ForBuilder(); + } +} diff --git a/test/unit/accord/utils/RandomSource.java b/test/unit/accord/utils/RandomSource.java new file mode 100644 index 0000000000..3d4861e5e4 --- /dev/null +++ b/test/unit/accord/utils/RandomSource.java @@ -0,0 +1,361 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package accord.utils; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.NavigableSet; +import java.util.Random; +import java.util.Set; +import java.util.stream.DoubleStream; +import java.util.stream.IntStream; +import java.util.stream.LongStream; + +public interface RandomSource +{ + static RandomSource wrap(Random random) + { + return new WrappedRandomSource(random); + } + + void nextBytes(byte[] bytes); + + boolean nextBoolean(); + + int nextInt(); + + default int nextInt(int maxExclusive) + { + return nextInt(0, maxExclusive); + } + + default int nextInt(int minInclusive, int maxExclusive) + { + // this is diff behavior than ThreadLocalRandom, which returns nextInt + if (minInclusive >= maxExclusive) + throw new IllegalArgumentException(String.format("Min (%s) should be less than max (%d).", minInclusive, maxExclusive)); + + int result = nextInt(); + int delta = maxExclusive - minInclusive; + int mask = delta - 1; + if ((delta & mask) == 0) // power of two + result = (result & mask) + minInclusive; + else if (delta > 0) + { + // reject over-represented candidates + for (int u = result >>> 1; // ensure nonnegative + u + mask - (result = u % delta) < 0; // rejection check + u = nextInt() >>> 1) // retry + ; + result += minInclusive; + } + else + { + // range not representable as int + while (result < minInclusive || result >= maxExclusive) + result = nextInt(); + } + return result; + } + + default IntStream ints() + { + return IntStream.generate(this::nextInt); + } + + default IntStream ints(int maxExclusive) + { + return IntStream.generate(() -> nextInt(maxExclusive)); + } + + default IntStream ints(int minInclusive, int maxExclusive) + { + return IntStream.generate(() -> nextInt(minInclusive, maxExclusive)); + } + + long nextLong(); + + default long nextLong(long maxExclusive) + { + return nextLong(0, maxExclusive); + } + + default long nextLong(long minInclusive, long maxExclusive) + { + // this is diff behavior than ThreadLocalRandom, which returns nextLong + if (minInclusive >= maxExclusive) + throw new IllegalArgumentException(String.format("Min (%s) should be less than max (%d).", minInclusive, maxExclusive)); + + long result = nextLong(); + long delta = maxExclusive - minInclusive; + long mask = delta - 1; + if ((delta & mask) == 0L) // power of two + result = (result & mask) + minInclusive; + else if (delta > 0L) + { + // reject over-represented candidates + for (long u = result >>> 1; // ensure nonnegative + u + mask - (result = u % delta) < 0L; // rejection check + u = nextLong() >>> 1) // retry + ; + result += minInclusive; + } + else + { + // range not representable as long + while (result < minInclusive || result >= maxExclusive) + result = nextLong(); + } + return result; + } + + default LongStream longs() + { + return LongStream.generate(this::nextLong); + } + + default LongStream longs(long maxExclusive) + { + return LongStream.generate(() -> nextLong(maxExclusive)); + } + + default LongStream longs(long minInclusive, long maxExclusive) + { + return LongStream.generate(() -> nextLong(minInclusive, maxExclusive)); + } + + float nextFloat(); + + double nextDouble(); + + default double nextDouble(double maxExclusive) + { + return nextDouble(0, maxExclusive); + } + + default double nextDouble(double minInclusive, double maxExclusive) + { + if (minInclusive >= maxExclusive) + throw new IllegalArgumentException(String.format("Min (%s) should be less than max (%d).", minInclusive, maxExclusive)); + + double result = nextDouble(); + result = result * (maxExclusive - minInclusive) + minInclusive; + if (result >= maxExclusive) // correct for rounding + result = Double.longBitsToDouble(Double.doubleToLongBits(maxExclusive) - 1); + return result; + } + + default DoubleStream doubles() + { + return DoubleStream.generate(this::nextDouble); + } + + default DoubleStream doubles(double maxExclusive) + { + return DoubleStream.generate(() -> nextDouble(maxExclusive)); + } + + default DoubleStream doubles(double minInclusive, double maxExclusive) + { + return DoubleStream.generate(() -> nextDouble(minInclusive, maxExclusive)); + } + + double nextGaussian(); + + default int pickInt(int first, int second, int... rest) + { + int offset = nextInt(0, rest.length + 2); + switch (offset) + { + case 0: return first; + case 1: return second; + default: return rest[offset - 2]; + } + } + + default int pickInt(int[] array) + { + return pickInt(array, 0, array.length); + } + + default int pickInt(int[] array, int offset, int length) + { + Invariants.checkIndexInBounds(array.length, offset, length); + if (length == 1) + return array[offset]; + return array[nextInt(offset, offset + length)]; + } + + default long pickLong(long first, long second, long... rest) + { + int offset = nextInt(0, rest.length + 2); + switch (offset) + { + case 0: return first; + case 1: return second; + default: return rest[offset - 2]; + } + } + + default long pickLong(long[] array) + { + return pickLong(array, 0, array.length); + } + + default long pickLong(long[] array, int offset, int length) + { + Invariants.checkIndexInBounds(array.length, offset, length); + if (length == 1) + return array[offset]; + return array[nextInt(offset, offset + length)]; + } + + default > T pick(Set set) + { + List values = new ArrayList<>(set); + // Non-ordered sets may have different iteration order on different environments, which would make a seed produce different histories! + // To avoid such a problem, make sure to apply a deterministic function (sort). + if (!(set instanceof NavigableSet)) + values.sort(Comparator.naturalOrder()); + return pick(values); + } + + default T pick(T first, T second, T... rest) + { + int offset = nextInt(0, rest.length + 2); + switch (offset) + { + case 0: return first; + case 1: return second; + default: return rest[offset - 2]; + } + } + + default T pick(T[] array) + { + return array[nextInt(array.length)]; + } + + default T pick(List values) + { + return pick(values, 0, values.size()); + } + + default T pick(List values, int offset, int length) + { + Invariants.checkIndexInBounds(values.size(), offset, length); + if (length == 1) + return values.get(offset); + return values.get(nextInt(offset, offset + length)); + } + + void setSeed(long seed); + + RandomSource fork(); + + /** + * Returns true with a probability of {@code chance}. This logic is logically the same as + * 
{@code nextFloat() < chance}
+ * + * @param chance cumulative probability in range [0..1] + */ + default boolean decide(float chance) + { + return nextFloat() < chance; + } + + /** + * Returns true with a probability of {@code chance}. This logic is logically the same as + * 
{@code nextDouble() < chance}
+ * + * @param chance cumulative probability in range [0..1] + */ + default boolean decide(double chance) + { + return nextDouble() < chance; + } + + default long reset() + { + long seed = nextLong(); + setSeed(seed); + return seed; + } + + default Random asJdkRandom() + { + return new Random() + { + @Override + public void setSeed(long seed) + { + RandomSource.this.setSeed(seed); + } + + @Override + public void nextBytes(byte[] bytes) + { + RandomSource.this.nextBytes(bytes); + } + + @Override + public int nextInt() + { + return RandomSource.this.nextInt(); + } + + @Override + public int nextInt(int bound) + { + return RandomSource.this.nextInt(bound); + } + + @Override + public long nextLong() + { + return RandomSource.this.nextLong(); + } + + @Override + public boolean nextBoolean() + { + return RandomSource.this.nextBoolean(); + } + + @Override + public float nextFloat() + { + return RandomSource.this.nextFloat(); + } + + @Override + public double nextDouble() + { + return RandomSource.this.nextDouble(); + } + + @Override + public double nextGaussian() + { + return RandomSource.this.nextGaussian(); + } + }; + } +} diff --git a/test/unit/accord/utils/WrappedRandomSource.java b/test/unit/accord/utils/WrappedRandomSource.java new file mode 100644 index 0000000000..3d02c101fb --- /dev/null +++ b/test/unit/accord/utils/WrappedRandomSource.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package accord.utils; + +import java.util.Random; + +class WrappedRandomSource implements RandomSource +{ + private final Random random; + + WrappedRandomSource(Random random) + { + this.random = random; + } + + @Override + public Random asJdkRandom() + { + return random; + } + + @Override + public void nextBytes(byte[] bytes) + { + random.nextBytes(bytes); + } + + @Override + public boolean nextBoolean() + { + return random.nextBoolean(); + } + + @Override + public int nextInt() + { + return random.nextInt(); + } + + @Override + public int nextInt(int maxExclusive) + { + return random.nextInt(maxExclusive); + } + + @Override + public long nextLong() + { + return random.nextLong(); + } + + @Override + public float nextFloat() + { + return random.nextFloat(); + } + + @Override + public double nextDouble() + { + return random.nextDouble(); + } + + @Override + public double nextGaussian() + { + return random.nextGaussian(); + } + + @Override + public void setSeed(long seed) + { + random.setSeed(seed); + } + + @Override + public RandomSource fork() + { + return new WrappedRandomSource(new Random(nextLong())); + } +} diff --git a/test/unit/org/apache/cassandra/concurrent/ForwardingExecutorPlus.java b/test/unit/org/apache/cassandra/concurrent/ForwardingExecutorPlus.java new file mode 100644 index 0000000000..71c88a947e --- /dev/null +++ b/test/unit/org/apache/cassandra/concurrent/ForwardingExecutorPlus.java @@ -0,0 +1,219 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.concurrent; + +import java.util.List; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; + +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; + +import org.apache.cassandra.utils.WithResources; +import org.apache.cassandra.utils.concurrent.AsyncPromise; +import org.apache.cassandra.utils.concurrent.Future; + +public class ForwardingExecutorPlus implements ExecutorPlus +{ + private final ExecutorService delegate; + + public ForwardingExecutorPlus(ExecutorService delegate) + { + this.delegate = delegate; + } + + protected ExecutorService delegate() + { + return delegate; + } + + @Override + public void shutdown() + { + delegate().shutdown(); + } + + @Override + public List shutdownNow() + { + return delegate().shutdownNow(); + } + + @Override + public boolean isShutdown() + { + return delegate().isShutdown(); + } + + @Override + public boolean isTerminated() + { + return delegate().isTerminated(); + } + + @Override + public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException + { + return delegate().awaitTermination(timeout, unit); + } + + @Override + public Future submit(Callable task) + { + return wrap(delegate().submit(task)); + } + + @Override + public Future submit(Runnable task, T result) + { + return wrap(delegate().submit(task, result)); + } + + @Override + public Future submit(Runnable task) + { + return wrap(delegate().submit(task)); + } + + @Override + public void execute(WithResources withResources, Runnable task) + { + execute(TaskFactory.standard().toExecute(withResources, task)); + } + + @Override + public Future submit(WithResources withResources, Callable task) + { + class Catch + { + T value; + } + Catch c = new Catch(); + Runnable exec = TaskFactory.standard().toExecute(withResources, () -> { + try + { + c.value = task.call(); + } + catch (Exception e) + { + throw new RuntimeException(e); + } + }); + return submit(() -> { + exec.run(); + return c.value; + }); + } + + @Override + public Future submit(WithResources withResources, Runnable task) + { + return submit(TaskFactory.standard().toExecute(withResources, task)); + } + + @Override + public Future submit(WithResources withResources, Runnable task, T result) + { + return submit(Executors.callable(TaskFactory.standard().toSubmit(withResources, task), result)); + } + + @Override + public boolean inExecutor() + { + return false; + } + + @Override + public void execute(Runnable command) + { + delegate().execute(command); + } + + @Override + public int getCorePoolSize() + { + return 0; + } + + @Override + public void setCorePoolSize(int newCorePoolSize) + { + + } + + @Override + public int getMaximumPoolSize() + { + return 0; + } + + @Override + public void setMaximumPoolSize(int newMaximumPoolSize) + { + + } + + @Override + public int getActiveTaskCount() + { + return 0; + } + + @Override + public long getCompletedTaskCount() + { + return 0; + } + + @Override + public int getPendingTaskCount() + { + return 0; + } + + private Future wrap(java.util.concurrent.Future submit) + { + if (submit instanceof Future) + return (Future) submit; + if (submit instanceof ListenableFuture) + { + AsyncPromise promise = new AsyncPromise<>(); + Futures.addCallback((ListenableFuture) submit, new FutureCallback() + { + @Override + public void onSuccess(T result) + { + promise.setSuccess(result); + } + + @Override + public void onFailure(Throwable t) + { + promise.setFailure(t); + } + }, MoreExecutors.directExecutor()); + return promise; + } + throw new IllegalStateException("Unexpected future type: " + submit.getClass()); + } +} diff --git a/test/unit/org/apache/cassandra/concurrent/ForwardingLocalAwareExecutorPlus.java b/test/unit/org/apache/cassandra/concurrent/ForwardingLocalAwareExecutorPlus.java new file mode 100644 index 0000000000..e0bde7646d --- /dev/null +++ b/test/unit/org/apache/cassandra/concurrent/ForwardingLocalAwareExecutorPlus.java @@ -0,0 +1,167 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.concurrent; + +import java.util.List; +import java.util.concurrent.Callable; +import java.util.concurrent.TimeUnit; + +import org.apache.cassandra.utils.WithResources; +import org.apache.cassandra.utils.concurrent.Future; + +public class ForwardingLocalAwareExecutorPlus implements LocalAwareExecutorPlus +{ + private final ExecutorPlus delegate; + + public ForwardingLocalAwareExecutorPlus(ExecutorPlus delegate) + { + this.delegate = delegate; + } + + protected ExecutorPlus delegate() + { + return delegate; + } + + @Override + public void shutdown() + { + delegate().shutdown(); + } + + @Override + public List shutdownNow() + { + return delegate().shutdownNow(); + } + + @Override + public boolean isShutdown() + { + return delegate().isShutdown(); + } + + @Override + public boolean isTerminated() + { + return delegate().isTerminated(); + } + + @Override + public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException + { + return delegate().awaitTermination(timeout, unit); + } + + @Override + public Future submit(Callable task) + { + return delegate().submit(task); + } + + @Override + public Future submit(Runnable task, T result) + { + return delegate().submit(task, result); + } + + @Override + public Future submit(Runnable task) + { + return delegate().submit(task); + } + + @Override + public void execute(WithResources withResources, Runnable task) + { + delegate().execute(withResources, task); + } + + @Override + public Future submit(WithResources withResources, Callable task) + { + return delegate().submit(withResources, task); + } + + @Override + public Future submit(WithResources withResources, Runnable task) + { + return delegate().submit(withResources, task); + } + + @Override + public Future submit(WithResources withResources, Runnable task, T result) + { + return delegate().submit(withResources, task, result); + } + + @Override + public boolean inExecutor() + { + return delegate().inExecutor(); + } + + @Override + public void execute(Runnable command) + { + delegate().execute(command); + } + + @Override + public int getCorePoolSize() + { + return delegate().getCorePoolSize(); + } + + @Override + public void setCorePoolSize(int newCorePoolSize) + { + delegate().setCorePoolSize(newCorePoolSize); + } + + @Override + public int getMaximumPoolSize() + { + return delegate().getMaximumPoolSize(); + } + + @Override + public void setMaximumPoolSize(int newMaximumPoolSize) + { + delegate().setMaximumPoolSize(newMaximumPoolSize); + } + + @Override + public int getActiveTaskCount() + { + return delegate().getActiveTaskCount(); + } + + @Override + public long getCompletedTaskCount() + { + return delegate().getCompletedTaskCount(); + } + + @Override + public int getPendingTaskCount() + { + return delegate().getPendingTaskCount(); + } +} diff --git a/test/unit/org/apache/cassandra/concurrent/ForwardingScheduledExecutorPlus.java b/test/unit/org/apache/cassandra/concurrent/ForwardingScheduledExecutorPlus.java new file mode 100644 index 0000000000..b6f0bb6067 --- /dev/null +++ b/test/unit/org/apache/cassandra/concurrent/ForwardingScheduledExecutorPlus.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.concurrent; + +import java.util.concurrent.Callable; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; + +import static com.google.common.primitives.Longs.max; +import static java.util.concurrent.TimeUnit.NANOSECONDS; +import static org.apache.cassandra.utils.Clock.Global.nanoTime; + +public class ForwardingScheduledExecutorPlus extends ForwardingExecutorPlus implements ScheduledExecutorPlus +{ + private final ScheduledExecutorService delegate; + + public ForwardingScheduledExecutorPlus(ScheduledExecutorService delegate) + { + super(delegate); + this.delegate = delegate; + } + + protected ScheduledExecutorService delegate() + { + return delegate; + } + + @Override + public ScheduledFuture scheduleSelfRecurring(Runnable run, long delay, TimeUnit units) + { + return schedule(run, delay, units); + } + + @Override + public ScheduledFuture scheduleAt(Runnable run, long deadline) + { + return schedule(run, max(0, deadline - nanoTime()), NANOSECONDS); + } + + @Override + public ScheduledFuture scheduleTimeoutAt(Runnable run, long deadline) + { + return scheduleTimeoutWithDelay(run, max(0, deadline - nanoTime()), NANOSECONDS); + } + + @Override + public ScheduledFuture scheduleTimeoutWithDelay(Runnable run, long delay, TimeUnit units) + { + return schedule(run, delay, units); + } + + @Override + public ScheduledFuture schedule(Runnable command, long delay, TimeUnit unit) + { + return delegate().schedule(command, delay, unit); + } + + @Override + public ScheduledFuture schedule(Callable callable, long delay, TimeUnit unit) + { + return delegate().schedule(callable, delay, unit); + } + + @Override + public ScheduledFuture scheduleAtFixedRate(Runnable command, long initialDelay, long period, TimeUnit unit) + { + return delegate().scheduleAtFixedRate(command, initialDelay, period, unit); + } + + @Override + public ScheduledFuture scheduleWithFixedDelay(Runnable command, long initialDelay, long delay, TimeUnit unit) + { + return delegate().scheduleWithFixedDelay(command, initialDelay, delay, unit); + } +} diff --git a/test/unit/org/apache/cassandra/concurrent/SimulatedExecutorFactory.java b/test/unit/org/apache/cassandra/concurrent/SimulatedExecutorFactory.java new file mode 100644 index 0000000000..ffb1a0fabd --- /dev/null +++ b/test/unit/org/apache/cassandra/concurrent/SimulatedExecutorFactory.java @@ -0,0 +1,506 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.concurrent; + +import java.util.Collections; +import java.util.LinkedList; +import java.util.List; +import java.util.PriorityQueue; +import java.util.Queue; +import java.util.concurrent.AbstractExecutorService; +import java.util.concurrent.Callable; +import java.util.concurrent.Delayed; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.RejectedExecutionHandler; +import java.util.concurrent.RunnableFuture; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.function.LongSupplier; + +import accord.utils.Gens; +import accord.utils.RandomSource; +import org.apache.cassandra.utils.Clock; + +import static java.util.concurrent.TimeUnit.NANOSECONDS; + +public class SimulatedExecutorFactory implements ExecutorFactory, Clock +{ + private static class Item implements Comparable + { + private final long runAtNanos; + private final long seq; + private final org.apache.cassandra.utils.concurrent.RunnableFuture action; + + private Item(long runAtNanos, long seq, org.apache.cassandra.utils.concurrent.RunnableFuture action) + { + if (runAtNanos < 0) + throw new IllegalArgumentException("Time went backwards! Given " + runAtNanos); + this.runAtNanos = runAtNanos; + this.seq = seq; + this.action = action; + } + + @Override + public int compareTo(Item o) + { + int rc = Long.compare(runAtNanos, o.runAtNanos); + if (rc != 0) + return rc; + return Long.compare(seq, o.seq); + } + + @Override + public String toString() + { + return "Item{" + + "runAtNanos=" + runAtNanos + + ", seq=" + seq + + '}'; + } + } + + private final RandomSource rs; + private final long startTimeNanos; + private final PriorityQueue queue = new PriorityQueue<>(); + private long seq = 0; + private long nowNanos; + private int repeatedTasks = 0; + + public SimulatedExecutorFactory(RandomSource rs, long startTimeNanos) + { + this.rs = rs; + this.startTimeNanos = startTimeNanos; + } + + public boolean processOne() + { + // if we count the repeated tasks, then processAll will never complete + if (queue.size() == repeatedTasks) + return false; + Item item = queue.poll(); + if (item == null) + return false; + nowNanos = Math.max(nowNanos + 1, item.runAtNanos); + item.action.run(); + return true; + } + + @Override + public long nanoTime() + { + return nowNanos++; + } + + @Override + public long currentTimeMillis() + { + return TimeUnit.NANOSECONDS.toMillis(startTimeNanos + nanoTime()); + } + + + @Override + public ExecutorBuilder configureSequential(String name) + { + return new SimulatedExecutorBuilder<>().configureSequential(name); + } + + @Override + public ExecutorBuilder configurePooled(String name, int threads) + { + return new SimulatedExecutorBuilder<>().configurePooled(name, threads); + } + + @Override + public ExecutorBuilderFactory withJmx(String jmxPath) + { + return this; + } + + @Override + public LocalAwareSubFactory localAware() + { + return new SimulatedExecutorBuilder<>(); + } + + @Override + public ScheduledExecutorPlus scheduled(boolean executeOnShutdown, String name, int priority, SimulatorSemantics simulatorSemantics) + { + return new ForwardingScheduledExecutorPlus(new UnorderedScheduledExecutorService()); + } + + @Override + public Thread startThread(String name, Runnable runnable, InfiniteLoopExecutor.Daemon daemon) + { + throw new UnsupportedOperationException("Thread can't be simualted"); + } + + @Override + public Interruptible infiniteLoop(String name, Interruptible.Task task, InfiniteLoopExecutor.SimulatorSafe simulatorSafe, InfiniteLoopExecutor.Daemon daemon, InfiniteLoopExecutor.Interrupts interrupts) + { + throw new UnsupportedOperationException("TODO"); + } + + @Override + public ThreadGroup newThreadGroup(String name) + { + throw new UnsupportedOperationException("Thread can't be simualted"); + } + + private class SimulatedExecutorBuilder implements ExecutorBuilder, LocalAwareSubFactory, LocalAwareSubFactoryWithJMX + { + private int threads = -1; + + @Override + public ExecutorBuilder withKeepAlive(long keepAlive, TimeUnit keepAliveUnits) + { + return this; + } + + @Override + public ExecutorBuilder withKeepAlive() + { + return this; + } + + @Override + public ExecutorBuilder withThreadPriority(int threadPriority) + { + return this; + } + + @Override + public ExecutorBuilder withThreadGroup(ThreadGroup threadGroup) + { + return this; + } + + @Override + public ExecutorBuilder withDefaultThreadGroup() + { + return this; + } + + @Override + public ExecutorBuilder withQueueLimit(int queueLimit) + { + throw new UnsupportedOperationException("TODO"); + } + + @Override + public ExecutorBuilder withRejectedExecutionHandler(RejectedExecutionHandler rejectedExecutionHandler) + { + throw new UnsupportedOperationException("TODO"); + } + + @Override + public ExecutorBuilder withUncaughtExceptionHandler(Thread.UncaughtExceptionHandler uncaughtExceptionHandler) + { + throw new UnsupportedOperationException("TODO"); + } + + @Override + public ExecutorBuilder configureSequential(String name) + { + threads = 1; + return (ExecutorBuilder) this; + } + + @Override + public ExecutorBuilder configurePooled(String name, int threads) + { + this.threads = threads; + return (ExecutorBuilder) this; + } + + @Override + public LocalAwareSubFactoryWithJMX withJmx(String jmxPath) + { + return this; + } + + @Override + public LocalAwareSubFactoryWithJMX withJmxInternal() + { + return this; + } + + @Override + public LocalAwareExecutorPlus shared(String name, int threads, ExecutorPlus.MaximumPoolSizeListener onSetMaxSize) + { + return new ForwardingLocalAwareExecutorPlus(build0()); + } + + @Override + public E build() + { + return (E) build0(); + } + + private ForwardingExecutorPlus build0() + { + return new ForwardingExecutorPlus(threads == 1 ? + new OrderedExecutorService() : + new UnorderedExecutorService()); + } + } + + private class UnorderedExecutorService extends AbstractExecutorService + { + private final LongSupplier jitterNanos; + private boolean shutdown = false; + + UnorderedExecutorService() + { + long maxSmall = TimeUnit.MICROSECONDS.toNanos(50); + long max = TimeUnit.MILLISECONDS.toNanos(5); + LongSupplier small = () -> rs.nextLong(0, maxSmall); + LongSupplier large = () -> rs.nextLong(maxSmall, max); + this.jitterNanos = Gens.bools().runs(rs.nextInt(1, 11) / 100.0D, rs.nextInt(3, 15)).mapToLong(b -> b ? large.getAsLong() : small.getAsLong()).asLongSupplier(rs); + } + + @Override + protected RunnableFuture newTaskFor(Runnable runnable, T value) + { + return new FutureTask<>(Executors.callable(runnable, value)); + } + + @Override + protected RunnableFuture newTaskFor(Callable callable) + { + return new FutureTask<>(callable); + } + + protected org.apache.cassandra.utils.concurrent.RunnableFuture taskFor(Runnable command) + { + if (command instanceof org.apache.cassandra.utils.concurrent.RunnableFuture) + return (org.apache.cassandra.utils.concurrent.RunnableFuture) command; + return new FutureTask<>(Executors.callable(command)); + } + + @Override + public void shutdown() + { + shutdown = true; + } + + @Override + public List shutdownNow() + { + return Collections.emptyList(); + } + + @Override + public boolean isShutdown() + { + return shutdown; + } + + @Override + public boolean isTerminated() + { + return shutdown; + } + + @Override + public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException + { + return shutdown; + } + + @Override + public void execute(Runnable command) + { + checkNotShutdown(); + queue.add(new Item(nowWithJitter(), SimulatedExecutorFactory.this.seq++, taskFor(command))); + } + + protected void checkNotShutdown() + { + if (isShutdown()) + throw new RejectedExecutionException("Shutdown"); + } + + protected long nowWithJitter() + { + return nanoTime() + jitterNanos.getAsLong(); + } + } + + private class OrderedExecutorService extends UnorderedExecutorService + { + private final Queue pending = new LinkedList<>(); + + @Override + public void execute(Runnable command) + { + checkNotShutdown(); + boolean wasEmpty = pending.isEmpty(); + Item task = new Item(nowWithJitter(), SimulatedExecutorFactory.this.seq++, taskFor(command)); + pending.add(task); + if (wasEmpty) + runNextTask(); + } + + private void runNextTask() + { + Item next = pending.peek(); + if (next == null) + return; + + next.action.addCallback((s, f) -> afterExecution()); + queue.add(next); + } + + private void afterExecution() + { + pending.poll(); + runNextTask(); + } + } + + private class UnorderedScheduledExecutorService extends UnorderedExecutorService implements ScheduledExecutorService + { + private class ScheduledFuture extends FutureTask implements java.util.concurrent.ScheduledFuture + { + private final long sequenceNumber; + private final long periodNanos; + private long nextExecuteAtNanos; + + ScheduledFuture(long sequenceNumber, long initialDelay, long value, TimeUnit unit, Callable call) + { + super(call); + this.sequenceNumber = sequenceNumber; + periodNanos = unit.toNanos(value); + nextExecuteAtNanos = triggerTimeNanos(initialDelay, unit); + } + + private long triggerTimeNanos(long delay, TimeUnit unit) + { + long delayNanos = unit.toNanos(delay < 0 ? 0 : delay); + return nanoTime() + delayNanos; + } + + @Override + public long getDelay(TimeUnit unit) + { + return unit.convert(nextExecuteAtNanos - nanoTime(), TimeUnit.NANOSECONDS); + } + + @Override + public int compareTo(Delayed other) + { + if (other == this) // compare zero if same object + return 0; + if (other instanceof ScheduledFuture) + { + ScheduledFuture x = (ScheduledFuture) other; + long diff = nextExecuteAtNanos - x.nextExecuteAtNanos; + if (diff < 0) + return -1; + else if (diff > 0) + return 1; + else if (sequenceNumber < x.sequenceNumber) + return -1; + else + return 1; + } + long diff = getDelay(NANOSECONDS) - other.getDelay(NANOSECONDS); + return (diff < 0) ? -1 : (diff > 0) ? 1 : 0; + } + + @Override + public void run() + { + boolean periodic = periodNanos != 0; + if (!periodic) + { + super.run(); + } + else + { + if (isCancelled()) + return; + // run without setting the result + try + { + call(); + long nowNanos = nanoTime(); + if (periodNanos > 0) + { + // scheduleAtFixedRate + nextExecuteAtNanos += periodNanos; + } + else + { + // scheduleWithFixedDelay + nextExecuteAtNanos = nowNanos + (-periodNanos); + } + long delayNanos = nextExecuteAtNanos - nowNanos; + if (delayNanos < 0) + delayNanos = 0; + schedule(this, delayNanos, NANOSECONDS); + } + catch (Throwable t) + { + tryFailure(t); + } + } + } + } + + @Override + public ScheduledFuture schedule(Runnable command, long delay, TimeUnit unit) + { + return schedule(Executors.callable(command), delay, unit); + } + + @Override + public ScheduledFuture schedule(Callable callable, long delay, TimeUnit unit) + { + checkNotShutdown(); + ScheduledFuture task = new ScheduledFuture<>(seq++, delay, 0, NANOSECONDS, callable); + queue.add(new Item(nowWithJitter() + unit.toNanos(delay), task.sequenceNumber, task)); + return task; + } + + @Override + public ScheduledFuture scheduleAtFixedRate(Runnable command, long initialDelay, long period, TimeUnit unit) + { + checkNotShutdown(); + ScheduledFuture task = new ScheduledFuture<>(seq++, initialDelay, period, unit, Executors.callable(command)); + repeatedTasks++; + task.addCallback((s, f) -> repeatedTasks--); + queue.add(new Item(nowWithJitter() + unit.toNanos(initialDelay), task.sequenceNumber, task)); + return task; + } + + @Override + public ScheduledFuture scheduleWithFixedDelay(Runnable command, long initialDelay, long delay, TimeUnit unit) + { + checkNotShutdown(); + ScheduledFuture task = new ScheduledFuture<>(seq++, initialDelay, -delay, unit, Executors.callable(command)); + repeatedTasks++; + task.addCallback((s, f) -> repeatedTasks--); + queue.add(new Item(nowWithJitter() + unit.toNanos(initialDelay), task.sequenceNumber, task)); + return task; + } + } +} diff --git a/test/unit/org/apache/cassandra/config/DatabaseDescriptorRefTest.java b/test/unit/org/apache/cassandra/config/DatabaseDescriptorRefTest.java index b1e9c1ec0d..b4da790360 100644 --- a/test/unit/org/apache/cassandra/config/DatabaseDescriptorRefTest.java +++ b/test/unit/org/apache/cassandra/config/DatabaseDescriptorRefTest.java @@ -140,6 +140,11 @@ public class DatabaseDescriptorRefTest "org.apache.cassandra.config.GuardrailsOptions$ConsistencyLevels", "org.apache.cassandra.config.GuardrailsOptions$TableProperties", "org.apache.cassandra.config.ParameterizedClass", + "org.apache.cassandra.config.RepairConfig", + "org.apache.cassandra.config.RepairRetrySpec", + "org.apache.cassandra.config.RetrySpec", + "org.apache.cassandra.config.RetrySpec$MaxAttempt", + "org.apache.cassandra.config.RetrySpec$Type", "org.apache.cassandra.config.ReplicaFilteringProtectionOptions", "org.apache.cassandra.config.StartupChecksOptions", "org.apache.cassandra.config.SubnetGroups", diff --git a/test/unit/org/apache/cassandra/config/UnitConfigOverride.java b/test/unit/org/apache/cassandra/config/UnitConfigOverride.java new file mode 100644 index 0000000000..bd6294f0bd --- /dev/null +++ b/test/unit/org/apache/cassandra/config/UnitConfigOverride.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.config; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.util.Arrays; +import java.util.List; +import javax.annotation.Nullable; + +public class UnitConfigOverride +{ + private enum ConfigType + { + CDC("cassandra.yaml", "cdc.yaml"), + COMPRESSION("cassandra.yaml", "commitlog_compression_LZ4.yaml"), // not 100% correct, LZ4 could be changed at build time + OA("cassandra.yaml", "storage_compatibility_mode_none.yaml"), + SYSTEM_KEYSPACE_DIRECTORY("cassandra.yaml", "system_keyspaces_directory.yaml"), + TRIE("cassandra.yaml", "trie_memtable.yaml", "storage_compatibility_mode_none.yaml"); + + public final List configs; + + ConfigType(String... configs) + { + this.configs = Arrays.asList(configs); + } + } + + @Nullable + private static final ConfigType CONFIG_TYPE = null; + + @SuppressWarnings("ConstantConditions") + public static void maybeOverrideConfig() + { + // CONFIG_TYPE is meant to be changed by users while debugging, so this condition won't be false on every machine + if (CONFIG_TYPE != null) + setConfigType(CONFIG_TYPE); + } + + @SuppressWarnings("SameParameterValue") + private static void setConfigType(ConfigType type) + { + File confDir = new File("test/conf"); + try + { + File tmp = Files.createTempFile("cassandra-conf", ".yaml").toFile(); + tmp.deleteOnExit(); + for (String name : type.configs) + { + try (FileOutputStream out = new FileOutputStream(tmp, true); + InputStream in = new FileInputStream(new File(confDir, name))) + { + in.transferTo(out); + } + } + CassandraRelevantProperties.CASSANDRA_CONFIG.setString(tmp.toURI().toString()); + } catch (IOException e) + { + throw new UncheckedIOException(e); + } + } +} diff --git a/test/unit/org/apache/cassandra/config/YamlConfigurationLoaderTest.java b/test/unit/org/apache/cassandra/config/YamlConfigurationLoaderTest.java index efa47b05d7..11833783a3 100644 --- a/test/unit/org/apache/cassandra/config/YamlConfigurationLoaderTest.java +++ b/test/unit/org/apache/cassandra/config/YamlConfigurationLoaderTest.java @@ -49,6 +49,51 @@ import static org.junit.Assert.assertTrue; public class YamlConfigurationLoaderTest { + @Test + public void repairRetryEmpty() + { + RepairRetrySpec repair_retries = loadRepairRetry(ImmutableMap.of()); + // repair is empty + assertThat(repair_retries.isEnabled()).isFalse(); + assertThat(repair_retries.isMerkleTreeRetriesEnabled()).isFalse(); + } + + @Test + public void repairRetryInheritance() + { + RepairRetrySpec repair_retries = loadRepairRetry(ImmutableMap.of("max_attempts", "3")); + assertThat(repair_retries.isEnabled()).isTrue(); + assertThat(repair_retries.getMaxAttempts()).isEqualTo(3); + RetrySpec spec = repair_retries.getMerkleTreeResponseSpec(); + assertThat(spec.isEnabled()).isTrue(); + assertThat(spec.getMaxAttempts()).isEqualTo(3); + } + + @Test + public void repairRetryOverride() + { + RepairRetrySpec repair_retries = loadRepairRetry(ImmutableMap.of( + "merkle_tree_response", ImmutableMap.of("max_attempts", 10, + "base_sleep_time", "1s", + "max_sleep_time", "10s") + )); + assertThat(repair_retries.isEnabled()).isFalse(); + assertThat(repair_retries.getMaxAttempts()).isNull(); + assertThat(repair_retries.baseSleepTime).isEqualTo(RetrySpec.DEFAULT_BASE_SLEEP); + assertThat(repair_retries.maxSleepTime).isEqualTo(RetrySpec.DEFAULT_MAX_SLEEP); + + RetrySpec spec = repair_retries.getMerkleTreeResponseSpec(); + assertThat(spec.isEnabled()).isTrue(); + assertThat(spec.maxAttempts).isEqualTo(10); + assertThat(spec.baseSleepTime).isEqualTo(RetrySpec.DEFAULT_MAX_SLEEP); + assertThat(spec.maxSleepTime).isEqualTo(new DurationSpec.LongMillisecondsBound("10s")); + } + + private static RepairRetrySpec loadRepairRetry(Map map) + { + return YamlConfigurationLoader.fromMap(ImmutableMap.of("repair", ImmutableMap.of("retries", map)), true, Config.class).repair.retries; + } + @Test public void validateTypes() { diff --git a/test/unit/org/apache/cassandra/db/ReadCommandTest.java b/test/unit/org/apache/cassandra/db/ReadCommandTest.java index f1efe97722..46eb5e7078 100644 --- a/test/unit/org/apache/cassandra/db/ReadCommandTest.java +++ b/test/unit/org/apache/cassandra/db/ReadCommandTest.java @@ -1332,14 +1332,14 @@ public class ReadCommandTest // check for sessions which have exceeded timeout and been purged Range range = new Range<>(cfs.metadata().partitioner.getMinimumToken(), cfs.metadata().partitioner.getRandomToken()); - ActiveRepairService.instance.registerParentRepairSession(pendingSession, - REPAIR_COORDINATOR, - Lists.newArrayList(cfs), - Sets.newHashSet(range), - true, - repairedAt, - true, - PreviewKind.NONE); + ActiveRepairService.instance().registerParentRepairSession(pendingSession, + REPAIR_COORDINATOR, + Lists.newArrayList(cfs), + Sets.newHashSet(range), + true, + repairedAt, + true, + PreviewKind.NONE); LocalSessionAccessor.prepareUnsafe(pendingSession, null, Sets.newHashSet(REPAIR_COORDINATOR)); } diff --git a/test/unit/org/apache/cassandra/db/compaction/AbstractPendingRepairTest.java b/test/unit/org/apache/cassandra/db/compaction/AbstractPendingRepairTest.java index b0bb23cc64..ff6fd203de 100644 --- a/test/unit/org/apache/cassandra/db/compaction/AbstractPendingRepairTest.java +++ b/test/unit/org/apache/cassandra/db/compaction/AbstractPendingRepairTest.java @@ -57,7 +57,7 @@ public class AbstractPendingRepairTest extends AbstractRepairTest public static void setupClass() { SchemaLoader.prepareServer(); - ARS = ActiveRepairService.instance; + ARS = ActiveRepairService.instance(); LocalSessionAccessor.startup(); // cutoff messaging service diff --git a/test/unit/org/apache/cassandra/db/compaction/AntiCompactionTest.java b/test/unit/org/apache/cassandra/db/compaction/AntiCompactionTest.java index d3112d8ad4..97da2a4076 100644 --- a/test/unit/org/apache/cassandra/db/compaction/AntiCompactionTest.java +++ b/test/unit/org/apache/cassandra/db/compaction/AntiCompactionTest.java @@ -111,11 +111,11 @@ public class AntiCompactionTest private void registerParentRepairSession(TimeUUID sessionID, Iterable> ranges, long repairedAt, TimeUUID pendingRepair) throws IOException { - ActiveRepairService.instance.registerParentRepairSession(sessionID, - InetAddressAndPort.getByName("10.0.0.1"), - Lists.newArrayList(cfs), ImmutableSet.copyOf(ranges), - pendingRepair != null || repairedAt != UNREPAIRED_SSTABLE, - repairedAt, true, PreviewKind.NONE); + ActiveRepairService.instance().registerParentRepairSession(sessionID, + InetAddressAndPort.getByName("10.0.0.1"), + Lists.newArrayList(cfs), ImmutableSet.copyOf(ranges), + pendingRepair != null || repairedAt != UNREPAIRED_SSTABLE, + repairedAt, true, PreviewKind.NONE); } private static RangesAtEndpoint atEndpoint(Collection> full, Collection> trans) diff --git a/test/unit/org/apache/cassandra/db/compaction/CancelCompactionsTest.java b/test/unit/org/apache/cassandra/db/compaction/CancelCompactionsTest.java index ebddac7bf3..ba054eac4c 100644 --- a/test/unit/org/apache/cassandra/db/compaction/CancelCompactionsTest.java +++ b/test/unit/org/apache/cassandra/db/compaction/CancelCompactionsTest.java @@ -261,7 +261,7 @@ public class CancelCompactionsTest extends CQLTester Range range = new Range<>(token(-1), token(49)); TimeUUID prsid = nextTimeUUID(); - ActiveRepairService.instance.registerParentRepairSession(prsid, InetAddressAndPort.getLocalHost(), Collections.singletonList(cfs), Collections.singleton(range), true, 1, true, PreviewKind.NONE); + ActiveRepairService.instance().registerParentRepairSession(prsid, InetAddressAndPort.getLocalHost(), Collections.singletonList(cfs), Collections.singleton(range), true, 1, true, PreviewKind.NONE); InetAddressAndPort local = FBUtilities.getBroadcastAddressAndPort(); RangesAtEndpoint rae = RangesAtEndpoint.builder(local).add(new Replica(local, range, true)).build(); diff --git a/test/unit/org/apache/cassandra/db/compaction/CompactionStrategyManagerPendingRepairTest.java b/test/unit/org/apache/cassandra/db/compaction/CompactionStrategyManagerPendingRepairTest.java index 76bbd88e3f..62a1258043 100644 --- a/test/unit/org/apache/cassandra/db/compaction/CompactionStrategyManagerPendingRepairTest.java +++ b/test/unit/org/apache/cassandra/db/compaction/CompactionStrategyManagerPendingRepairTest.java @@ -262,7 +262,7 @@ public class CompactionStrategyManagerPendingRepairTest extends AbstractPendingR Assert.assertFalse(hasTransientStrategiesFor(repairID)); // sstable should have pendingRepair cleared, and repairedAt set correctly - long expectedRepairedAt = ActiveRepairService.instance.getParentRepairSession(repairID).repairedAt; + long expectedRepairedAt = ActiveRepairService.instance().getParentRepairSession(repairID).repairedAt; Assert.assertFalse(sstable.isPendingRepair()); Assert.assertTrue(sstable.isRepaired()); Assert.assertEquals(expectedRepairedAt, sstable.getSSTableMetadata().repairedAt); diff --git a/test/unit/org/apache/cassandra/db/compaction/LeveledCompactionStrategyTest.java b/test/unit/org/apache/cassandra/db/compaction/LeveledCompactionStrategyTest.java index e9e2d42ab4..d56003ae2a 100644 --- a/test/unit/org/apache/cassandra/db/compaction/LeveledCompactionStrategyTest.java +++ b/test/unit/org/apache/cassandra/db/compaction/LeveledCompactionStrategyTest.java @@ -66,6 +66,7 @@ import org.apache.cassandra.schema.KeyspaceParams; import org.apache.cassandra.schema.MockSchema; import org.apache.cassandra.service.ActiveRepairService; import org.apache.cassandra.streaming.PreviewKind; +import org.apache.cassandra.utils.Clock; import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.Pair; import org.apache.cassandra.utils.TimeUUID; @@ -208,16 +209,16 @@ public class LeveledCompactionStrategyTest Range range = new Range<>(Util.token(""), Util.token("")); long gcBefore = keyspace.getColumnFamilyStore(CF_STANDARDDLEVELED).gcBefore(FBUtilities.nowInSeconds()); TimeUUID parentRepSession = nextTimeUUID(); - ActiveRepairService.instance.registerParentRepairSession(parentRepSession, - FBUtilities.getBroadcastAddressAndPort(), - Arrays.asList(cfs), - Arrays.asList(range), - false, - ActiveRepairService.UNREPAIRED_SSTABLE, - true, - PreviewKind.NONE); + ActiveRepairService.instance().registerParentRepairSession(parentRepSession, + FBUtilities.getBroadcastAddressAndPort(), + Arrays.asList(cfs), + Arrays.asList(range), + false, + ActiveRepairService.UNREPAIRED_SSTABLE, + true, + PreviewKind.NONE); RepairJobDesc desc = new RepairJobDesc(parentRepSession, nextTimeUUID(), KEYSPACE1, CF_STANDARDDLEVELED, Arrays.asList(range)); - Validator validator = new Validator(new ValidationState(desc, FBUtilities.getBroadcastAddressAndPort()), gcBefore, PreviewKind.NONE); + Validator validator = new Validator(new ValidationState(Clock.Global.clock(), desc, FBUtilities.getBroadcastAddressAndPort()), gcBefore, PreviewKind.NONE); ValidationManager.instance.submitValidation(cfs, validator).get(); } diff --git a/test/unit/org/apache/cassandra/db/repair/AbstractPendingAntiCompactionTest.java b/test/unit/org/apache/cassandra/db/repair/AbstractPendingAntiCompactionTest.java index 79be4c8643..c96b85ff77 100644 --- a/test/unit/org/apache/cassandra/db/repair/AbstractPendingAntiCompactionTest.java +++ b/test/unit/org/apache/cassandra/db/repair/AbstractPendingAntiCompactionTest.java @@ -76,7 +76,7 @@ public abstract class AbstractPendingAntiCompactionTest { SchemaLoader.prepareServer(); local = InetAddressAndPort.getByName("127.0.0.1"); - ActiveRepairService.instance.consistent.local.start(); + ActiveRepairService.instance().consistent.local.start(); } @Before diff --git a/test/unit/org/apache/cassandra/db/repair/CompactionManagerGetSSTablesForValidationTest.java b/test/unit/org/apache/cassandra/db/repair/CompactionManagerGetSSTablesForValidationTest.java index bcf75827b0..011db44d09 100644 --- a/test/unit/org/apache/cassandra/db/repair/CompactionManagerGetSSTablesForValidationTest.java +++ b/test/unit/org/apache/cassandra/db/repair/CompactionManagerGetSSTablesForValidationTest.java @@ -30,6 +30,7 @@ import org.junit.Test; import org.apache.cassandra.SchemaLoader; import org.apache.cassandra.Util; +import org.apache.cassandra.repair.SharedContext; import org.apache.cassandra.cql3.statements.schema.CreateTableStatement; import org.apache.cassandra.repair.state.ValidationState; import org.apache.cassandra.schema.TableMetadata; @@ -46,6 +47,7 @@ import org.apache.cassandra.repair.RepairJobDesc; import org.apache.cassandra.repair.Validator; import org.apache.cassandra.schema.KeyspaceParams; import org.apache.cassandra.service.ActiveRepairService; +import org.apache.cassandra.utils.Clock; import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.TimeUUID; @@ -105,14 +107,14 @@ public class CompactionManagerGetSSTablesForValidationTest { sessionID = nextTimeUUID(); Range range = new Range<>(MT, MT); - ActiveRepairService.instance.registerParentRepairSession(sessionID, - coordinator, - Lists.newArrayList(cfs), - Sets.newHashSet(range), - incremental, - incremental ? System.currentTimeMillis() : ActiveRepairService.UNREPAIRED_SSTABLE, - true, - PreviewKind.NONE); + ActiveRepairService.instance().registerParentRepairSession(sessionID, + coordinator, + Lists.newArrayList(cfs), + Sets.newHashSet(range), + incremental, + incremental ? System.currentTimeMillis() : ActiveRepairService.UNREPAIRED_SSTABLE, + true, + PreviewKind.NONE); desc = new RepairJobDesc(sessionID, nextTimeUUID(), ks, tbl, singleton(range)); } @@ -141,8 +143,8 @@ public class CompactionManagerGetSSTablesForValidationTest modifySSTables(); // get sstables for repair - Validator validator = new Validator(new ValidationState(desc, coordinator), FBUtilities.nowInSeconds(), true, PreviewKind.NONE); - Set sstables = Sets.newHashSet(getSSTablesToValidate(cfs, validator.desc.ranges, validator.desc.parentSessionId, validator.isIncremental)); + Validator validator = new Validator(new ValidationState(Clock.Global.clock(), desc, coordinator), FBUtilities.nowInSeconds(), true, PreviewKind.NONE); + Set sstables = Sets.newHashSet(getSSTablesToValidate(cfs, SharedContext.Global.instance, validator.desc.ranges, validator.desc.parentSessionId, validator.isIncremental)); Assert.assertNotNull(sstables); Assert.assertEquals(1, sstables.size()); Assert.assertTrue(sstables.contains(pendingRepair)); @@ -156,8 +158,8 @@ public class CompactionManagerGetSSTablesForValidationTest modifySSTables(); // get sstables for repair - Validator validator = new Validator(new ValidationState(desc, coordinator), FBUtilities.nowInSeconds(), false, PreviewKind.NONE); - Set sstables = Sets.newHashSet(getSSTablesToValidate(cfs, validator.desc.ranges, validator.desc.parentSessionId, validator.isIncremental)); + Validator validator = new Validator(new ValidationState(Clock.Global.clock(), desc, coordinator), FBUtilities.nowInSeconds(), false, PreviewKind.NONE); + Set sstables = Sets.newHashSet(getSSTablesToValidate(cfs, SharedContext.Global.instance, validator.desc.ranges, validator.desc.parentSessionId, validator.isIncremental)); Assert.assertNotNull(sstables); Assert.assertEquals(2, sstables.size()); Assert.assertTrue(sstables.contains(pendingRepair)); @@ -172,8 +174,8 @@ public class CompactionManagerGetSSTablesForValidationTest modifySSTables(); // get sstables for repair - Validator validator = new Validator(new ValidationState(desc, coordinator), FBUtilities.nowInSeconds(), false, PreviewKind.NONE); - Set sstables = Sets.newHashSet(getSSTablesToValidate(cfs, validator.desc.ranges, validator.desc.parentSessionId, validator.isIncremental)); + Validator validator = new Validator(new ValidationState(Clock.Global.clock(), desc, coordinator), FBUtilities.nowInSeconds(), false, PreviewKind.NONE); + Set sstables = Sets.newHashSet(getSSTablesToValidate(cfs, SharedContext.Global.instance, validator.desc.ranges, validator.desc.parentSessionId, validator.isIncremental)); Assert.assertNotNull(sstables); Assert.assertEquals(3, sstables.size()); Assert.assertTrue(sstables.contains(pendingRepair)); diff --git a/test/unit/org/apache/cassandra/db/repair/PendingAntiCompactionTest.java b/test/unit/org/apache/cassandra/db/repair/PendingAntiCompactionTest.java index d4a2b991d4..4960bfd188 100644 --- a/test/unit/org/apache/cassandra/db/repair/PendingAntiCompactionTest.java +++ b/test/unit/org/apache/cassandra/db/repair/PendingAntiCompactionTest.java @@ -142,7 +142,7 @@ public class PendingAntiCompactionTest extends AbstractPendingAntiCompactionTest // create a session so the anti compaction can fine it TimeUUID sessionID = nextTimeUUID(); - ActiveRepairService.instance.registerParentRepairSession(sessionID, InetAddressAndPort.getLocalHost(), tables, ranges, true, 1, true, PreviewKind.NONE); + ActiveRepairService.instance().registerParentRepairSession(sessionID, InetAddressAndPort.getLocalHost(), tables, ranges, true, 1, true, PreviewKind.NONE); PendingAntiCompaction pac; ExecutorService executor = Executors.newSingleThreadExecutor(); @@ -370,13 +370,13 @@ public class PendingAntiCompactionTest extends AbstractPendingAntiCompactionTest PendingAntiCompaction.AcquisitionCallable acquisitionCallable = new PendingAntiCompaction.AcquisitionCallable(cfs, FULL_RANGE, nextTimeUUID(), 0, 0); PendingAntiCompaction.AcquireResult result = acquisitionCallable.call(); TimeUUID sessionID = nextTimeUUID(); - ActiveRepairService.instance.registerParentRepairSession(sessionID, - InetAddressAndPort.getByName("127.0.0.1"), - Lists.newArrayList(cfs), - FULL_RANGE, - true,0, - true, - PreviewKind.NONE); + ActiveRepairService.instance().registerParentRepairSession(sessionID, + InetAddressAndPort.getByName("127.0.0.1"), + Lists.newArrayList(cfs), + FULL_RANGE, + true, 0, + true, + PreviewKind.NONE); CompactionManager.instance.performAnticompaction(result.cfs, atEndpoint(FULL_RANGE, NO_RANGES), result.refs, result.txn, sessionID, () -> false); } @@ -390,13 +390,13 @@ public class PendingAntiCompactionTest extends AbstractPendingAntiCompactionTest PendingAntiCompaction.AcquisitionCallable acquisitionCallable = new PendingAntiCompaction.AcquisitionCallable(cfs, FULL_RANGE, nextTimeUUID(), 0, 0); PendingAntiCompaction.AcquireResult result = acquisitionCallable.call(); TimeUUID sessionID = nextTimeUUID(); - ActiveRepairService.instance.registerParentRepairSession(sessionID, - InetAddressAndPort.getByName("127.0.0.1"), - Lists.newArrayList(cfs), - FULL_RANGE, - true,0, - true, - PreviewKind.NONE); + ActiveRepairService.instance().registerParentRepairSession(sessionID, + InetAddressAndPort.getByName("127.0.0.1"), + Lists.newArrayList(cfs), + FULL_RANGE, + true, 0, + true, + PreviewKind.NONE); // attempt to anti-compact the sstable in half SSTableReader sstable = Iterables.getOnlyElement(cfs.getLiveSSTables()); diff --git a/test/unit/org/apache/cassandra/db/streaming/CassandraStreamManagerTest.java b/test/unit/org/apache/cassandra/db/streaming/CassandraStreamManagerTest.java index eba574ebaa..41dcdcb66b 100644 --- a/test/unit/org/apache/cassandra/db/streaming/CassandraStreamManagerTest.java +++ b/test/unit/org/apache/cassandra/db/streaming/CassandraStreamManagerTest.java @@ -261,10 +261,10 @@ public class CassandraStreamManagerTest @Test public void checkAvailableDiskSpaceAndCompactionsFailing() { - int threshold = ActiveRepairService.instance.getRepairPendingCompactionRejectThreshold(); - ActiveRepairService.instance.setRepairPendingCompactionRejectThreshold(1); + int threshold = ActiveRepairService.instance().getRepairPendingCompactionRejectThreshold(); + ActiveRepairService.instance().setRepairPendingCompactionRejectThreshold(1); assertFalse(StreamSession.checkAvailableDiskSpaceAndCompactions(createSummaries(), nextTimeUUID(), null, false)); - ActiveRepairService.instance.setRepairPendingCompactionRejectThreshold(threshold); + ActiveRepairService.instance().setRepairPendingCompactionRejectThreshold(threshold); } private Collection createSummaries() diff --git a/test/unit/org/apache/cassandra/db/virtual/LocalRepairTablesTest.java b/test/unit/org/apache/cassandra/db/virtual/LocalRepairTablesTest.java index 2a6da1ac69..aec6c1574e 100644 --- a/test/unit/org/apache/cassandra/db/virtual/LocalRepairTablesTest.java +++ b/test/unit/org/apache/cassandra/db/virtual/LocalRepairTablesTest.java @@ -39,7 +39,7 @@ import org.apache.cassandra.dht.Token; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.repair.CommonRange; import org.apache.cassandra.repair.RepairJobDesc; -import org.apache.cassandra.repair.RepairRunnable; +import org.apache.cassandra.repair.RepairCoordinator; import org.apache.cassandra.repair.messages.PrepareMessage; import org.apache.cassandra.repair.messages.RepairOption; import org.apache.cassandra.repair.state.Completable; @@ -52,6 +52,7 @@ import org.apache.cassandra.repair.state.ValidationState; import org.apache.cassandra.schema.Schema; import org.apache.cassandra.service.ActiveRepairService; import org.apache.cassandra.streaming.PreviewKind; +import org.apache.cassandra.utils.Clock; import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.TimeUUID; @@ -75,7 +76,7 @@ public class LocalRepairTablesTest extends CQLTester @Before public void cleanupRepairs() { - ActiveRepairService.instance.clearLocalRepairState(); + ActiveRepairService.instance().clearLocalRepairState(); } @Test @@ -90,7 +91,7 @@ public class LocalRepairTablesTest extends CQLTester assertState("repairs", state, CoordinatorState.State.SETUP); List tables = Collections.singletonList(table()); - RepairRunnable.NeighborsAndRanges neighbors = neighbors(); + RepairCoordinator.NeighborsAndRanges neighbors = neighbors(); state.phase.start(tables, neighbors); assertState("repairs", state, CoordinatorState.State.START); List> expectedRanges = neighbors.commonRanges.stream().map(a -> a.ranges.stream().map(Object::toString).collect(Collectors.toList())).collect(Collectors.toList()); @@ -265,9 +266,9 @@ public class LocalRepairTablesTest extends CQLTester return Schema.instance.getColumnFamilyStoreInstance(Schema.instance.getTableMetadata(ks, name).id); } - private static RepairRunnable.NeighborsAndRanges neighbors() + private static RepairCoordinator.NeighborsAndRanges neighbors() { - return new RepairRunnable.NeighborsAndRanges(false, ADDRESSES, ImmutableList.of(COMMON_RANGE)); + return new RepairCoordinator.NeighborsAndRanges(false, ADDRESSES, ImmutableList.of(COMMON_RANGE)); } private static Range range(long a, long b) @@ -290,15 +291,15 @@ public class LocalRepairTablesTest extends CQLTester private static CoordinatorState coordinator() { RepairOption options = RepairOption.parse(Collections.emptyMap(), DatabaseDescriptor.getPartitioner()); - CoordinatorState state = new CoordinatorState(0, "test", options); - ActiveRepairService.instance.register(state); + CoordinatorState state = new CoordinatorState(Clock.Global.clock(), 0, "test", options); + ActiveRepairService.instance().register(state); return state; } private static SessionState session() { CoordinatorState parent = coordinator(); - SessionState state = new SessionState(parent.id, REPAIR_KS, new String[]{ REPAIR_TABLE }, COMMON_RANGE); + SessionState state = new SessionState(Clock.Global.clock(), parent.id, REPAIR_KS, new String[]{ REPAIR_TABLE }, COMMON_RANGE); parent.register(state); return state; } @@ -306,7 +307,7 @@ public class LocalRepairTablesTest extends CQLTester private static JobState job() { SessionState session = session(); - JobState state = new JobState(new RepairJobDesc(session.parentRepairSession, session.id, session.keyspace, session.cfnames[0], session.commonRange.ranges), session.commonRange.endpoints); + JobState state = new JobState(Clock.Global.clock(), new RepairJobDesc(session.parentRepairSession, session.id, session.keyspace, session.cfnames[0], session.commonRange.ranges), session.commonRange.endpoints); session.register(state); return state; } @@ -315,7 +316,7 @@ public class LocalRepairTablesTest extends CQLTester { JobState job = job(); // job isn't needed but makes getting the descriptor easier ParticipateState participate = participate(); - ValidationState state = new ValidationState(job.desc, ADDRESSES.stream().findFirst().get()); + ValidationState state = new ValidationState(Clock.Global.clock(), job.desc, ADDRESSES.stream().findFirst().get()); participate.register(state); return state; } @@ -323,8 +324,8 @@ public class LocalRepairTablesTest extends CQLTester private ParticipateState participate() { List> ranges = Arrays.asList(new Range<>(new Murmur3Partitioner.LongToken(0), new Murmur3Partitioner.LongToken(42))); - ParticipateState state = new ParticipateState(FBUtilities.getBroadcastAddressAndPort(), new PrepareMessage(TimeUUID.Generator.nextTimeUUID(), Collections.emptyList(), ranges, true, 42, true, PreviewKind.ALL)); - ActiveRepairService.instance.register(state); + ParticipateState state = new ParticipateState(Clock.Global.clock(), FBUtilities.getBroadcastAddressAndPort(), new PrepareMessage(TimeUUID.Generator.nextTimeUUID(), Collections.emptyList(), ranges, true, 42, true, PreviewKind.ALL)); + ActiveRepairService.instance().register(state); return state; } diff --git a/test/unit/org/apache/cassandra/index/sai/functional/CompactionTest.java b/test/unit/org/apache/cassandra/index/sai/functional/CompactionTest.java index e13cca5108..da9a051f4f 100644 --- a/test/unit/org/apache/cassandra/index/sai/functional/CompactionTest.java +++ b/test/unit/org/apache/cassandra/index/sai/functional/CompactionTest.java @@ -96,14 +96,14 @@ public class CompactionTest extends SAITester { InetAddressAndPort endpoint = InetAddressAndPort.getByName("10.0.0.1"); TimeUUID parentRepairSession = TimeUUID.Generator.nextTimeUUID(); - ActiveRepairService.instance.registerParentRepairSession(parentRepairSession, - endpoint, - Lists.newArrayList(cfs), - Collections.singleton(range), - true, - 1000, - false, - PreviewKind.NONE); + ActiveRepairService.instance().registerParentRepairSession(parentRepairSession, + endpoint, + Lists.newArrayList(cfs), + Collections.singleton(range), + true, + 1000, + false, + PreviewKind.NONE); RangesAtEndpoint replicas = RangesAtEndpoint.builder(endpoint).add(Replica.fullReplica(endpoint, range)).build(); CompactionManager.instance.performAnticompaction(cfs, replicas, refs, txn, parentRepairSession, () -> false); } diff --git a/test/unit/org/apache/cassandra/repair/AbstractRepairTest.java b/test/unit/org/apache/cassandra/repair/AbstractRepairTest.java index eb4ca9f445..a4c1304bf3 100644 --- a/test/unit/org/apache/cassandra/repair/AbstractRepairTest.java +++ b/test/unit/org/apache/cassandra/repair/AbstractRepairTest.java @@ -88,14 +88,14 @@ public abstract class AbstractRepairTest TimeUUID sessionId = nextTimeUUID(); long repairedAt = isIncremental ? System.currentTimeMillis() : ActiveRepairService.UNREPAIRED_SSTABLE; - ActiveRepairService.instance.registerParentRepairSession(sessionId, - COORDINATOR, - Lists.newArrayList(cfs), - Sets.newHashSet(RANGE1, RANGE2, RANGE3), - isIncremental, - repairedAt, - isGlobal, - PreviewKind.NONE); + ActiveRepairService.instance().registerParentRepairSession(sessionId, + COORDINATOR, + Lists.newArrayList(cfs), + Sets.newHashSet(RANGE1, RANGE2, RANGE3), + isIncremental, + repairedAt, + isGlobal, + PreviewKind.NONE); return sessionId; } } diff --git a/test/unit/org/apache/cassandra/repair/ConcurrentIrWithPreviewFuzzTest.java b/test/unit/org/apache/cassandra/repair/ConcurrentIrWithPreviewFuzzTest.java new file mode 100644 index 0000000000..e2c3fb6893 --- /dev/null +++ b/test/unit/org/apache/cassandra/repair/ConcurrentIrWithPreviewFuzzTest.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.repair; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; + +import org.junit.Test; + +import accord.utils.Gen; +import accord.utils.Gens; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.config.RetrySpec; +import org.apache.cassandra.repair.consistent.LocalSessions; +import org.apache.cassandra.repair.state.Completable; +import org.apache.cassandra.utils.Closeable; +import org.assertj.core.api.Assertions; + +import static accord.utils.Property.qt; + +public class ConcurrentIrWithPreviewFuzzTest extends FuzzTestBase +{ + @Test + public void concurrentIrWithPreview() + { + // to avoid unlucky timing issues, retry until success; given enough retries we should eventually become success + DatabaseDescriptor.getRepairRetrySpec().maxAttempts = new RetrySpec.MaxAttempt(Integer.MAX_VALUE); + qt().withPure(false).withExamples(2).withTimeout(Duration.ofMinutes(1)).check(rs -> { + Cluster cluster = new Cluster(rs); + enableMessageFaults(cluster); + + Gen coordinatorGen = Gens.pick(cluster.nodes.keySet()).map(cluster.nodes::get); + + List closeables = new ArrayList<>(); + for (int example = 0; example < 100; example++) + { + Cluster.Node irCoordinator = coordinatorGen.next(rs); + Cluster.Node previewCoordinator = coordinatorGen.next(rs); + RepairCoordinator ir = irCoordinator.repair(KEYSPACE, irOption(rs, irCoordinator, KEYSPACE, ignore -> TABLES)); + ir.run(); + RepairCoordinator preview = previewCoordinator.repair(KEYSPACE, previewOption(rs, previewCoordinator, KEYSPACE, ignore -> TABLES), false); + preview.run(); + + closeables.add(cluster.nodes.get(pickParticipant(rs, previewCoordinator, preview)).doValidation(ignore -> (cfs, validator) -> addMismatch(rs, cfs, validator))); + // cause a delay in validation to have more failing previews + closeables.add(cluster.nodes.get(pickParticipant(rs, previewCoordinator, preview)).doValidation(next -> (cfs, validator) -> { + if (validator.desc.parentSessionId.equals(preview.state.id)) + cluster.unorderedScheduled.schedule(() -> next.accept(cfs, validator), 1, TimeUnit.HOURS); + else next.acceptOrFail(cfs, validator); + })); + // make sure listeners don't leak + closeables.add(LocalSessions::unsafeClearListeners); + + cluster.processAll(); + + // IR will always pass, but preview is what may fail (if the coordinator is the same) + Assertions.assertThat(ir.state.getResult()).describedAs("Unexpected state: %s -> %s; example %d", ir.state, ir.state.getResult(), example).isEqualTo(Completable.Result.success(repairSuccessMessage(ir))); + + Assertions.assertThat(preview.state.getResult()).describedAs("Unexpected state: %s; example %d", preview.state, example).isNotNull(); + + if (irCoordinator == previewCoordinator) + { + Assertions.assertThat(preview.state.getResult().message).describedAs("Unexpected state: %s -> %s; example %d", preview.state, preview.state.getResult(), example).contains("failed with error An incremental repair with session id"); + } + else + { + assertSuccess(example, true, preview); + } + closeables.forEach(Closeable::close); + closeables.clear(); + } + }); + } +} diff --git a/test/unit/org/apache/cassandra/repair/FailedAckTest.java b/test/unit/org/apache/cassandra/repair/FailedAckTest.java new file mode 100644 index 0000000000..a8e1163bef --- /dev/null +++ b/test/unit/org/apache/cassandra/repair/FailedAckTest.java @@ -0,0 +1,136 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.repair; + +import java.util.ArrayList; +import java.util.List; + +import org.junit.Test; + +import accord.utils.Gen; +import accord.utils.Gens; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.config.RetrySpec; +import org.apache.cassandra.db.compaction.ICompactionManager; +import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.net.Message; +import org.apache.cassandra.net.Verb; +import org.apache.cassandra.repair.consistent.ConsistentSession; +import org.apache.cassandra.repair.consistent.LocalSession; +import org.apache.cassandra.repair.messages.ValidationRequest; +import org.apache.cassandra.repair.state.Completable; +import org.apache.cassandra.utils.Closeable; +import org.assertj.core.api.Assertions; +import org.mockito.Mockito; + +import static accord.utils.Property.qt; + +public class FailedAckTest extends FuzzTestBase +{ + private enum RepairStage + { PREPARE, VALIDATION } // SYNC doesn't have a good entry point for injecting the failure, if we win the race to register it, we accept right away + + @Test + public void failedAck() + { + DatabaseDescriptor.getRepairRetrySpec().maxAttempts = new RetrySpec.MaxAttempt(Integer.MAX_VALUE); + DatabaseDescriptor.setRepairPendingCompactionRejectThreshold(1); + Gen stageGen = Gens.enums().all(RepairStage.class); + qt().withPure(false).withExamples(10).check(rs -> { + Cluster cluster = new Cluster(rs); + enableMessageFaults(cluster); + + Gen coordinatorGen = Gens.pick(cluster.nodes.keySet()).map(cluster.nodes::get); + + List closeables = new ArrayList<>(); + for (int example = 0; example < 100; example++) + { + Cluster.Node coordinator = coordinatorGen.next(rs); + + RepairCoordinator repair = coordinator.repair(KEYSPACE, irOption(rs, coordinator, KEYSPACE, ignore -> TABLES), false); + repair.run(); + // make sure the failing node is not the coordinator, else messaging isn't used + InetAddressAndPort failingAddress = rs.pick(repair.state.getNeighborsAndRanges().participants); + Cluster.Node failingNode = cluster.nodes.get(failingAddress); + RepairStage stage = stageGen.next(rs); + switch (stage) + { + case PREPARE: + { + ICompactionManager cm = failingNode.compactionManager(); + Mockito.when(cm.getPendingTasks()).thenReturn(42); + closeables.add(() -> Mockito.when(cm.getPendingTasks()).thenReturn(0)); + } + break; + case VALIDATION: + { + cluster.addListener(new MessageListener() { + @Override + public void preHandle(Cluster.Node node, Message msg) + { + if (node != failingNode) return; + if (msg.verb() != Verb.VALIDATION_REQ) return; + ValidationRequest req = (ValidationRequest) msg.payload; + if (rs.nextBoolean()) + { + // fail ctx.repair().consistent.local.maybeSetRepairing(desc.parentSessionId); + LocalSession session = node.activeRepairService.consistent.local.getSession(req.desc.parentSessionId); + session.setState(ConsistentSession.State.FAILED); + } + else + { + // fail previewKind(desc.parentSessionId); + node.activeRepairService.removeParentRepairSession(req.desc.parentSessionId); + } + cluster.removeListener(this); + } + }); + } + break; + default: + throw new IllegalArgumentException("Unknown stage: " + stage); + } + + cluster.processAll(); + Assertions.assertThat(repair.state.getResult().kind).describedAs("Unexpected state: %s -> %s; example %d", repair.state, repair.state.getResult(), example).isEqualTo(Completable.Result.Kind.FAILURE); + switch (stage) + { + case PREPARE: + { + Assertions.assertThat(repair.state.getResult().message) + .describedAs("Unexpected state: %s -> %s; example %d", repair.state, repair.state.getResult(), example) + .contains("Got negative replies from endpoints [" + failingAddress + "]"); + } + break; + case VALIDATION: + { + Assertions.assertThat(repair.state.getResult().message) + .describedAs("Unexpected state: %s -> %s; example %d", repair.state, repair.state.getResult(), example) + .contains("Got VALIDATION_REQ failure from " + failingAddress + ": UNKNOWN"); + } + break; + default: + throw new IllegalArgumentException("Unknown stage: " + stage); + } + closeables.forEach(Closeable::close); + closeables.clear(); + } + }); + } +} diff --git a/test/unit/org/apache/cassandra/repair/FailingRepairFuzzTest.java b/test/unit/org/apache/cassandra/repair/FailingRepairFuzzTest.java new file mode 100644 index 0000000000..7c6a4601e1 --- /dev/null +++ b/test/unit/org/apache/cassandra/repair/FailingRepairFuzzTest.java @@ -0,0 +1,162 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.repair; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +import com.google.common.collect.ImmutableList; +import org.junit.Test; + +import accord.utils.Gen; +import accord.utils.Gens; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.config.RetrySpec; +import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.repair.state.Completable; +import org.apache.cassandra.streaming.StreamEventHandler; +import org.apache.cassandra.streaming.StreamState; +import org.apache.cassandra.utils.Closeable; +import org.assertj.core.api.AbstractStringAssert; +import org.assertj.core.api.Assertions; + +import static accord.utils.Property.qt; + +public class FailingRepairFuzzTest extends FuzzTestBase +{ + private enum RepairJobStage { VALIDATION, SYNC } + + @Test + public void failingRepair() + { + // to avoid unlucky timing issues, retry until success; given enough retries we should eventually become success + DatabaseDescriptor.getRepairRetrySpec().maxAttempts = new RetrySpec.MaxAttempt(Integer.MAX_VALUE); + Gen stageGen = Gens.enums().all(RepairJobStage.class); + qt().withPure(false).withExamples(10).check(rs -> { + Cluster cluster = new Cluster(rs); + enableMessageFaults(cluster); + + Gen coordinatorGen = Gens.pick(cluster.nodes.keySet()).map(cluster.nodes::get); + + List closeables = new ArrayList<>(); + for (int example = 0; example < 100; example++) + { + Cluster.Node coordinator = coordinatorGen.next(rs); + + RepairCoordinator repair = coordinator.repair(KEYSPACE, repairOption(rs, coordinator, KEYSPACE, TABLES), false); + repair.run(); + InetAddressAndPort failingAddress = pickParticipant(rs, coordinator, repair); + Cluster.Node failingNode = cluster.nodes.get(failingAddress); + RepairJobStage stage = stageGen.next(rs); + // because of local syncs reaching out to the failing address, a different address may actually be what failed + Set syncFailedAddresses = new HashSet<>(); + switch (stage) + { + case VALIDATION: + { + closeables.add(failingNode.doValidation((cfs, validator) -> { + long delayNanos = rs.nextLong(TimeUnit.MILLISECONDS.toNanos(5), TimeUnit.MINUTES.toNanos(1)); + cluster.unorderedScheduled.schedule(() -> validator.fail(new SimulatedFault("Validation failed")), delayNanos, TimeUnit.NANOSECONDS); + })); + } + break; + case SYNC: + { + closeables.add(failingNode.doValidation((cfs, validator) -> addMismatch(rs, cfs, validator))); + List addresses = ImmutableList.builder().add(coordinator.addressAndPort).addAll(repair.state.getNeighborsAndRanges().participants).build(); + for (InetAddressAndPort address : addresses) + { + closeables.add(cluster.nodes.get(address).doSync(plan -> { + long delayNanos = rs.nextLong(TimeUnit.SECONDS.toNanos(5), TimeUnit.MINUTES.toNanos(10)); + cluster.unorderedScheduled.schedule(() -> { + if (address == failingAddress || plan.getCoordinator().getPeers().contains(failingAddress)) + { + syncFailedAddresses.add(address); + SimulatedFault fault = new SimulatedFault("Sync failed"); + for (StreamEventHandler handler : plan.handlers()) + handler.onFailure(fault); + } + else + { + StreamState success = new StreamState(plan.planId(), plan.streamOperation(), Collections.emptySet()); + for (StreamEventHandler handler : plan.handlers()) + handler.onSuccess(success); + } + }, delayNanos, TimeUnit.NANOSECONDS); + return null; + })); + } + } + break; + default: + throw new IllegalArgumentException("Unknown stage: " + stage); + } + + cluster.processAll(); + Assertions.assertThat(repair.state.getResult().kind).describedAs("Unexpected state: %s -> %s; example %d", repair.state, repair.state.getResult(), example).isEqualTo(Completable.Result.Kind.FAILURE); + switch (stage) + { + case VALIDATION: + { + // Got VALIDATION_REQ failure from Ero2.MJ.N8kkw2.w3iFYdDw.HJiVYC32.mWb.b.xwi3tZ.s5k1l.mb.asTy_7QmQ.Q3.u.kjgh.GKjx.g1aKfkjB.YlyKg9.DyQszn7F.Ox2DMYIph.xlgH.EV.A9yEz2J.l6UHdC.C6FYLXE.J0CNHBH./[4905:e9f:2f00:e418:baac:b8d9:9ff9:6604]:33363: UNKNOWN" + Assertions.assertThat(repair.state.getResult().message) + .describedAs("Unexpected state: %s -> %s; example %d", repair.state, repair.state.getResult(), example) + // ValidationResponse with null tree seen + .containsAnyOf("Validation failed in " + failingAddress, + // ack was dropped and on retry the participate detected dup so rejected as the task failed + "Got VALIDATION_REQ failure from " + failingAddress + ": UNKNOWN"); + } + break; + case SYNC: + AbstractStringAssert a = Assertions.assertThat(repair.state.getResult().message).describedAs("Unexpected state: %s -> %s; example %d", repair.state, repair.state.getResult(), example); + // SymmetricRemoteSyncTask + AsymmetricRemoteSyncTask + // ... Sync failed between /[81fc:714:2c56:a2d3:faf3:eb7c:e4dd:cb9e]:54401 and /220.3.10.72:21402 + // LocalSyncTask + // ... failed with error Sync failed + // Dedup nack, but may be remote or local sync! + // ... Got SYNC_REQ failure from ...: UNKNOWN + String failingMsg = repair.state.getResult().message; + if (failingMsg.contains("Sync failed between")) + { + a.contains("Sync failed between").contains(failingAddress.toString()); + } + else if (failingMsg.contains("Got SYNC_REQ failure from")) + { + Assertions.assertThat(syncFailedAddresses).isNotEmpty(); + a.containsAnyOf(syncFailedAddresses.stream().map(s -> "Got SYNC_REQ failure from " + s + ": UNKNOWN").collect(Collectors.toList()).toArray(String[]::new)); + } + else + { + a.contains("failed with error Sync failed"); + } + break; + default: + throw new IllegalArgumentException("Unknown stage: " + stage); + } + closeables.forEach(Closeable::close); + closeables.clear(); + } + }); + } +} diff --git a/test/unit/org/apache/cassandra/repair/FuzzTestBase.java b/test/unit/org/apache/cassandra/repair/FuzzTestBase.java new file mode 100644 index 0000000000..ac7cb5260a --- /dev/null +++ b/test/unit/org/apache/cassandra/repair/FuzzTestBase.java @@ -0,0 +1,1296 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.repair; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.sql.Timestamp; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Random; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.BiFunction; +import java.util.function.Consumer; +import java.util.function.Function; +import java.util.function.LongSupplier; +import java.util.function.Supplier; +import javax.annotation.Nullable; + +import com.google.common.collect.Iterables; +import com.google.common.collect.Maps; +import org.apache.cassandra.config.UnitConfigOverride; +import org.junit.Before; +import org.junit.BeforeClass; + +import accord.utils.DefaultRandom; +import accord.utils.Gen; +import accord.utils.Gens; +import accord.utils.RandomSource; +import org.agrona.collections.Long2ObjectHashMap; +import org.agrona.collections.LongHashSet; +import org.apache.cassandra.concurrent.ExecutorBuilder; +import org.apache.cassandra.concurrent.ExecutorBuilderFactory; +import org.apache.cassandra.concurrent.ExecutorFactory; +import org.apache.cassandra.concurrent.ExecutorPlus; +import org.apache.cassandra.concurrent.InfiniteLoopExecutor; +import org.apache.cassandra.concurrent.Interruptible; +import org.apache.cassandra.concurrent.ScheduledExecutorPlus; +import org.apache.cassandra.concurrent.SequentialExecutorPlus; +import org.apache.cassandra.concurrent.SimulatedExecutorFactory; +import org.apache.cassandra.concurrent.Stage; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.cql3.CQLTester; +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.Digest; +import org.apache.cassandra.db.compaction.ICompactionManager; +import org.apache.cassandra.db.marshal.EmptyType; +import org.apache.cassandra.db.repair.CassandraTableRepairManager; +import org.apache.cassandra.db.repair.PendingAntiCompaction; +import org.apache.cassandra.dht.Murmur3Partitioner; +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.Token; +import org.apache.cassandra.exceptions.RequestFailureReason; +import org.apache.cassandra.gms.ApplicationState; +import org.apache.cassandra.gms.EndpointState; +import org.apache.cassandra.gms.HeartBeatState; +import org.apache.cassandra.gms.IEndpointStateChangeSubscriber; +import org.apache.cassandra.gms.IFailureDetector; +import org.apache.cassandra.gms.IGossiper; +import org.apache.cassandra.gms.VersionedValue; +import org.apache.cassandra.io.util.DataInputBuffer; +import org.apache.cassandra.io.util.DataOutputBuffer; +import org.apache.cassandra.locator.IEndpointSnitch; +import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.locator.LocalStrategy; +import org.apache.cassandra.locator.RangesAtEndpoint; +import org.apache.cassandra.locator.TokenMetadata; +import org.apache.cassandra.net.ConnectionType; +import org.apache.cassandra.net.IVerbHandler; +import org.apache.cassandra.net.Message; +import org.apache.cassandra.net.MessageDelivery; +import org.apache.cassandra.net.MessagingService; +import org.apache.cassandra.net.RequestCallback; +import org.apache.cassandra.repair.messages.RepairOption; +import org.apache.cassandra.repair.messages.ValidationResponse; +import org.apache.cassandra.repair.state.Completable; +import org.apache.cassandra.repair.state.CoordinatorState; +import org.apache.cassandra.repair.state.JobState; +import org.apache.cassandra.repair.state.SessionState; +import org.apache.cassandra.repair.state.ValidationState; +import org.apache.cassandra.schema.KeyspaceMetadata; +import org.apache.cassandra.schema.KeyspaceParams; +import org.apache.cassandra.schema.SchemaConstants; +import org.apache.cassandra.schema.SystemDistributedKeyspace; +import org.apache.cassandra.schema.TableId; +import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.schema.Tables; +import org.apache.cassandra.service.ActiveRepairService; +import org.apache.cassandra.service.StorageService; +import org.apache.cassandra.streaming.StreamEventHandler; +import org.apache.cassandra.streaming.StreamReceiveException; +import org.apache.cassandra.streaming.StreamSession; +import org.apache.cassandra.streaming.StreamState; +import org.apache.cassandra.streaming.StreamingChannel; +import org.apache.cassandra.streaming.StreamingDataInputPlus; +import org.apache.cassandra.tools.nodetool.Repair; +import org.apache.cassandra.utils.AbstractTypeGenerators; +import org.apache.cassandra.utils.CassandraGenerators; +import org.apache.cassandra.utils.Clock; +import org.apache.cassandra.utils.Closeable; +import org.apache.cassandra.utils.FailingBiConsumer; +import org.apache.cassandra.utils.Generators; +import org.apache.cassandra.utils.MBeanWrapper; +import org.apache.cassandra.utils.MerkleTree; +import org.apache.cassandra.utils.MerkleTrees; +import org.apache.cassandra.utils.concurrent.AsyncPromise; +import org.apache.cassandra.utils.concurrent.Future; +import org.apache.cassandra.utils.concurrent.ImmediateFuture; +import org.apache.cassandra.utils.progress.ProgressEventType; +import org.assertj.core.api.Assertions; +import org.mockito.Mockito; +import org.quicktheories.impl.JavaRandom; + +import static org.apache.cassandra.config.CassandraRelevantProperties.CLOCK_GLOBAL; +import static org.apache.cassandra.config.CassandraRelevantProperties.ORG_APACHE_CASSANDRA_DISABLE_MBEAN_REGISTRATION; + +public abstract class FuzzTestBase extends CQLTester.InMemory +{ + private static final int MISMATCH_NUM_PARTITIONS = 1; + private static final Gen IDENTIFIER_GEN = fromQT(Generators.IDENTIFIER_GEN); + private static final Gen KEYSPACE_NAME_GEN = fromQT(CassandraGenerators.KEYSPACE_NAME_GEN); + private static final Gen TABLE_ID_GEN = fromQT(CassandraGenerators.TABLE_ID_GEN); + private static final Gen ADDRESS_W_PORT = fromQT(CassandraGenerators.INET_ADDRESS_AND_PORT_GEN); + + private static boolean SETUP_SCHEMA = false; + static String KEYSPACE; + static List TABLES; + + @BeforeClass + public static void setUpClass() + { + ORG_APACHE_CASSANDRA_DISABLE_MBEAN_REGISTRATION.setBoolean(true); + CLOCK_GLOBAL.setString(ClockAccess.class.getName()); + // when running in CI an external actor will replace the test configs based off the test type (such as trie, cdc, etc.), this could then have failing tests + // that do not repo with the same seed! To fix that, go to UnitConfigOverride and update the config type to match the one that failed in CI, this should then + // use the same config, so the seed should not reproduce. + UnitConfigOverride.maybeOverrideConfig(); + + DatabaseDescriptor.daemonInitialization(); + DatabaseDescriptor.setPartitionerUnsafe(Murmur3Partitioner.instance); // TOOD (coverage): random select + DatabaseDescriptor.setLocalDataCenter("test"); + StreamingChannel.Factory.Global.unsafeSet(new StreamingChannel.Factory() + { + private final AtomicInteger counter = new AtomicInteger(); + + @Override + public StreamingChannel create(InetSocketAddress to, int messagingVersion, StreamingChannel.Kind kind) throws IOException + { + StreamingChannel mock = Mockito.mock(StreamingChannel.class); + int id = counter.incrementAndGet(); + StreamSession session = Mockito.mock(StreamSession.class); + StreamReceiveException access = new StreamReceiveException(session, "mock access rejected"); + StreamingDataInputPlus input = Mockito.mock(StreamingDataInputPlus.class, invocationOnMock -> { + throw access; + }); + Mockito.doNothing().when(input).close(); + Mockito.when(mock.in()).thenReturn(input); + Mockito.when(mock.id()).thenReturn(id); + Mockito.when(mock.peer()).thenReturn(to); + Mockito.when(mock.connectedTo()).thenReturn(to); + Mockito.when(mock.send(Mockito.any())).thenReturn(ImmediateFuture.success(null)); + Mockito.when(mock.close()).thenReturn(ImmediateFuture.success(null)); + return mock; + } + }); + ExecutorFactory delegate = ExecutorFactory.Global.executorFactory(); + ExecutorFactory.Global.unsafeSet(new ExecutorFactory() + { + @Override + public LocalAwareSubFactory localAware() + { + return delegate.localAware(); + } + + @Override + public ScheduledExecutorPlus scheduled(boolean executeOnShutdown, String name, int priority, SimulatorSemantics simulatorSemantics) + { + return delegate.scheduled(executeOnShutdown, name, priority, simulatorSemantics); + } + + private boolean shouldMock() + { + return StackWalker.getInstance().walk(frame -> { + StackWalker.StackFrame caller = frame.skip(3).findFirst().get(); + return caller.getClassName().startsWith("org.apache.cassandra.streaming."); + }); + } + + @Override + public Thread startThread(String name, Runnable runnable, InfiniteLoopExecutor.Daemon daemon) + { + if (shouldMock()) return new Thread(); + return delegate.startThread(name, runnable, daemon); + } + + @Override + public Interruptible infiniteLoop(String name, Interruptible.Task task, InfiniteLoopExecutor.SimulatorSafe simulatorSafe, InfiniteLoopExecutor.Daemon daemon, InfiniteLoopExecutor.Interrupts interrupts) + { + return delegate.infiniteLoop(name, task, simulatorSafe, daemon, interrupts); + } + + @Override + public ThreadGroup newThreadGroup(String name) + { + return delegate.newThreadGroup(name); + } + + @Override + public ExecutorBuilderFactory withJmx(String jmxPath) + { + return delegate.withJmx(jmxPath); + } + + @Override + public ExecutorBuilder configureSequential(String name) + { + return delegate.configureSequential(name); + } + + @Override + public ExecutorBuilder configurePooled(String name, int threads) + { + return delegate.configurePooled(name, threads); + } + }); + + // will both make sure this is loaded and used + if (!(Clock.Global.clock() instanceof ClockAccess)) throw new IllegalStateException("Unable to override clock"); + + // set the repair rcp timeout high so we don't hit it... this class is mostly testing repair reaching success + // so don't want to deal with unlucky histories... + DatabaseDescriptor.setRepairRpcTimeout(TimeUnit.DAYS.toMillis(1)); + + + InMemory.setUpClass(); + } + + @Before + public void setupSchema() + { + if (SETUP_SCHEMA) return; + SETUP_SCHEMA = true; + // StorageService can not be mocked out, nor can ColumnFamilyStores, so make sure that the keyspace is a "local" keyspace to avoid replication as the peers don't actually exist for replication + schemaChange(String.format("CREATE KEYSPACE %s WITH REPLICATION = {'class': '%s'}", SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, HackStrat.class.getName())); + for (TableMetadata table : SystemDistributedKeyspace.metadata().tables) + schemaChange(table.toCqlString(false, false)); + + createSchema(); + } + + private void createSchema() + { + // The main reason to use random here with a fixed seed is just to have a set of tables that are not hard coded. + // The tables will have diversity to them that most likely doesn't matter to repair (hence why the tables are shared), but + // is useful just in case some assumptions change. + RandomSource rs = new DefaultRandom(42); + String ks = KEYSPACE_NAME_GEN.next(rs); + List tableNames = Gens.lists(IDENTIFIER_GEN).unique().ofSizeBetween(10, 100).next(rs); + JavaRandom qt = new JavaRandom(rs.asJdkRandom()); + Tables.Builder tableBuilder = Tables.builder(); + List ids = Gens.lists(TABLE_ID_GEN).unique().ofSize(tableNames.size()).next(rs); + for (int i = 0; i < tableNames.size(); i++) + { + String name = tableNames.get(i); + TableId id = ids.get(i); + TableMetadata tableMetadata = new CassandraGenerators.TableMetadataBuilder().withKeyspaceName(ks).withTableName(name).withTableId(id).withTableKinds(TableMetadata.Kind.REGULAR) + // shouldn't matter, just wanted to avoid UDT as that needs more setup + .withDefaultTypeGen(AbstractTypeGenerators.builder().withTypeKinds(AbstractTypeGenerators.TypeKind.PRIMITIVE).withoutPrimitive(EmptyType.instance).build()).build().generate(qt); + tableBuilder.add(tableMetadata); + } + KeyspaceParams params = KeyspaceParams.simple(3); + KeyspaceMetadata metadata = KeyspaceMetadata.create(ks, params, tableBuilder.build()); + + // create + schemaChange(metadata.toCqlString(false, false)); + KEYSPACE = ks; + for (TableMetadata table : metadata.tables) + schemaChange(table.toCqlString(false, false)); + TABLES = tableNames; + } + + static void enableMessageFaults(Cluster cluster) + { + cluster.allowedMessageFaults(new BiFunction<>() + { + private final LongHashSet noFaults = new LongHashSet(); + private final LongHashSet allowDrop = new LongHashSet(); + + @Override + public Set apply(Cluster.Node node, Message message) + { + switch (message.verb()) + { + case PREPARE_MSG: + case VALIDATION_REQ: + case VALIDATION_RSP: + case SYNC_REQ: + case SYNC_RSP: + case SNAPSHOT_MSG: + case CLEANUP_MSG: + allowDrop.add(message.id()); + return Faults.DROPPED; + // these messages are not resilent to ephemeral issues + case PREPARE_CONSISTENT_REQ: + case PREPARE_CONSISTENT_RSP: + case FINALIZE_PROPOSE_MSG: + case FINALIZE_PROMISE_MSG: + case FINALIZE_COMMIT_MSG: + case FAILED_SESSION_MSG: + + noFaults.add(message.id()); + return Faults.NONE; + default: + if (noFaults.contains(message.id())) return Faults.NONE; + if (allowDrop.contains(message.id())) return Faults.DROPPED; + // was a new message added and the test not updated? + IllegalStateException e = new IllegalStateException("Verb: " + message.verb()); + cluster.failures.add(e); + throw e; + } + } + }); + } + + static void runAndAssertSuccess(Cluster cluster, int example, boolean shouldSync, RepairCoordinator repair) + { + cluster.processAll(); + assertSuccess(example, shouldSync, repair); + } + + static void assertSuccess(int example, boolean shouldSync, RepairCoordinator repair) + { + Assertions.assertThat(repair.state.getResult()).describedAs("Unexpected state: %s -> %s; example %d", repair.state, repair.state.getResult(), example).isEqualTo(Completable.Result.success(repairSuccessMessage(repair))); + Assertions.assertThat(repair.state.getStateTimesMillis().keySet()).isEqualTo(EnumSet.allOf(CoordinatorState.State.class)); + Assertions.assertThat(repair.state.getSessions()).isNotEmpty(); + boolean shouldSnapshot = repair.state.options.getParallelism() != RepairParallelism.PARALLEL + && (!repair.state.options.isIncremental() || repair.state.options.isPreview()); + for (SessionState session : repair.state.getSessions()) + { + Assertions.assertThat(session.getStateTimesMillis().keySet()).isEqualTo(EnumSet.allOf(SessionState.State.class)); + Assertions.assertThat(session.getJobs()).isNotEmpty(); + for (JobState job : session.getJobs()) + { + EnumSet expected = EnumSet.allOf(JobState.State.class); + if (!shouldSnapshot) + { + expected.remove(JobState.State.SNAPSHOT_START); + expected.remove(JobState.State.SNAPSHOT_COMPLETE); + } + if (!shouldSync) + { + expected.remove(JobState.State.STREAM_START); + } + Set actual = job.getStateTimesMillis().keySet(); + Assertions.assertThat(actual).isEqualTo(expected); + } + } + } + + static String repairSuccessMessage(RepairCoordinator repair) + { + RepairOption options = repair.state.options; + if (options.isPreview()) + { + String suffix; + switch (options.getPreviewKind()) + { + case UNREPAIRED: + case ALL: + suffix = "Previewed data was in sync"; + break; + case REPAIRED: + suffix = "Repaired data is in sync"; + break; + default: + throw new IllegalArgumentException("Unexpected preview repair kind: " + options.getPreviewKind()); + } + return "Repair preview completed successfully; " + suffix; + } + return "Repair completed successfully"; + } + + InetAddressAndPort pickParticipant(RandomSource rs, Cluster.Node coordinator, RepairCoordinator repair) + { + if (repair.state.isComplete()) + throw new IllegalStateException("Repair is completed! " + repair.state.getResult()); + List participaents = new ArrayList<>(repair.state.getNeighborsAndRanges().participants.size() + 1); + if (rs.nextBoolean()) participaents.add(coordinator.broadcastAddressAndPort()); + participaents.addAll(repair.state.getNeighborsAndRanges().participants); + participaents.sort(Comparator.naturalOrder()); + + InetAddressAndPort selected = rs.pick(participaents); + return selected; + } + + static void addMismatch(RandomSource rs, ColumnFamilyStore cfs, Validator validator) + { + ValidationState state = validator.state; + int maxDepth = DatabaseDescriptor.getRepairSessionMaxTreeDepth(); + state.phase.start(MISMATCH_NUM_PARTITIONS, 1024); + + MerkleTrees trees = new MerkleTrees(cfs.getPartitioner()); + for (Range range : validator.desc.ranges) + { + int depth = (int) Math.min(Math.ceil(Math.log(MISMATCH_NUM_PARTITIONS) / Math.log(2)), maxDepth); + trees.addMerkleTree((int) Math.pow(2, depth), range); + } + Set allTokens = new HashSet<>(); + for (Range range : validator.desc.ranges) + { + Gen gen = fromQT(CassandraGenerators.tokensInRange(range)); + Set tokens = new LinkedHashSet<>(); + for (int i = 0, size = rs.nextInt(1, 10); i < size; i++) + { + for (int attempt = 0; !tokens.add(gen.next(rs)) && attempt < 5; attempt++) + { + } + } + // tokens may or may not be of the expected size; this depends on how wide the range is + for (Token token : tokens) + trees.split(token); + allTokens.addAll(tokens); + } + for (Token token : allTokens) + { + findCorrectRange(trees, token, range -> { + Digest digest = Digest.forValidator(); + digest.update(ByteBuffer.wrap(token.toString().getBytes(StandardCharsets.UTF_8))); + range.addHash(new MerkleTree.RowHash(token, digest.digest(), 1)); + }); + } + state.partitionsProcessed++; + state.bytesRead = 1024; + state.phase.sendingTrees(); + Stage.ANTI_ENTROPY.execute(() -> { + state.phase.success(); + validator.respond(new ValidationResponse(validator.desc, trees)); + }); + } + + private static void findCorrectRange(MerkleTrees trees, Token token, Consumer fn) + { + MerkleTrees.TreeRangeIterator it = trees.rangeIterator(); + while (it.hasNext()) + { + MerkleTree.TreeRange next = it.next(); + if (next.contains(token)) + { + fn.accept(next); + return; + } + } + } + + private enum RepairType + {FULL, IR} + + private enum PreviewType + {NONE, REPAIRED, UNREPAIRED} + + static RepairOption repairOption(RandomSource rs, Cluster.Node coordinator, String ks, List tableNames) + { + return repairOption(rs, coordinator, ks, Gens.lists(Gens.pick(tableNames)).ofSizeBetween(1, tableNames.size()), Gens.enums().all(RepairType.class), Gens.enums().all(PreviewType.class), Gens.enums().all(RepairParallelism.class)); + } + + static RepairOption irOption(RandomSource rs, Cluster.Node coordinator, String ks, Gen> tablesGen) + { + return repairOption(rs, coordinator, ks, tablesGen, Gens.constant(RepairType.IR), Gens.constant(PreviewType.NONE), Gens.enums().all(RepairParallelism.class)); + } + + static RepairOption previewOption(RandomSource rs, Cluster.Node coordinator, String ks, Gen> tablesGen) + { + return repairOption(rs, coordinator, ks, tablesGen, Gens.constant(RepairType.FULL), Gens.constant(PreviewType.REPAIRED), Gens.enums().all(RepairParallelism.class)); + } + + private static RepairOption repairOption(RandomSource rs, Cluster.Node coordinator, String ks, Gen> tablesGen, Gen repairTypeGen, Gen previewTypeGen, Gen repairParallelismGen) + { + List args = new ArrayList<>(); + args.add(ks); + args.addAll(tablesGen.next(rs)); + args.add("-pr"); + RepairType type = repairTypeGen.next(rs); + switch (type) + { + case IR: + // default + break; + case FULL: + args.add("--full"); + break; + default: + throw new AssertionError("Unsupported repair type: " + type); + } + PreviewType previewType = previewTypeGen.next(rs); + switch (previewType) + { + case NONE: + break; + case REPAIRED: + args.add("--validate"); + break; + case UNREPAIRED: + args.add("--preview"); + break; + default: + throw new AssertionError("Unsupported preview type: " + previewType); + } + RepairParallelism parallelism = repairParallelismGen.next(rs); + switch (parallelism) + { + case SEQUENTIAL: + args.add("--sequential"); + break; + case PARALLEL: + // default + break; + case DATACENTER_AWARE: + args.add("--dc-parallel"); + break; + default: + throw new AssertionError("Unknown parallelism: " + parallelism); + } + if (rs.nextBoolean()) args.add("--optimise-streams"); + RepairOption options = RepairOption.parse(Repair.parseOptionMap(() -> "test", args), DatabaseDescriptor.getPartitioner()); + if (options.getRanges().isEmpty()) + { + if (options.isPrimaryRange()) + { + // when repairing only primary range, neither dataCenters nor hosts can be set + if (options.getDataCenters().isEmpty() && options.getHosts().isEmpty()) + options.getRanges().addAll(coordinator.getPrimaryRanges(ks)); + // except dataCenters only contain local DC (i.e. -local) + else if (options.isInLocalDCOnly()) + options.getRanges().addAll(coordinator.getPrimaryRangesWithinDC(ks)); + else + throw new IllegalArgumentException("You need to run primary range repair on all nodes in the cluster."); + } + else + { + Iterables.addAll(options.getRanges(), coordinator.getLocalReplicas(ks).onlyFull().ranges()); + } + } + return options; + } + + enum Faults + { + DELAY, DROP; + + public static final Set NONE = Collections.emptySet(); + public static final Set DROPPED = EnumSet.of(DELAY, DROP); + } + + private static class Connection + { + final InetAddressAndPort from, to; + + private Connection(InetAddressAndPort from, InetAddressAndPort to) + { + this.from = from; + this.to = to; + } + + @Override + public boolean equals(Object o) + { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Connection that = (Connection) o; + return from.equals(that.from) && to.equals(that.to); + } + + @Override + public int hashCode() + { + return Objects.hash(from, to); + } + + @Override + public String toString() + { + return "Connection{" + "from=" + from + ", to=" + to + '}'; + } + } + + interface MessageListener + { + default void preHandle(Cluster.Node node, Message msg) {} + } + + static class Cluster + { + private static final FailingBiConsumer DEFAULT_VALIDATION = ValidationManager::doValidation; + + final Map nodes; + private final IFailureDetector failureDetector = Mockito.mock(IFailureDetector.class); + private final IEndpointSnitch snitch = Mockito.mock(IEndpointSnitch.class); + private final SimulatedExecutorFactory globalExecutor; + final ScheduledExecutorPlus unorderedScheduled; + final ExecutorPlus orderedExecutor; + private final Gossip gossiper = new Gossip(); + private final MBeanWrapper mbean = Mockito.mock(MBeanWrapper.class); + private final List failures = new ArrayList<>(); + private final List listeners = new ArrayList<>(); + private final RandomSource rs; + private BiFunction, Set> allowedMessageFaults = (a, b) -> Collections.emptySet(); + + private final Map networkLatencies = new HashMap<>(); + private final Map> networkDrops = new HashMap<>(); + + Cluster(RandomSource rs) + { + ClockAccess.includeThreadAsOwner(); + this.rs = rs; + globalExecutor = new SimulatedExecutorFactory(rs, fromQT(Generators.TIMESTAMP_GEN.map(Timestamp::getTime)).mapToLong(TimeUnit.MILLISECONDS::toNanos).next(rs)); + orderedExecutor = globalExecutor.configureSequential("ignore").build(); + unorderedScheduled = globalExecutor.scheduled("ignored"); + + // We run tests in an isolated JVM per class, so not cleaing up is safe... but if that assumption ever changes, will need to cleanup + Stage.ANTI_ENTROPY.unsafeSetExecutor(orderedExecutor); + Stage.INTERNAL_RESPONSE.unsafeSetExecutor(unorderedScheduled); + Mockito.when(failureDetector.isAlive(Mockito.any())).thenReturn(true); + int numNodes = rs.nextInt(3, 10); + List dcs = Gens.lists(IDENTIFIER_GEN).unique().ofSizeBetween(1, Math.min(10, numNodes)).next(rs); + Map nodes = Maps.newHashMapWithExpectedSize(numNodes); + Gen tokenGen = fromQT(CassandraGenerators.token(DatabaseDescriptor.getPartitioner())); + Gen hostIdGen = fromQT(Generators.UUID_RANDOM_GEN); + Set tokens = new HashSet<>(); + Set hostIds = new HashSet<>(); + for (int i = 0; i < numNodes; i++) + { + InetAddressAndPort addressAndPort = ADDRESS_W_PORT.next(rs); + while (nodes.containsKey(addressAndPort)) addressAndPort = ADDRESS_W_PORT.next(rs); + Token token; + while (!tokens.add(token = tokenGen.next(rs))) + { + } + UUID hostId; + while (!hostIds.add(hostId = hostIdGen.next(rs))) + { + } + + String dc = rs.pick(dcs); + String rack = "rack"; + Mockito.when(snitch.getDatacenter(Mockito.eq(addressAndPort))).thenReturn(dc); + Mockito.when(snitch.getRack(Mockito.eq(addressAndPort))).thenReturn(rack); + + VersionedValue.VersionedValueFactory valueFactory = new VersionedValue.VersionedValueFactory(DatabaseDescriptor.getPartitioner()); + EndpointState state = new EndpointState(new HeartBeatState(42, 42)); + state.addApplicationState(ApplicationState.STATUS, valueFactory.normal(Collections.singleton(token))); + state.addApplicationState(ApplicationState.STATUS_WITH_PORT, valueFactory.normal(Collections.singleton(token))); + state.addApplicationState(ApplicationState.HOST_ID, valueFactory.hostId(hostId)); + state.addApplicationState(ApplicationState.TOKENS, valueFactory.tokens(Collections.singleton(token))); + state.addApplicationState(ApplicationState.DC, valueFactory.datacenter(dc)); + state.addApplicationState(ApplicationState.RACK, valueFactory.rack(rack)); + state.addApplicationState(ApplicationState.RELEASE_VERSION, valueFactory.releaseVersion()); + + gossiper.endpoints.put(addressAndPort, state); + + Node node = new Node(hostId, addressAndPort, Collections.singletonList(token), new Messaging(addressAndPort)); + nodes.put(addressAndPort, node); + } + this.nodes = nodes; + + TokenMetadata tm = StorageService.instance.getTokenMetadata(); + tm.clearUnsafe(); + for (Node inst : nodes.values()) + { + tm.updateHostId(inst.hostId(), inst.broadcastAddressAndPort()); + for (Token token : inst.tokens()) + tm.updateNormalToken(token, inst.broadcastAddressAndPort()); + } + } + + public Closeable addListener(MessageListener listener) + { + listeners.add(listener); + return () -> removeListener(listener); + } + + public void removeListener(MessageListener listener) + { + listeners.remove(listener); + } + + public void allowedMessageFaults(BiFunction, Set> fn) + { + this.allowedMessageFaults = fn; + } + + public void checkFailures() + { + if (Thread.interrupted()) + failures.add(new InterruptedException()); + if (failures.isEmpty()) return; + AssertionError error = new AssertionError("Unexpected exceptions found"); + failures.forEach(error::addSuppressed); + failures.clear(); + throw error; + } + + public boolean processOne() + { + boolean result = globalExecutor.processOne(); + checkFailures(); + return result; + } + + public void processAll() + { + while (processOne()) + { + } + } + + private static class CallbackContext + { + final RequestCallback callback; + + private CallbackContext(RequestCallback callback) + { + this.callback = Objects.requireNonNull(callback); + } + + public void onResponse(Message msg) + { + callback.onResponse(msg); + } + + public void onFailure(InetAddressAndPort from, RequestFailureReason failureReason) + { + if (callback.invokeOnFailure()) callback.onFailure(from, failureReason); + } + } + + private class Messaging implements MessageDelivery + { + final InetAddressAndPort broadcastAddressAndPort; + final Long2ObjectHashMap callbacks = new Long2ObjectHashMap<>(); + + private Messaging(InetAddressAndPort broadcastAddressAndPort) + { + this.broadcastAddressAndPort = broadcastAddressAndPort; + } + + @Override + public void send(Message message, InetAddressAndPort to) + { + message = message.withFrom(broadcastAddressAndPort); + maybeEnqueue(message, to, null); + } + + @Override + public void sendWithCallback(Message message, InetAddressAndPort to, RequestCallback cb) + { + message = message.withFrom(broadcastAddressAndPort); + maybeEnqueue(message, to, cb); + } + + @Override + public void sendWithCallback(Message message, InetAddressAndPort to, RequestCallback cb, ConnectionType specifyConnection) + { + message = message.withFrom(broadcastAddressAndPort); + maybeEnqueue(message, to, cb); + } + + private void maybeEnqueue(Message message, InetAddressAndPort to, @Nullable RequestCallback callback) + { + CallbackContext cb; + if (callback != null) + { + cb = new CallbackContext(callback); + callbacks.put(message.id(), cb); + } + else + { + cb = null; + } + boolean toSelf = this.broadcastAddressAndPort.equals(to); + Node node = nodes.get(to); + Set allowedFaults = allowedMessageFaults.apply(node, message); + if (allowedFaults.isEmpty()) + { + // enqueue so stack overflow doesn't happen with the inlining + unorderedScheduled.submit(() -> node.handle(message)); + } + else + { + Runnable enqueue = () -> { + if (!allowedFaults.contains(Faults.DELAY)) + { + unorderedScheduled.submit(() -> node.handle(message)); + } + else + { + if (toSelf) unorderedScheduled.submit(() -> node.handle(message)); + else + unorderedScheduled.schedule(() -> node.handle(message), networkJitterNanos(to), TimeUnit.NANOSECONDS); + } + }; + + if (!allowedFaults.contains(Faults.DROP)) enqueue.run(); + else + { + if (!toSelf && networkDrops(to)) + { +// logger.warn("Dropped message {}", message); + // drop + } + else + { + enqueue.run(); + } + } + + if (cb != null) + { + unorderedScheduled.schedule(() -> { + CallbackContext ctx = callbacks.remove(message.id()); + if (ctx != null) + { + assert ctx == cb; + ctx.onFailure(to, RequestFailureReason.TIMEOUT); + } + }, message.verb().expiresAfterNanos(), TimeUnit.NANOSECONDS); + } + } + } + + private long networkJitterNanos(InetAddressAndPort to) + { + return networkLatencies.computeIfAbsent(new Connection(broadcastAddressAndPort, to), ignore -> { + long min = TimeUnit.MICROSECONDS.toNanos(500); + long maxSmall = TimeUnit.MILLISECONDS.toNanos(5); + long max = TimeUnit.SECONDS.toNanos(5); + LongSupplier small = () -> rs.nextLong(min, maxSmall); + LongSupplier large = () -> rs.nextLong(maxSmall, max); + return Gens.bools().runs(rs.nextInt(1, 11) / 100.0D, rs.nextInt(3, 15)).mapToLong(b -> b ? large.getAsLong() : small.getAsLong()).asLongSupplier(rs); + }).getAsLong(); + } + + private boolean networkDrops(InetAddressAndPort to) + { + return networkDrops.computeIfAbsent(new Connection(broadcastAddressAndPort, to), ignore -> Gens.bools().runs(rs.nextInt(1, 11) / 100.0D, rs.nextInt(3, 15)).asSupplier(rs)).get(); + } + + @Override + public Future> sendWithResult(Message message, InetAddressAndPort to) + { + AsyncPromise> promise = new AsyncPromise<>(); + sendWithCallback(message, to, new RequestCallback() + { + @Override + public void onResponse(Message msg) + { + promise.trySuccess(msg); + } + + @Override + public void onFailure(InetAddressAndPort from, RequestFailureReason failureReason) + { + promise.tryFailure(new MessagingService.FailureResponseException(from, failureReason)); + } + + @Override + public boolean invokeOnFailure() + { + return true; + } + }); + return promise; + } + + @Override + public void respond(V response, Message message) + { + send(message.responseWith(response), message.respondTo()); + } + } + + private class Gossip implements IGossiper + { + private final Map endpoints = new HashMap<>(); + + @Override + public void register(IEndpointStateChangeSubscriber subscriber) + { + + } + + @Override + public void unregister(IEndpointStateChangeSubscriber subscriber) + { + + } + + @Nullable + @Override + public EndpointState getEndpointStateForEndpoint(InetAddressAndPort ep) + { + return endpoints.get(ep); + } + } + + class Node implements SharedContext + { + private final ICompactionManager compactionManager = Mockito.mock(ICompactionManager.class); + final UUID hostId; + final InetAddressAndPort addressAndPort; + final Collection tokens; + final ActiveRepairService activeRepairService; + final IVerbHandler verbHandler; + final Messaging messaging; + final IValidationManager validationManager; + private FailingBiConsumer doValidation = DEFAULT_VALIDATION; + private final StreamExecutor defaultStreamExecutor = plan -> { + long delayNanos = rs.nextLong(TimeUnit.SECONDS.toNanos(5), TimeUnit.MINUTES.toNanos(10)); + unorderedScheduled.schedule(() -> { + StreamState success = new StreamState(plan.planId(), plan.streamOperation(), Collections.emptySet()); + for (StreamEventHandler handler : plan.handlers()) + handler.onSuccess(success); + }, delayNanos, TimeUnit.NANOSECONDS); + return null; + }; + private StreamExecutor streamExecutor = defaultStreamExecutor; + + private Node(UUID hostId, InetAddressAndPort addressAndPort, Collection tokens, Messaging messaging) + { + this.hostId = hostId; + this.addressAndPort = addressAndPort; + this.tokens = tokens; + this.messaging = messaging; + this.activeRepairService = new ActiveRepairService(this); + this.validationManager = (cfs, validator) -> unorderedScheduled.submit(() -> { + try + { + doValidation.acceptOrFail(cfs, validator); + } + catch (Throwable e) + { + validator.fail(e); + } + }); + this.verbHandler = new RepairMessageVerbHandler(this); + + activeRepairService.start(); + } + + public Closeable doValidation(FailingBiConsumer fn) + { + FailingBiConsumer previous = this.doValidation; + if (previous != DEFAULT_VALIDATION) + throw new IllegalStateException("Attemptted to override validation, but was already overridden"); + this.doValidation = fn; + return () -> this.doValidation = previous; + } + + public Closeable doValidation(Function, FailingBiConsumer> fn) + { + FailingBiConsumer previous = this.doValidation; + this.doValidation = fn.apply(previous); + return () -> this.doValidation = previous; + } + + public Closeable doSync(StreamExecutor streamExecutor) + { + StreamExecutor previous = this.streamExecutor; + if (previous != defaultStreamExecutor) + throw new IllegalStateException("Attemptted to override sync, but was already overridden"); + this.streamExecutor = streamExecutor; + return () -> this.streamExecutor = previous; + } + + void handle(Message msg) + { + msg = serde(msg); + if (msg == null) + { + logger.warn("Got a message that failed to serialize/deserialize"); + return; + } + for (MessageListener l : listeners) + l.preHandle(this, msg); + if (msg.verb().isResponse()) + { + // handle callbacks + if (messaging.callbacks.containsKey(msg.id())) + { + CallbackContext callback = messaging.callbacks.remove(msg.id()); + if (callback == null) return; + try + { + if (msg.isFailureResponse()) + callback.onFailure(msg.from(), (RequestFailureReason) msg.payload); + else callback.onResponse(msg); + } + catch (Throwable t) + { + failures.add(t); + } + } + } + else + { + try + { + verbHandler.doVerb(msg); + } + catch (Throwable e) + { + failures.add(e); + } + } + } + + public UUID hostId() + { + return hostId; + } + + @Override + public InetAddressAndPort broadcastAddressAndPort() + { + return addressAndPort; + } + + public Collection tokens() + { + return tokens; + } + + public IFailureDetector failureDetector() + { + return failureDetector; + } + + @Override + public IEndpointSnitch snitch() + { + return snitch; + } + + @Override + public IGossiper gossiper() + { + return gossiper; + } + + @Override + public ICompactionManager compactionManager() + { + return compactionManager; + } + + public ExecutorFactory executorFactory() + { + return globalExecutor; + } + + public ScheduledExecutorPlus optionalTasks() + { + return unorderedScheduled; + } + + @Override + public Supplier random() + { + return () -> rs.fork().asJdkRandom(); + } + + public Clock clock() + { + return globalExecutor; + } + + public MessageDelivery messaging() + { + return messaging; + } + + public MBeanWrapper mbean() + { + return mbean; + } + + public RepairCoordinator repair(String ks, RepairOption options) + { + return repair(ks, options, true); + } + + public RepairCoordinator repair(String ks, RepairOption options, boolean addFailureOnErrorNotification) + { + RepairCoordinator repair = new RepairCoordinator(this, (name, tables) -> StorageService.instance.getValidColumnFamilies(false, false, name, tables), name -> StorageService.instance.getReplicas(name, broadcastAddressAndPort()), 42, options, ks); + if (addFailureOnErrorNotification) + { + repair.addProgressListener((tag, event) -> { + if (event.getType() == ProgressEventType.ERROR) + failures.add(new AssertionError(event.getMessage())); + }); + } + return repair; + } + + public RangesAtEndpoint getLocalReplicas(String ks) + { + return StorageService.instance.getReplicas(ks, broadcastAddressAndPort()); + } + + public Collection> getPrimaryRanges(String ks) + { + return StorageService.instance.getPrimaryRangesForEndpoint(ks, broadcastAddressAndPort()); + } + + public Collection> getPrimaryRangesWithinDC(String ks) + { + return StorageService.instance.getPrimaryRangeForEndpointWithinDC(ks, broadcastAddressAndPort()); + } + + @Override + public ActiveRepairService repair() + { + return activeRepairService; + } + + @Override + public IValidationManager validationManager() + { + return validationManager; + } + + @Override + public TableRepairManager repairManager(ColumnFamilyStore store) + { + return new CassandraTableRepairManager(store, this) + { + @Override + public void snapshot(String name, Collection> ranges, boolean force) + { + // no-op + } + }; + } + + @Override + public StreamExecutor streamExecutor() + { + return streamExecutor; + } + } + + private Message serde(Message msg) + { + try (DataOutputBuffer b = DataOutputBuffer.scratchBuffer.get()) + { + int messagingVersion = MessagingService.Version.CURRENT.value; + Message.serializer.serialize(msg, b, messagingVersion); + DataInputBuffer in = new DataInputBuffer(b.unsafeGetBufferAndFlip(), false); + return Message.serializer.deserialize(in, msg.from(), messagingVersion); + } + catch (Throwable e) + { + failures.add(e); + return null; + } + } + } + + private static Gen fromQT(org.quicktheories.core.Gen qt) + { + return rs -> { + JavaRandom r = new JavaRandom(rs.asJdkRandom()); + return qt.generate(r); + }; + } + + public static class HackStrat extends LocalStrategy + { + public HackStrat(String keyspaceName, TokenMetadata tokenMetadata, IEndpointSnitch snitch, Map configOptions) + { + super(keyspaceName, tokenMetadata, snitch, configOptions); + } + } + + /** + * Since the clock is accessable via {@link Global#currentTimeMillis()} and {@link Global#nanoTime()}, and only repair subsystem has the requirement not to touch that, this class + * acts as a safty check that validates that repair does not touch these methods but allows the other subsystems to do so. + */ + public static class ClockAccess implements Clock + { + private static final Set OWNERS = Collections.synchronizedSet(new HashSet<>()); + private final Clock delegate = new Default(); + + public static void includeThreadAsOwner() + { + OWNERS.add(Thread.currentThread()); + } + + @Override + public long nanoTime() + { + checkAccess(); + return delegate.nanoTime(); + } + + @Override + public long currentTimeMillis() + { + checkAccess(); + return delegate.currentTimeMillis(); + } + + private enum Access + {MAIN_THREAD_ONLY, REJECT, IGNORE} + + private void checkAccess() + { + Access access = StackWalker.getInstance().walk(frames -> { + Iterator it = frames.iterator(); + boolean topLevel = false; + while (it.hasNext()) + { + StackWalker.StackFrame next = it.next(); + if (!topLevel) + { + // need to find the top level! + while (!Clock.Global.class.getName().equals(next.getClassName())) + { + assert it.hasNext(); + next = it.next(); + } + topLevel = true; + assert it.hasNext(); + next = it.next(); + } + if (FuzzTestBase.class.getName().equals(next.getClassName())) return Access.MAIN_THREAD_ONLY; + if (next.getClassName().startsWith("org.apache.cassandra.db.") || next.getClassName().startsWith("org.apache.cassandra.gms.") || next.getClassName().startsWith("org.apache.cassandra.cql3.") || next.getClassName().startsWith("org.apache.cassandra.metrics.") || next.getClassName().startsWith("org.apache.cassandra.utils.concurrent.") + || next.getClassName().startsWith("org.apache.cassandra.utils.TimeUUID") // this would be good to solve + || next.getClassName().startsWith(PendingAntiCompaction.class.getName())) + return Access.IGNORE; + if (next.getClassName().startsWith("org.apache.cassandra.repair") || ActiveRepairService.class.getName().startsWith(next.getClassName())) + return Access.REJECT; + } + return Access.IGNORE; + }); + Thread current = Thread.currentThread(); + switch (access) + { + case IGNORE: + return; + case REJECT: + throw new IllegalStateException("Rejecting access"); + case MAIN_THREAD_ONLY: + if (!OWNERS.contains(current)) throw new IllegalStateException("Accessed in wrong thread: " + current); + break; + } + } + } + + static class SimulatedFault extends RuntimeException + { + SimulatedFault(String message) + { + super(message); + } + } +} diff --git a/test/unit/org/apache/cassandra/repair/HappyPathFuzzTest.java b/test/unit/org/apache/cassandra/repair/HappyPathFuzzTest.java new file mode 100644 index 0000000000..b6e34fbcf0 --- /dev/null +++ b/test/unit/org/apache/cassandra/repair/HappyPathFuzzTest.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.repair; + +import java.util.ArrayList; +import java.util.List; + +import org.junit.Test; + +import accord.utils.Gen; +import accord.utils.Gens; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.config.RetrySpec; +import org.apache.cassandra.utils.Closeable; + +import static accord.utils.Property.qt; + +public class HappyPathFuzzTest extends FuzzTestBase +{ + @Test + public void happyPath() + { + // disable all retries, no delays/drops are possible + DatabaseDescriptor.getRepairRetrySpec().maxAttempts = RetrySpec.MaxAttempt.DISABLED; + qt().withPure(false).withExamples(10).check(rs -> { + Cluster cluster = new Cluster(rs); + Gen coordinatorGen = Gens.pick(cluster.nodes.keySet()).map(cluster.nodes::get); + + List closeables = new ArrayList<>(); + for (int example = 0; example < 100; example++) + { + Cluster.Node coordinator = coordinatorGen.next(rs); + + RepairCoordinator repair = coordinator.repair(KEYSPACE, repairOption(rs, coordinator, KEYSPACE, TABLES)); + repair.run(); + boolean shouldSync = rs.nextBoolean(); + if (shouldSync) + closeables.add(cluster.nodes.get(pickParticipant(rs, coordinator, repair)).doValidation((cfs, validator) -> addMismatch(rs, cfs, validator))); + + runAndAssertSuccess(cluster, example, shouldSync, repair); + closeables.forEach(Closeable::close); + closeables.clear(); + } + }); + } +} diff --git a/test/unit/org/apache/cassandra/repair/LocalSyncTaskTest.java b/test/unit/org/apache/cassandra/repair/LocalSyncTaskTest.java index 4409b3cdef..95f630dc05 100644 --- a/test/unit/org/apache/cassandra/repair/LocalSyncTaskTest.java +++ b/test/unit/org/apache/cassandra/repair/LocalSyncTaskTest.java @@ -94,7 +94,7 @@ public class LocalSyncTaskTest extends AbstractRepairTest // note: we reuse the same endpoint which is bogus in theory but fine here TreeResponse r1 = new TreeResponse(local, tree1); TreeResponse r2 = new TreeResponse(ep2, tree2); - LocalSyncTask task = new LocalSyncTask(desc, r1.endpoint, r2.endpoint, MerkleTrees.difference(r1.trees, r2.trees), + LocalSyncTask task = new LocalSyncTask(SharedContext.Global.instance, desc, r1.endpoint, r2.endpoint, MerkleTrees.difference(r1.trees, r2.trees), NO_PENDING_REPAIR, true, true, PreviewKind.NONE); task.run(); @@ -109,10 +109,10 @@ public class LocalSyncTaskTest extends AbstractRepairTest Keyspace keyspace = Keyspace.open(KEYSPACE1); ColumnFamilyStore cfs = keyspace.getColumnFamilyStore("Standard1"); - ActiveRepairService.instance.registerParentRepairSession(parentRepairSession, FBUtilities.getBroadcastAddressAndPort(), - Arrays.asList(cfs), Arrays.asList(range), false, - ActiveRepairService.UNREPAIRED_SSTABLE, false, - PreviewKind.NONE); + ActiveRepairService.instance().registerParentRepairSession(parentRepairSession, FBUtilities.getBroadcastAddressAndPort(), + Arrays.asList(cfs), Arrays.asList(range), false, + ActiveRepairService.UNREPAIRED_SSTABLE, false, + PreviewKind.NONE); RepairJobDesc desc = new RepairJobDesc(parentRepairSession, nextTimeUUID(), KEYSPACE1, "Standard1", Arrays.asList(range)); @@ -132,7 +132,7 @@ public class LocalSyncTaskTest extends AbstractRepairTest // note: we reuse the same endpoint which is bogus in theory but fine here TreeResponse r1 = new TreeResponse(local, tree1); TreeResponse r2 = new TreeResponse(InetAddressAndPort.getByName("127.0.0.2"), tree2); - LocalSyncTask task = new LocalSyncTask(desc, r1.endpoint, r2.endpoint, MerkleTrees.difference(r1.trees, r2.trees), + LocalSyncTask task = new LocalSyncTask(SharedContext.Global.instance, desc, r1.endpoint, r2.endpoint, MerkleTrees.difference(r1.trees, r2.trees), NO_PENDING_REPAIR, true, true, PreviewKind.NONE); NettyStreamingConnectionFactory.MAX_CONNECT_ATTEMPTS = 1; try @@ -152,13 +152,13 @@ public class LocalSyncTaskTest extends AbstractRepairTest public void fullRepairStreamPlan() throws Exception { TimeUUID sessionID = registerSession(cfs, true, true); - ActiveRepairService.ParentRepairSession prs = ActiveRepairService.instance.getParentRepairSession(sessionID); + ActiveRepairService.ParentRepairSession prs = ActiveRepairService.instance().getParentRepairSession(sessionID); RepairJobDesc desc = new RepairJobDesc(sessionID, nextTimeUUID(), KEYSPACE1, CF_STANDARD, prs.getRanges()); TreeResponse r1 = new TreeResponse(local, createInitialTree(desc, DatabaseDescriptor.getPartitioner())); TreeResponse r2 = new TreeResponse(PARTICIPANT2, createInitialTree(desc, DatabaseDescriptor.getPartitioner())); - LocalSyncTask task = new LocalSyncTask(desc, r1.endpoint, r2.endpoint, MerkleTrees.difference(r1.trees, r2.trees), + LocalSyncTask task = new LocalSyncTask(SharedContext.Global.instance, desc, r1.endpoint, r2.endpoint, MerkleTrees.difference(r1.trees, r2.trees), NO_PENDING_REPAIR, true, true, PreviewKind.NONE); StreamPlan plan = task.createStreamPlan(); @@ -178,13 +178,13 @@ public class LocalSyncTaskTest extends AbstractRepairTest public void incrementalRepairStreamPlan() throws Exception { TimeUUID sessionID = registerSession(cfs, true, true); - ActiveRepairService.ParentRepairSession prs = ActiveRepairService.instance.getParentRepairSession(sessionID); + ActiveRepairService.ParentRepairSession prs = ActiveRepairService.instance().getParentRepairSession(sessionID); RepairJobDesc desc = new RepairJobDesc(sessionID, nextTimeUUID(), KEYSPACE1, CF_STANDARD, prs.getRanges()); TreeResponse r1 = new TreeResponse(local, createInitialTree(desc, DatabaseDescriptor.getPartitioner())); TreeResponse r2 = new TreeResponse(PARTICIPANT2, createInitialTree(desc, DatabaseDescriptor.getPartitioner())); - LocalSyncTask task = new LocalSyncTask(desc, r1.endpoint, r2.endpoint, MerkleTrees.difference(r1.trees, r2.trees), + LocalSyncTask task = new LocalSyncTask(SharedContext.Global.instance, desc, r1.endpoint, r2.endpoint, MerkleTrees.difference(r1.trees, r2.trees), desc.parentSessionId, true, true, PreviewKind.NONE); StreamPlan plan = task.createStreamPlan(); @@ -200,13 +200,13 @@ public class LocalSyncTaskTest extends AbstractRepairTest public void transientRemoteStreamPlan() throws NoSuchRepairSessionException { TimeUUID sessionID = registerSession(cfs, true, true); - ActiveRepairService.ParentRepairSession prs = ActiveRepairService.instance.getParentRepairSession(sessionID); + ActiveRepairService.ParentRepairSession prs = ActiveRepairService.instance().getParentRepairSession(sessionID); RepairJobDesc desc = new RepairJobDesc(sessionID, nextTimeUUID(), KEYSPACE1, CF_STANDARD, prs.getRanges()); TreeResponse r1 = new TreeResponse(local, createInitialTree(desc, DatabaseDescriptor.getPartitioner())); TreeResponse r2 = new TreeResponse(PARTICIPANT2, createInitialTree(desc, DatabaseDescriptor.getPartitioner())); - LocalSyncTask task = new LocalSyncTask(desc, r1.endpoint, r2.endpoint, MerkleTrees.difference(r1.trees, r2.trees), + LocalSyncTask task = new LocalSyncTask(SharedContext.Global.instance, desc, r1.endpoint, r2.endpoint, MerkleTrees.difference(r1.trees, r2.trees), desc.parentSessionId, true, false, PreviewKind.NONE); StreamPlan plan = task.createStreamPlan(); assertNumInOut(plan, 1, 0); @@ -219,13 +219,13 @@ public class LocalSyncTaskTest extends AbstractRepairTest public void transientLocalStreamPlan() throws NoSuchRepairSessionException { TimeUUID sessionID = registerSession(cfs, true, true); - ActiveRepairService.ParentRepairSession prs = ActiveRepairService.instance.getParentRepairSession(sessionID); + ActiveRepairService.ParentRepairSession prs = ActiveRepairService.instance().getParentRepairSession(sessionID); RepairJobDesc desc = new RepairJobDesc(sessionID, nextTimeUUID(), KEYSPACE1, CF_STANDARD, prs.getRanges()); TreeResponse r1 = new TreeResponse(local, createInitialTree(desc, DatabaseDescriptor.getPartitioner())); TreeResponse r2 = new TreeResponse(PARTICIPANT2, createInitialTree(desc, DatabaseDescriptor.getPartitioner())); - LocalSyncTask task = new LocalSyncTask(desc, r1.endpoint, r2.endpoint, MerkleTrees.difference(r1.trees, r2.trees), + LocalSyncTask task = new LocalSyncTask(SharedContext.Global.instance, desc, r1.endpoint, r2.endpoint, MerkleTrees.difference(r1.trees, r2.trees), desc.parentSessionId, false, true, PreviewKind.NONE); StreamPlan plan = task.createStreamPlan(); assertNumInOut(plan, 0, 1); diff --git a/test/unit/org/apache/cassandra/repair/NeighborsAndRangesTest.java b/test/unit/org/apache/cassandra/repair/NeighborsAndRangesTest.java index 9dcd7dc2a3..d5c578d730 100644 --- a/test/unit/org/apache/cassandra/repair/NeighborsAndRangesTest.java +++ b/test/unit/org/apache/cassandra/repair/NeighborsAndRangesTest.java @@ -29,7 +29,7 @@ import org.junit.Test; import org.apache.cassandra.locator.InetAddressAndPort; -import static org.apache.cassandra.repair.RepairRunnable.NeighborsAndRanges; +import static org.apache.cassandra.repair.RepairCoordinator.NeighborsAndRanges; public class NeighborsAndRangesTest extends AbstractRepairTest { diff --git a/test/unit/org/apache/cassandra/repair/RepairJobTest.java b/test/unit/org/apache/cassandra/repair/RepairJobTest.java index db7255f2f5..641926ed8a 100644 --- a/test/unit/org/apache/cassandra/repair/RepairJobTest.java +++ b/test/unit/org/apache/cassandra/repair/RepairJobTest.java @@ -37,6 +37,9 @@ import java.util.stream.Collectors; import com.google.common.collect.ImmutableMap; import com.google.common.util.concurrent.ListenableFuture; + +import org.apache.cassandra.repair.messages.SyncResponse; +import org.apache.cassandra.repair.messages.ValidationResponse; import org.assertj.core.api.Assertions; import org.junit.After; import org.junit.Before; @@ -44,7 +47,6 @@ import org.junit.BeforeClass; import org.junit.Test; import org.apache.cassandra.SchemaLoader; -import org.apache.cassandra.concurrent.ExecutorFactory; import org.apache.cassandra.concurrent.ExecutorPlus; import org.apache.cassandra.db.ConsistencyLevel; import org.apache.cassandra.db.Keyspace; @@ -67,7 +69,6 @@ import org.apache.cassandra.service.paxos.cleanup.PaxosCleanupRequest; import org.apache.cassandra.service.paxos.cleanup.PaxosCleanupResponse; import org.apache.cassandra.service.paxos.cleanup.PaxosCleanupSession; import org.apache.cassandra.streaming.PreviewKind; -import org.apache.cassandra.streaming.SessionSummary; import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.MerkleTree; @@ -82,7 +83,6 @@ import static org.apache.cassandra.repair.RepairParallelism.SEQUENTIAL; import static org.apache.cassandra.streaming.PreviewKind.NONE; import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID; import static org.apache.cassandra.utils.asserts.SyncTaskAssert.assertThat; -import static org.apache.cassandra.utils.asserts.SyncTaskListAssert.assertThat; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -127,13 +127,14 @@ public class RepairJobTest PreviewKind previewKind, boolean optimiseStreams, boolean repairPaxos, boolean paxosOnly, String... cfnames) { - super(parentRepairSession, commonRange, keyspace, parallelismDegree, isIncremental, pullRepair, + super(SharedContext.Global.instance, parentRepairSession, commonRange, keyspace, parallelismDegree, isIncremental, pullRepair, previewKind, optimiseStreams, repairPaxos, paxosOnly, cfnames); } - protected ExecutorPlus createExecutor() + @Override + protected ExecutorPlus createExecutor(SharedContext ctx) { - return ExecutorFactory.Global.executorFactory() + return ctx.executorFactory() .configurePooled("RepairJobTask", Integer.MAX_VALUE) .withDefaultThreadGroup() .withKeepAlive(THREAD_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS) @@ -141,7 +142,7 @@ public class RepairJobTest } @Override - public void syncComplete(RepairJobDesc desc, SyncNodePair nodes, boolean success, List summaries) + public void syncComplete(RepairJobDesc desc, Message message) { for (Callable callback : syncCompleteCallbacks) { @@ -154,7 +155,7 @@ public class RepairJobTest throw Throwables.cleaned(e); } } - super.syncComplete(desc, nodes, success, summaries); + super.syncComplete(desc, message); } public void registerSyncCompleteCallback(Callable callback) @@ -183,9 +184,9 @@ public class RepairJobTest Set neighbors = new HashSet<>(Arrays.asList(addr2, addr3)); TimeUUID parentRepairSession = nextTimeUUID(); - ActiveRepairService.instance.registerParentRepairSession(parentRepairSession, FBUtilities.getBroadcastAddressAndPort(), - Collections.singletonList(Keyspace.open(KEYSPACE).getColumnFamilyStore(CF)), FULL_RANGE, false, - ActiveRepairService.UNREPAIRED_SSTABLE, false, PreviewKind.NONE); + ActiveRepairService.instance().registerParentRepairSession(parentRepairSession, FBUtilities.getBroadcastAddressAndPort(), + Collections.singletonList(Keyspace.open(KEYSPACE).getColumnFamilyStore(CF)), FULL_RANGE, false, + ActiveRepairService.UNREPAIRED_SSTABLE, false, PreviewKind.NONE); this.session = new MeasureableRepairSession(parentRepairSession, new CommonRange(neighbors, emptySet(), FULL_RANGE), @@ -202,7 +203,7 @@ public class RepairJobTest @After public void reset() { - ActiveRepairService.instance.terminateSessions(); + ActiveRepairService.instance().terminateSessions(); MessagingService.instance().outboundSink.clear(); MessagingService.instance().inboundSink.clear(); FBUtilities.reset(); @@ -262,7 +263,7 @@ public class RepairJobTest // Use addr4 instead of one of the provided trees to force everything to be remote sync tasks as // LocalSyncTasks try to reach over the network. - List syncTasks = RepairJob.createStandardSyncTasks(sessionJobDesc, mockTreeResponses, + List syncTasks = RepairJob.createStandardSyncTasks(SharedContext.Global.instance, sessionJobDesc, mockTreeResponses, addr4, // local noTransient(), session.isIncremental, @@ -339,7 +340,9 @@ public class RepairJobTest List tasks = job.validationTasks; assertEquals(3, tasks.size()); assertFalse(tasks.stream().anyMatch(ValidationTask::isActive)); - assertFalse(tasks.stream().allMatch(ValidationTask::isDone)); + // Switched from false to true in CASSANDRA-18816. + // The reason is that not failing the future can cause the failing cleanup logic to not always happen + assertTrue(tasks.stream().allMatch(ValidationTask::isDone)); } @Test @@ -360,7 +363,7 @@ public class RepairJobTest treeResponse(addr2, RANGE_1, "different", RANGE_2, "same", RANGE_3, "different"), treeResponse(addr3, RANGE_1, "same", RANGE_2, "same", RANGE_3, "same")); - Map tasks = toMap(RepairJob.createStandardSyncTasks(JOB_DESC, + Map tasks = toMap(RepairJob.createStandardSyncTasks(SharedContext.Global.instance, JOB_DESC, treeResponses, addr1, // local noTransient(), // transient @@ -396,7 +399,7 @@ public class RepairJobTest List treeResponses = Arrays.asList(treeResponse(addr1, RANGE_1, "same", RANGE_2, "same", RANGE_3, "same"), treeResponse(addr2, RANGE_1, "different", RANGE_2, "same", RANGE_3, "different")); - Map tasks = toMap(RepairJob.createStandardSyncTasks(JOB_DESC, + Map tasks = toMap(RepairJob.createStandardSyncTasks(SharedContext.Global.instance, JOB_DESC, treeResponses, addr1, // local transientPredicate(addr2), @@ -426,7 +429,7 @@ public class RepairJobTest List treeResponses = Arrays.asList(treeResponse(addr1, RANGE_1, "same", RANGE_2, "same", RANGE_3, "same"), treeResponse(addr2, RANGE_1, "different", RANGE_2, "same", RANGE_3, "different")); - Map tasks = toMap(RepairJob.createStandardSyncTasks(JOB_DESC, + Map tasks = toMap(RepairJob.createStandardSyncTasks(SharedContext.Global.instance, JOB_DESC, treeResponses, addr1, // local transientPredicate(addr1), @@ -486,7 +489,7 @@ public class RepairJobTest List treeResponses = Arrays.asList(treeResponse(addr1, RANGE_1, "same", RANGE_2, "same", RANGE_3, "same"), treeResponse(addr2, RANGE_1, "same", RANGE_2, "same", RANGE_3, "same")); - Map tasks = toMap(RepairJob.createStandardSyncTasks(JOB_DESC, + Map tasks = toMap(RepairJob.createStandardSyncTasks(SharedContext.Global.instance, JOB_DESC, treeResponses, local, // local isTransient, @@ -504,7 +507,7 @@ public class RepairJobTest treeResponse(addr2, RANGE_1, "two", RANGE_2, "two", RANGE_3, "two"), treeResponse(addr3, RANGE_1, "three", RANGE_2, "three", RANGE_3, "three")); - Map tasks = toMap(RepairJob.createStandardSyncTasks(JOB_DESC, + Map tasks = toMap(RepairJob.createStandardSyncTasks(SharedContext.Global.instance, JOB_DESC, treeResponses, addr1, // local ep -> ep.equals(addr3), // transient @@ -535,7 +538,7 @@ public class RepairJobTest treeResponse(addr5, RANGE_1, "five", RANGE_2, "five", RANGE_3, "five")); Predicate isTransient = ep -> ep.equals(addr4) || ep.equals(addr5); - Map tasks = toMap(RepairJob.createStandardSyncTasks(JOB_DESC, + Map tasks = toMap(RepairJob.createStandardSyncTasks(SharedContext.Global.instance, JOB_DESC, treeResponses, addr1, // local isTransient, // transient @@ -602,7 +605,7 @@ public class RepairJobTest treeResponse(addr5, RANGE_1, "five", RANGE_2, "five", RANGE_3, "five")); Predicate isTransient = ep -> ep.equals(addr4) || ep.equals(addr5); - Map tasks = toMap(RepairJob.createStandardSyncTasks(JOB_DESC, + Map tasks = toMap(RepairJob.createStandardSyncTasks(SharedContext.Global.instance, JOB_DESC, treeResponses, local, // local isTransient, // transient @@ -651,7 +654,7 @@ public class RepairJobTest treeResponse(addr4, RANGE_1, "four", RANGE_2, "four", RANGE_3, "four"), treeResponse(addr5, RANGE_1, "five", RANGE_2, "five", RANGE_3, "five")); - Map tasks = toMap(RepairJob.createStandardSyncTasks(JOB_DESC, + Map tasks = toMap(RepairJob.createStandardSyncTasks(SharedContext.Global.instance, JOB_DESC, treeResponses, addr4, // local ep -> ep.equals(addr4) || ep.equals(addr5), // transient @@ -669,7 +672,7 @@ public class RepairJobTest treeResponse(addr2, RANGE_1, "two", RANGE_2, "two", RANGE_3, "two"), treeResponse(addr3, RANGE_1, "three", RANGE_2, "three", RANGE_3, "three")); - Map tasks = toMap(RepairJob.createOptimisedSyncingSyncTasks(JOB_DESC, + Map tasks = toMap(RepairJob.createOptimisedSyncingSyncTasks(SharedContext.Global.instance, JOB_DESC, treeResponses, addr1, // local noTransient(), @@ -704,7 +707,7 @@ public class RepairJobTest treeResponse(addr2, RANGE_1, "one", RANGE_2, "two"), treeResponse(addr3, RANGE_1, "three", RANGE_2, "two")); - Map tasks = toMap(RepairJob.createOptimisedSyncingSyncTasks(JOB_DESC, + Map tasks = toMap(RepairJob.createOptimisedSyncingSyncTasks(SharedContext.Global.instance, JOB_DESC, treeResponses, addr4, // local noTransient(), @@ -737,7 +740,7 @@ public class RepairJobTest treeResponse(addr3, RANGE_1, "same", RANGE_2, "same", RANGE_3, "same")); RepairJobDesc desc = new RepairJobDesc(nextTimeUUID(), nextTimeUUID(), "ks", "cf", Collections.emptyList()); - Map tasks = toMap(RepairJob.createOptimisedSyncingSyncTasks(desc, + Map tasks = toMap(RepairJob.createOptimisedSyncingSyncTasks(SharedContext.Global.instance, desc, treeResponses, addr1, // local ep -> ep.equals(addr3), @@ -890,12 +893,12 @@ public class RepairJobTest MessagingService.instance().callbacks.removeAndRespond(message.id(), to, message.emptyResponse()); break; case VALIDATION_REQ: - session.validationComplete(sessionJobDesc, to, mockTrees.get(to)); + MerkleTrees tree = mockTrees.get(to); + session.validationComplete(sessionJobDesc, Message.builder(Verb.VALIDATION_RSP, tree != null ? new ValidationResponse(sessionJobDesc, tree) : new ValidationResponse(sessionJobDesc)).from(to).build()); break; case SYNC_REQ: SyncRequest syncRequest = (SyncRequest) message.payload; - session.syncComplete(sessionJobDesc, new SyncNodePair(syncRequest.src, syncRequest.dst), - true, Collections.emptyList()); + session.syncComplete(sessionJobDesc, Message.builder(Verb.SYNC_RSP, new SyncResponse(sessionJobDesc, new SyncNodePair(syncRequest.src, syncRequest.dst), true, Collections.emptyList())).from(to).build()); break; default: break; diff --git a/test/unit/org/apache/cassandra/repair/RepairSessionTest.java b/test/unit/org/apache/cassandra/repair/RepairSessionTest.java index 4db6efb680..98baf46c57 100644 --- a/test/unit/org/apache/cassandra/repair/RepairSessionTest.java +++ b/test/unit/org/apache/cassandra/repair/RepairSessionTest.java @@ -63,7 +63,7 @@ public class RepairSessionTest IPartitioner p = Murmur3Partitioner.instance; Range repairRange = new Range<>(p.getToken(ByteBufferUtil.bytes(0)), p.getToken(ByteBufferUtil.bytes(100))); Set endpoints = Sets.newHashSet(remote); - RepairSession session = new RepairSession(parentSessionId, + RepairSession session = new RepairSession(SharedContext.Global.instance, parentSessionId, new CommonRange(endpoints, Collections.emptySet(), Arrays.asList(repairRange)), "Keyspace1", RepairParallelism.SEQUENTIAL, false, false, diff --git a/test/unit/org/apache/cassandra/repair/SlowMessageFuzzTest.java b/test/unit/org/apache/cassandra/repair/SlowMessageFuzzTest.java new file mode 100644 index 0000000000..00fb2817cb --- /dev/null +++ b/test/unit/org/apache/cassandra/repair/SlowMessageFuzzTest.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.repair; + +import java.util.ArrayList; +import java.util.List; + +import org.junit.Test; + +import accord.utils.Gen; +import accord.utils.Gens; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.config.RetrySpec; +import org.apache.cassandra.utils.Closeable; + +import static accord.utils.Property.qt; + +public class SlowMessageFuzzTest extends FuzzTestBase +{ + @Test + public void slowMessages() + { + // to avoid unlucky timing issues, retry until success; given enough retries we should eventually become success + DatabaseDescriptor.getRepairRetrySpec().maxAttempts = new RetrySpec.MaxAttempt(Integer.MAX_VALUE); + qt().withPure(false).withExamples(10).check(rs -> { + Cluster cluster = new Cluster(rs); + enableMessageFaults(cluster); + + Gen coordinatorGen = Gens.pick(cluster.nodes.keySet()).map(cluster.nodes::get); + + List closeables = new ArrayList<>(); + for (int example = 0; example < 100; example++) + { + Cluster.Node coordinator = coordinatorGen.next(rs); + + RepairCoordinator repair = coordinator.repair(KEYSPACE, repairOption(rs, coordinator, KEYSPACE, TABLES)); + repair.run(); + boolean shouldSync = rs.nextBoolean(); + if (shouldSync) + closeables.add(cluster.nodes.get(pickParticipant(rs, coordinator, repair)).doValidation((cfs, validator) -> addMismatch(rs, cfs, validator))); + + runAndAssertSuccess(cluster, example, shouldSync, repair); + closeables.forEach(Closeable::close); + closeables.clear(); + } + }); + } +} diff --git a/test/unit/org/apache/cassandra/repair/StreamingRepairTaskTest.java b/test/unit/org/apache/cassandra/repair/StreamingRepairTaskTest.java index a499138062..8aa39d06d9 100644 --- a/test/unit/org/apache/cassandra/repair/StreamingRepairTaskTest.java +++ b/test/unit/org/apache/cassandra/repair/StreamingRepairTaskTest.java @@ -27,12 +27,14 @@ import org.apache.cassandra.SchemaLoader; import org.apache.cassandra.cql3.statements.schema.CreateTableStatement; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.repair.messages.SyncRequest; +import org.apache.cassandra.repair.state.SyncState; import org.apache.cassandra.schema.KeyspaceParams; import org.apache.cassandra.schema.Schema; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.service.ActiveRepairService; import org.apache.cassandra.streaming.PreviewKind; import org.apache.cassandra.streaming.StreamPlan; +import org.apache.cassandra.utils.Clock; import org.apache.cassandra.utils.TimeUUID; import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID; @@ -63,11 +65,11 @@ public class StreamingRepairTaskTest extends AbstractRepairTest public void incrementalStreamPlan() throws NoSuchRepairSessionException { TimeUUID sessionID = registerSession(cfs, true, true); - ActiveRepairService.ParentRepairSession prs = ActiveRepairService.instance.getParentRepairSession(sessionID); + ActiveRepairService.ParentRepairSession prs = ActiveRepairService.instance().getParentRepairSession(sessionID); RepairJobDesc desc = new RepairJobDesc(sessionID, nextTimeUUID(), ks, tbl, prs.getRanges()); SyncRequest request = new SyncRequest(desc, PARTICIPANT1, PARTICIPANT2, PARTICIPANT3, prs.getRanges(), PreviewKind.NONE, false); - StreamingRepairTask task = new StreamingRepairTask(desc, request.initiator, request.src, request.dst, request.ranges, desc.sessionId, PreviewKind.NONE, false); + StreamingRepairTask task = new StreamingRepairTask(SharedContext.Global.instance, new SyncState(Clock.Global.clock(), desc, PARTICIPANT1, PARTICIPANT2, PARTICIPANT3), desc, request.initiator, request.src, request.dst, request.ranges, desc.sessionId, PreviewKind.NONE, false); StreamPlan plan = task.createStreamPlan(request.dst); Assert.assertFalse(plan.getFlushBeforeTransfer()); @@ -77,10 +79,10 @@ public class StreamingRepairTaskTest extends AbstractRepairTest public void fullStreamPlan() throws Exception { TimeUUID sessionID = registerSession(cfs, false, true); - ActiveRepairService.ParentRepairSession prs = ActiveRepairService.instance.getParentRepairSession(sessionID); + ActiveRepairService.ParentRepairSession prs = ActiveRepairService.instance().getParentRepairSession(sessionID); RepairJobDesc desc = new RepairJobDesc(sessionID, nextTimeUUID(), ks, tbl, prs.getRanges()); SyncRequest request = new SyncRequest(desc, PARTICIPANT1, PARTICIPANT2, PARTICIPANT3, prs.getRanges(), PreviewKind.NONE, false); - StreamingRepairTask task = new StreamingRepairTask(desc, request.initiator, request.src, request.dst, request.ranges, null, PreviewKind.NONE, false); + StreamingRepairTask task = new StreamingRepairTask(SharedContext.Global.instance, new SyncState(Clock.Global.clock(), desc, PARTICIPANT1, PARTICIPANT2, PARTICIPANT3), desc, request.initiator, request.src, request.dst, request.ranges, null, PreviewKind.NONE, false); StreamPlan plan = task.createStreamPlan(request.dst); Assert.assertTrue(plan.getFlushBeforeTransfer()); diff --git a/test/unit/org/apache/cassandra/repair/SymmetricRemoteSyncTaskTest.java b/test/unit/org/apache/cassandra/repair/SymmetricRemoteSyncTaskTest.java index 710bee6f96..37d4e09e78 100644 --- a/test/unit/org/apache/cassandra/repair/SymmetricRemoteSyncTaskTest.java +++ b/test/unit/org/apache/cassandra/repair/SymmetricRemoteSyncTaskTest.java @@ -41,7 +41,7 @@ public class SymmetricRemoteSyncTaskTest extends AbstractRepairTest { public InstrumentedSymmetricRemoteSyncTask(InetAddressAndPort e1, InetAddressAndPort e2) { - super(DESC, e1, e2, RANGE_LIST, PreviewKind.NONE); + super(SharedContext.Global.instance, DESC, e1, e2, RANGE_LIST, PreviewKind.NONE); } RepairMessage sentMessage = null; diff --git a/test/unit/org/apache/cassandra/repair/ValidationTaskTest.java b/test/unit/org/apache/cassandra/repair/ValidationTaskTest.java index 1ea40e9617..5639d4105c 100644 --- a/test/unit/org/apache/cassandra/repair/ValidationTaskTest.java +++ b/test/unit/org/apache/cassandra/repair/ValidationTaskTest.java @@ -33,7 +33,6 @@ import java.util.UUID; import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; public class ValidationTaskTest @@ -52,10 +51,12 @@ public class ValidationTaskTest { ValidationTask task = createTask(); assertTrue(task.isActive()); - task.abort(); + task.abort(new RuntimeException()); assertFalse(task.isActive()); task.treesReceived(new MerkleTrees(null)); - assertNull(task.get()); + // REVIEW: setting null would cause NPEs in sync task, so it was never correct to set null + assertTrue(task.isDone()); + assertFalse(task.isSuccess()); } @Test @@ -71,13 +72,13 @@ public class ValidationTaskTest assertEquals(1, trees.size()); // This relies on the fact that MerkleTrees clears its range -> tree map on release. - task.abort(); + task.abort(new RuntimeException()); assertEquals(0, trees.size()); } private ValidationTask createTask() throws UnknownHostException { InetAddressAndPort addressAndPort = InetAddressAndPort.getByName("127.0.0.1"); RepairJobDesc desc = new RepairJobDesc(nextTimeUUID(), nextTimeUUID(), UUID.randomUUID().toString(), UUID.randomUUID().toString(), null); - return new ValidationTask(desc, addressAndPort, 0, PreviewKind.NONE); + return new ValidationTask(SharedContext.Global.instance, desc, addressAndPort, 0, PreviewKind.NONE); } } diff --git a/test/unit/org/apache/cassandra/repair/ValidatorTest.java b/test/unit/org/apache/cassandra/repair/ValidatorTest.java index 437b360da6..7d0317f2c1 100644 --- a/test/unit/org/apache/cassandra/repair/ValidatorTest.java +++ b/test/unit/org/apache/cassandra/repair/ValidatorTest.java @@ -54,6 +54,7 @@ import org.apache.cassandra.schema.KeyspaceParams; import org.apache.cassandra.service.ActiveRepairService; import org.apache.cassandra.streaming.PreviewKind; import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.utils.Clock; import org.apache.cassandra.utils.MerkleTree; import org.apache.cassandra.utils.MerkleTrees; import org.apache.cassandra.utils.TimeUUID; @@ -111,7 +112,7 @@ public class ValidatorTest ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(columnFamily); - Validator validator = new Validator(new ValidationState(desc, remote), 0, PreviewKind.NONE); + Validator validator = new Validator(new ValidationState(Clock.Global.clock(), desc, remote), 0, PreviewKind.NONE); validator.state.phase.start(10, 10); MerkleTrees trees = new MerkleTrees(partitioner); trees.addMerkleTrees((int) Math.pow(2, 15), validator.desc.ranges); @@ -148,7 +149,7 @@ public class ValidatorTest InetAddressAndPort remote = InetAddressAndPort.getByName("127.0.0.2"); - Validator validator = new Validator(new ValidationState(desc, remote), 0, PreviewKind.NONE); + Validator validator = new Validator(new ValidationState(Clock.Global.clock(), desc, remote), 0, PreviewKind.NONE); validator.fail(new Throwable()); Message message = outgoingMessageSink.get(TEST_TIMEOUT, TimeUnit.SECONDS); @@ -202,12 +203,12 @@ public class ValidatorTest InetAddressAndPort host = InetAddressAndPort.getByName("127.0.0.2"); - ActiveRepairService.instance.registerParentRepairSession(repairSessionId, host, - Collections.singletonList(cfs), desc.ranges, false, ActiveRepairService.UNREPAIRED_SSTABLE, - false, PreviewKind.NONE); + ActiveRepairService.instance().registerParentRepairSession(repairSessionId, host, + Collections.singletonList(cfs), desc.ranges, false, ActiveRepairService.UNREPAIRED_SSTABLE, + false, PreviewKind.NONE); final CompletableFuture outgoingMessageSink = registerOutgoingMessageSink(); - Validator validator = new Validator(new ValidationState(desc, host), 0, true, false, PreviewKind.NONE); + Validator validator = new Validator(SharedContext.Global.instance, new ValidationState(Clock.Global.clock(), desc, host), 0, true, false, PreviewKind.NONE); ValidationManager.instance.submitValidation(cfs, validator); Message message = outgoingMessageSink.get(TEST_TIMEOUT, TimeUnit.SECONDS); @@ -259,12 +260,12 @@ public class ValidatorTest InetAddressAndPort host = InetAddressAndPort.getByName("127.0.0.2"); - ActiveRepairService.instance.registerParentRepairSession(repairSessionId, host, - Collections.singletonList(cfs), desc.ranges, false, ActiveRepairService.UNREPAIRED_SSTABLE, - false, PreviewKind.NONE); + ActiveRepairService.instance().registerParentRepairSession(repairSessionId, host, + Collections.singletonList(cfs), desc.ranges, false, ActiveRepairService.UNREPAIRED_SSTABLE, + false, PreviewKind.NONE); final CompletableFuture outgoingMessageSink = registerOutgoingMessageSink(); - Validator validator = new Validator(new ValidationState(desc, host), 0, true, false, PreviewKind.NONE); + Validator validator = new Validator(SharedContext.Global.instance, new ValidationState(Clock.Global.clock(), desc, host), 0, true, false, PreviewKind.NONE); ValidationManager.instance.submitValidation(cfs, validator); Message message = outgoingMessageSink.get(TEST_TIMEOUT, TimeUnit.SECONDS); @@ -321,12 +322,12 @@ public class ValidatorTest InetAddressAndPort host = InetAddressAndPort.getByName("127.0.0.2"); - ActiveRepairService.instance.registerParentRepairSession(repairSessionId, host, - Collections.singletonList(cfs), desc.ranges, false, ActiveRepairService.UNREPAIRED_SSTABLE, - false, PreviewKind.NONE); + ActiveRepairService.instance().registerParentRepairSession(repairSessionId, host, + Collections.singletonList(cfs), desc.ranges, false, ActiveRepairService.UNREPAIRED_SSTABLE, + false, PreviewKind.NONE); final CompletableFuture outgoingMessageSink = registerOutgoingMessageSink(); - Validator validator = new Validator(new ValidationState(desc, host), 0, true, false, PreviewKind.NONE); + Validator validator = new Validator(SharedContext.Global.instance, new ValidationState(Clock.Global.clock(), desc, host), 0, true, false, PreviewKind.NONE); ValidationManager.instance.submitValidation(cfs, validator); Message message = outgoingMessageSink.get(TEST_TIMEOUT, TimeUnit.SECONDS); diff --git a/test/unit/org/apache/cassandra/repair/consistent/AbstractConsistentSessionTest.java b/test/unit/org/apache/cassandra/repair/consistent/AbstractConsistentSessionTest.java index b7d49fc093..9716cb0809 100644 --- a/test/unit/org/apache/cassandra/repair/consistent/AbstractConsistentSessionTest.java +++ b/test/unit/org/apache/cassandra/repair/consistent/AbstractConsistentSessionTest.java @@ -81,14 +81,14 @@ public abstract class AbstractConsistentSessionTest { TimeUUID sessionId = nextTimeUUID(); - ActiveRepairService.instance.registerParentRepairSession(sessionId, - COORDINATOR, - Lists.newArrayList(cfs), - Sets.newHashSet(RANGE1, RANGE2, RANGE3), - true, - System.currentTimeMillis(), - true, - PreviewKind.NONE); + ActiveRepairService.instance().registerParentRepairSession(sessionId, + COORDINATOR, + Lists.newArrayList(cfs), + Sets.newHashSet(RANGE1, RANGE2, RANGE3), + true, + System.currentTimeMillis(), + true, + PreviewKind.NONE); return sessionId; } } diff --git a/test/unit/org/apache/cassandra/repair/consistent/CoordinatorMessagingTest.java b/test/unit/org/apache/cassandra/repair/consistent/CoordinatorMessagingTest.java index 0f457b6e2e..b9d859f3d7 100644 --- a/test/unit/org/apache/cassandra/repair/consistent/CoordinatorMessagingTest.java +++ b/test/unit/org/apache/cassandra/repair/consistent/CoordinatorMessagingTest.java @@ -106,7 +106,7 @@ public class CoordinatorMessagingTest extends AbstractRepairTest MockMessagingSpy spyCommit = createCommitSpy(); TimeUUID uuid = registerSession(cfs, true, true); - CoordinatorSession coordinator = ActiveRepairService.instance.consistent.coordinated.registerSession(uuid, PARTICIPANTS, false); + CoordinatorSession coordinator = ActiveRepairService.instance().consistent.coordinated.registerSession(uuid, PARTICIPANTS, false); AtomicBoolean repairSubmitted = new AtomicBoolean(false); Promise repairFuture = AsyncPromise.uncancellable(); Supplier> sessionSupplier = () -> @@ -144,7 +144,7 @@ public class CoordinatorMessagingTest extends AbstractRepairTest spyCommit.interceptNoMsg(100, TimeUnit.MILLISECONDS); Assert.assertEquals(ConsistentSession.State.FINALIZED, coordinator.getState()); - Assert.assertFalse(ActiveRepairService.instance.consistent.local.isSessionInProgress(uuid)); + Assert.assertFalse(ActiveRepairService.instance().consistent.local.isSessionInProgress(uuid)); } @@ -194,7 +194,7 @@ public class CoordinatorMessagingTest extends AbstractRepairTest MockMessagingSpy sendFailSessionExpectedSpy = createFailSessionSpy(Lists.newArrayList(PARTICIPANT1, PARTICIPANT2, PARTICIPANT3)); TimeUUID uuid = registerSession(cfs, true, true); - CoordinatorSession coordinator = ActiveRepairService.instance.consistent.coordinated.registerSession(uuid, PARTICIPANTS, false); + CoordinatorSession coordinator = ActiveRepairService.instance().consistent.coordinated.registerSession(uuid, PARTICIPANTS, false); AtomicBoolean repairSubmitted = new AtomicBoolean(false); Promise repairFuture = AsyncPromise.uncancellable(); Supplier> sessionSupplier = () -> @@ -223,7 +223,7 @@ public class CoordinatorMessagingTest extends AbstractRepairTest Assert.assertTrue(sessionResult.isDone()); Assert.assertNotNull(sessionResult.cause()); Assert.assertEquals(ConsistentSession.State.FAILED, coordinator.getState()); - Assert.assertFalse(ActiveRepairService.instance.consistent.local.isSessionInProgress(uuid)); + Assert.assertFalse(ActiveRepairService.instance().consistent.local.isSessionInProgress(uuid)); } @Test @@ -233,7 +233,7 @@ public class CoordinatorMessagingTest extends AbstractRepairTest MockMessagingSpy sendFailSessionUnexpectedSpy = createFailSessionSpy(Lists.newArrayList(PARTICIPANT1, PARTICIPANT2, PARTICIPANT3)); TimeUUID uuid = registerSession(cfs, true, true); - CoordinatorSession coordinator = ActiveRepairService.instance.consistent.coordinated.registerSession(uuid, PARTICIPANTS, false); + CoordinatorSession coordinator = ActiveRepairService.instance().consistent.coordinated.registerSession(uuid, PARTICIPANTS, false); AtomicBoolean repairSubmitted = new AtomicBoolean(false); Promise repairFuture = AsyncPromise.uncancellable(); Supplier> sessionSupplier = () -> @@ -265,7 +265,7 @@ public class CoordinatorMessagingTest extends AbstractRepairTest sendFailSessionUnexpectedSpy.interceptNoMsg(100, TimeUnit.MILLISECONDS); Assert.assertFalse(repairSubmitted.get()); Assert.assertEquals(ConsistentSession.State.PREPARING, coordinator.getState()); - Assert.assertFalse(ActiveRepairService.instance.consistent.local.isSessionInProgress(uuid)); + Assert.assertFalse(ActiveRepairService.instance().consistent.local.isSessionInProgress(uuid)); } private MockMessagingSpy createPrepareSpy(Collection failed, diff --git a/test/unit/org/apache/cassandra/repair/consistent/CoordinatorSessionTest.java b/test/unit/org/apache/cassandra/repair/consistent/CoordinatorSessionTest.java index 85574c740d..d10aa16cf8 100644 --- a/test/unit/org/apache/cassandra/repair/consistent/CoordinatorSessionTest.java +++ b/test/unit/org/apache/cassandra/repair/consistent/CoordinatorSessionTest.java @@ -34,6 +34,7 @@ import com.google.common.collect.Sets; import org.junit.Assert; import org.junit.Test; +import org.apache.cassandra.repair.SharedContext; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; import org.apache.cassandra.locator.InetAddressAndPort; @@ -63,7 +64,7 @@ public class CoordinatorSessionTest extends AbstractRepairTest static CoordinatorSession.Builder createBuilder() { - CoordinatorSession.Builder builder = CoordinatorSession.builder(); + CoordinatorSession.Builder builder = CoordinatorSession.builder(SharedContext.Global.instance); builder.withState(PREPARING); builder.withSessionID(nextTimeUUID()); builder.withCoordinator(COORDINATOR); diff --git a/test/unit/org/apache/cassandra/repair/consistent/CoordinatorSessionsTest.java b/test/unit/org/apache/cassandra/repair/consistent/CoordinatorSessionsTest.java index 725833e532..9f0ee4635d 100644 --- a/test/unit/org/apache/cassandra/repair/consistent/CoordinatorSessionsTest.java +++ b/test/unit/org/apache/cassandra/repair/consistent/CoordinatorSessionsTest.java @@ -26,6 +26,7 @@ import org.junit.BeforeClass; import org.junit.Test; import org.apache.cassandra.SchemaLoader; +import org.apache.cassandra.repair.SharedContext; import org.apache.cassandra.cql3.statements.schema.CreateTableStatement; import org.apache.cassandra.repair.AbstractRepairTest; import org.apache.cassandra.repair.NoSuchRepairSessionException; @@ -85,6 +86,11 @@ public class CoordinatorSessionsTest extends AbstractRepairTest private static class InstrumentedCoordinatorSessions extends CoordinatorSessions { + private InstrumentedCoordinatorSessions() + { + super(SharedContext.Global.instance); + } + protected CoordinatorSession buildSession(CoordinatorSession.Builder builder) { return new InstrumentedCoordinatorSession(builder); @@ -118,7 +124,7 @@ public class CoordinatorSessionsTest extends AbstractRepairTest @Test public void registerSessionTest() throws NoSuchRepairSessionException { - CoordinatorSessions sessions = new CoordinatorSessions(); + CoordinatorSessions sessions = new CoordinatorSessions(SharedContext.Global.instance); TimeUUID sessionID = registerSession(); CoordinatorSession session = sessions.registerSession(sessionID, PARTICIPANTS, false); @@ -127,7 +133,7 @@ public class CoordinatorSessionsTest extends AbstractRepairTest Assert.assertEquals(COORDINATOR, session.coordinator); Assert.assertEquals(Sets.newHashSet(cfm.id), session.tableIds); - ActiveRepairService.ParentRepairSession prs = ActiveRepairService.instance.getParentRepairSession(sessionID); + ActiveRepairService.ParentRepairSession prs = ActiveRepairService.instance().getParentRepairSession(sessionID); Assert.assertEquals(prs.repairedAt, session.repairedAt); Assert.assertEquals(prs.getRanges(), session.ranges); Assert.assertEquals(PARTICIPANTS, session.participants); diff --git a/test/unit/org/apache/cassandra/repair/consistent/LocalSessionAccessor.java b/test/unit/org/apache/cassandra/repair/consistent/LocalSessionAccessor.java index 6fd1c9e476..292151f18a 100644 --- a/test/unit/org/apache/cassandra/repair/consistent/LocalSessionAccessor.java +++ b/test/unit/org/apache/cassandra/repair/consistent/LocalSessionAccessor.java @@ -30,7 +30,7 @@ import org.apache.cassandra.utils.TimeUUID; */ public class LocalSessionAccessor { - private static final ActiveRepairService ARS = ActiveRepairService.instance; + private static final ActiveRepairService ARS = ActiveRepairService.instance(); public static void startup() { diff --git a/test/unit/org/apache/cassandra/repair/consistent/LocalSessionTest.java b/test/unit/org/apache/cassandra/repair/consistent/LocalSessionTest.java index a0411acd5c..0f9912437e 100644 --- a/test/unit/org/apache/cassandra/repair/consistent/LocalSessionTest.java +++ b/test/unit/org/apache/cassandra/repair/consistent/LocalSessionTest.java @@ -39,6 +39,7 @@ import org.junit.BeforeClass; import org.junit.Test; import org.apache.cassandra.SchemaLoader; +import org.apache.cassandra.repair.SharedContext; import org.apache.cassandra.cql3.QueryProcessor; import org.apache.cassandra.cql3.UntypedResultSet; import org.apache.cassandra.cql3.statements.schema.CreateTableStatement; @@ -81,7 +82,7 @@ public class LocalSessionTest extends AbstractRepairTest static LocalSession.Builder createBuilder() { - LocalSession.Builder builder = LocalSession.builder(); + LocalSession.Builder builder = LocalSession.builder(SharedContext.Global.instance); builder.withState(PREPARING); builder.withSessionID(nextTimeUUID()); builder.withCoordinator(COORDINATOR); @@ -131,6 +132,11 @@ public class LocalSessionTest extends AbstractRepairTest { Map> sentMessages = new HashMap<>(); + public InstrumentedLocalSessions() + { + super(SharedContext.Global.instance); + } + protected void sendMessage(InetAddressAndPort destination, Message message) { if (!sentMessages.containsKey(destination)) @@ -258,7 +264,7 @@ public class LocalSessionTest extends AbstractRepairTest @Test public void persistence() { - LocalSessions sessions = new LocalSessions(); + LocalSessions sessions = new LocalSessions(SharedContext.Global.instance); LocalSession expected = createSession(); sessions.save(expected); LocalSession actual = sessions.loadUnsafe(expected.sessionID); @@ -930,17 +936,17 @@ public class LocalSessionTest extends AbstractRepairTest @Test public void loadCorruptRow() throws Exception { - LocalSessions sessions = new LocalSessions(); + LocalSessions sessions = new LocalSessions(SharedContext.Global.instance); LocalSession session = createSession(); sessions.save(session); - sessions = new LocalSessions(); + sessions = new LocalSessions(SharedContext.Global.instance); sessions.start(); Assert.assertNotNull(sessions.getSession(session.sessionID)); QueryProcessor.instance.executeInternal("DELETE participants, participants_wp FROM system.repairs WHERE parent_id=?", session.sessionID); - sessions = new LocalSessions(); + sessions = new LocalSessions(SharedContext.Global.instance); sessions.start(); Assert.assertNull(sessions.getSession(session.sessionID)); UntypedResultSet res = QueryProcessor.executeInternal("SELECT * FROM system.repairs WHERE parent_id=?", session.sessionID); @@ -962,7 +968,7 @@ public class LocalSessionTest extends AbstractRepairTest @Test public void cleanupNoOp() throws Exception { - LocalSessions sessions = new LocalSessions(); + LocalSessions sessions = new LocalSessions(SharedContext.Global.instance); sessions.start(); long time = FBUtilities.nowInSeconds() - LocalSessions.AUTO_FAIL_TIMEOUT + 60; diff --git a/test/unit/org/apache/cassandra/repair/consistent/PendingRepairStatTest.java b/test/unit/org/apache/cassandra/repair/consistent/PendingRepairStatTest.java index c1b3c9b9a6..814b9c318d 100644 --- a/test/unit/org/apache/cassandra/repair/consistent/PendingRepairStatTest.java +++ b/test/unit/org/apache/cassandra/repair/consistent/PendingRepairStatTest.java @@ -32,6 +32,7 @@ import org.junit.Test; import org.apache.cassandra.SchemaLoader; import org.apache.cassandra.Util; import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.repair.SharedContext; import org.apache.cassandra.cql3.QueryProcessor; import org.apache.cassandra.cql3.statements.schema.CreateTableStatement; import org.apache.cassandra.db.ColumnFamilyStore; @@ -90,7 +91,7 @@ public class PendingRepairStatTest extends AbstractRepairTest static LocalSession createSession() { - LocalSession.Builder builder = LocalSession.builder(); + LocalSession.Builder builder = LocalSession.builder(SharedContext.Global.instance); builder.withState(PREPARING); builder.withSessionID(nextTimeUUID()); builder.withCoordinator(COORDINATOR); diff --git a/test/unit/org/apache/cassandra/service/ActiveRepairServiceTest.java b/test/unit/org/apache/cassandra/service/ActiveRepairServiceTest.java index 93fd9f289a..3b7d0cd9a0 100644 --- a/test/unit/org/apache/cassandra/service/ActiveRepairServiceTest.java +++ b/test/unit/org/apache/cassandra/service/ActiveRepairServiceTest.java @@ -73,7 +73,6 @@ import static org.apache.cassandra.repair.messages.RepairOption.HOSTS_KEY; import static org.apache.cassandra.repair.messages.RepairOption.INCREMENTAL_KEY; import static org.apache.cassandra.repair.messages.RepairOption.RANGES_KEY; import static org.apache.cassandra.service.ActiveRepairService.UNREPAIRED_SSTABLE; -import static org.apache.cassandra.service.ActiveRepairService.getRepairedAt; import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID; import static org.apache.cassandra.utils.concurrent.Condition.newOneTimeCondition; import static org.junit.Assert.assertEquals; @@ -134,7 +133,7 @@ public class ActiveRepairServiceTest Set neighbors = new HashSet<>(); for (Range range : ranges) { - neighbors.addAll(ActiveRepairService.getNeighbors(KEYSPACE5, ranges, range, null, null).endpoints()); + neighbors.addAll(ActiveRepairService.instance().getNeighbors(KEYSPACE5, ranges, range, null, null).endpoints()); } assertEquals(expected, neighbors); } @@ -157,7 +156,7 @@ public class ActiveRepairServiceTest Set neighbors = new HashSet<>(); for (Range range : ranges) { - neighbors.addAll(ActiveRepairService.getNeighbors(KEYSPACE5, ranges, range, null, null).endpoints()); + neighbors.addAll(ActiveRepairService.instance().getNeighbors(KEYSPACE5, ranges, range, null, null).endpoints()); } assertEquals(expected, neighbors); } @@ -179,7 +178,7 @@ public class ActiveRepairServiceTest Set neighbors = new HashSet<>(); for (Range range : ranges) { - neighbors.addAll(ActiveRepairService.getNeighbors(KEYSPACE5, ranges, range, Arrays.asList(DatabaseDescriptor.getLocalDataCenter()), null).endpoints()); + neighbors.addAll(ActiveRepairService.instance().getNeighbors(KEYSPACE5, ranges, range, Arrays.asList(DatabaseDescriptor.getLocalDataCenter()), null).endpoints()); } assertEquals(expected, neighbors); } @@ -207,7 +206,7 @@ public class ActiveRepairServiceTest Set neighbors = new HashSet<>(); for (Range range : ranges) { - neighbors.addAll(ActiveRepairService.getNeighbors(KEYSPACE5, ranges, range, Arrays.asList(DatabaseDescriptor.getLocalDataCenter()), null).endpoints()); + neighbors.addAll(ActiveRepairService.instance().getNeighbors(KEYSPACE5, ranges, range, Arrays.asList(DatabaseDescriptor.getLocalDataCenter()), null).endpoints()); } assertEquals(expected, neighbors); } @@ -230,7 +229,7 @@ public class ActiveRepairServiceTest Collection hosts = Arrays.asList(FBUtilities.getBroadcastAddressAndPort().getHostAddressAndPort(),expected.get(0).getHostAddressAndPort()); Iterable> ranges = StorageService.instance.getLocalReplicas(KEYSPACE5).ranges(); - assertEquals(expected.get(0), ActiveRepairService.getNeighbors(KEYSPACE5, ranges, + assertEquals(expected.get(0), ActiveRepairService.instance().getNeighbors(KEYSPACE5, ranges, ranges.iterator().next(), null, hosts).endpoints().iterator().next()); } @@ -242,14 +241,14 @@ public class ActiveRepairServiceTest //Dont give local endpoint Collection hosts = Arrays.asList("127.0.0.3"); Iterable> ranges = StorageService.instance.getLocalReplicas(KEYSPACE5).ranges(); - ActiveRepairService.getNeighbors(KEYSPACE5, ranges, ranges.iterator().next(), null, hosts); + ActiveRepairService.instance().getNeighbors(KEYSPACE5, ranges, ranges.iterator().next(), null, hosts); } @Test public void testParentRepairStatus() throws Throwable { - ActiveRepairService.instance.recordRepairStatus(1, ActiveRepairService.ParentRepairStatus.COMPLETED, ImmutableList.of("foo", "bar")); + ActiveRepairService.instance().recordRepairStatus(1, ActiveRepairService.ParentRepairStatus.COMPLETED, ImmutableList.of("foo", "bar")); List res = StorageService.instance.getParentRepairStatus(1); assertNotNull(res); assertEquals(ActiveRepairService.ParentRepairStatus.COMPLETED, ActiveRepairService.ParentRepairStatus.valueOf(res.get(0))); @@ -259,7 +258,7 @@ public class ActiveRepairServiceTest List emptyRes = StorageService.instance.getParentRepairStatus(44); assertNull(emptyRes); - ActiveRepairService.instance.recordRepairStatus(3, ActiveRepairService.ParentRepairStatus.FAILED, ImmutableList.of("some failure message", "bar")); + ActiveRepairService.instance().recordRepairStatus(3, ActiveRepairService.ParentRepairStatus.FAILED, ImmutableList.of("some failure message", "bar")); List failed = StorageService.instance.getParentRepairStatus(3); assertNotNull(failed); assertEquals(ActiveRepairService.ParentRepairStatus.FAILED, ActiveRepairService.ParentRepairStatus.valueOf(failed.get(0))); @@ -286,16 +285,16 @@ public class ActiveRepairServiceTest TimeUUID prsId = nextTimeUUID(); Set original = Sets.newHashSet(store.select(View.select(SSTableSet.CANONICAL, (s) -> !s.isRepaired())).sstables); Collection> ranges = Collections.singleton(new Range<>(store.getPartitioner().getMinimumToken(), store.getPartitioner().getMinimumToken())); - ActiveRepairService.instance.registerParentRepairSession(prsId, FBUtilities.getBroadcastAddressAndPort(), Collections.singletonList(store), - ranges, true, System.currentTimeMillis(), true, PreviewKind.NONE); + ActiveRepairService.instance().registerParentRepairSession(prsId, FBUtilities.getBroadcastAddressAndPort(), Collections.singletonList(store), + ranges, true, System.currentTimeMillis(), true, PreviewKind.NONE); store.getRepairManager().snapshot(prsId.toString(), ranges, false); TimeUUID prsId2 = nextTimeUUID(); - ActiveRepairService.instance.registerParentRepairSession(prsId2, FBUtilities.getBroadcastAddressAndPort(), - Collections.singletonList(store), - ranges, - true, System.currentTimeMillis(), - true, PreviewKind.NONE); + ActiveRepairService.instance().registerParentRepairSession(prsId2, FBUtilities.getBroadcastAddressAndPort(), + Collections.singletonList(store), + ranges, + true, System.currentTimeMillis(), + true, PreviewKind.NONE); createSSTables(store, 2); store.getRepairManager().snapshot(prsId.toString(), ranges, false); try (Refs refs = store.getSnapshotSSTableReaders(prsId.toString())) @@ -355,25 +354,25 @@ public class ActiveRepairServiceTest public void repairedAt() throws Exception { // regular incremental repair - Assert.assertNotEquals(UNREPAIRED_SSTABLE, getRepairedAt(opts(INCREMENTAL_KEY, b2s(true)), false)); + Assert.assertNotEquals(UNREPAIRED_SSTABLE, ActiveRepairService.instance().getRepairedAt(opts(INCREMENTAL_KEY, b2s(true)), false)); // subrange incremental repair - Assert.assertNotEquals(UNREPAIRED_SSTABLE, getRepairedAt(opts(INCREMENTAL_KEY, b2s(true), + Assert.assertNotEquals(UNREPAIRED_SSTABLE, ActiveRepairService.instance().getRepairedAt(opts(INCREMENTAL_KEY, b2s(true), RANGES_KEY, "1:2"), false)); // hosts incremental repair - Assert.assertEquals(UNREPAIRED_SSTABLE, getRepairedAt(opts(INCREMENTAL_KEY, b2s(true), + Assert.assertEquals(UNREPAIRED_SSTABLE, ActiveRepairService.instance().getRepairedAt(opts(INCREMENTAL_KEY, b2s(true), HOSTS_KEY, "127.0.0.1"), false)); // dc incremental repair - Assert.assertEquals(UNREPAIRED_SSTABLE, getRepairedAt(opts(INCREMENTAL_KEY, b2s(true), + Assert.assertEquals(UNREPAIRED_SSTABLE, ActiveRepairService.instance().getRepairedAt(opts(INCREMENTAL_KEY, b2s(true), DATACENTERS_KEY, "DC2"), false)); // forced incremental repair - Assert.assertNotEquals(UNREPAIRED_SSTABLE, getRepairedAt(opts(INCREMENTAL_KEY, b2s(true), + Assert.assertNotEquals(UNREPAIRED_SSTABLE, ActiveRepairService.instance().getRepairedAt(opts(INCREMENTAL_KEY, b2s(true), FORCE_REPAIR_KEY, b2s(true)), false)); - Assert.assertEquals(UNREPAIRED_SSTABLE, getRepairedAt(opts(INCREMENTAL_KEY, b2s(true), + Assert.assertEquals(UNREPAIRED_SSTABLE, ActiveRepairService.instance().getRepairedAt(opts(INCREMENTAL_KEY, b2s(true), FORCE_REPAIR_KEY, b2s(true)), true)); // full repair - Assert.assertEquals(UNREPAIRED_SSTABLE, getRepairedAt(opts(INCREMENTAL_KEY, b2s(false)), false)); + Assert.assertEquals(UNREPAIRED_SSTABLE, ActiveRepairService.instance().getRepairedAt(opts(INCREMENTAL_KEY, b2s(false)), false)); } @Test @@ -473,7 +472,7 @@ public class ActiveRepairServiceTest @Test public void testRepairSessionSpaceInMiB() { - ActiveRepairService activeRepairService = ActiveRepairService.instance; + ActiveRepairService activeRepairService = ActiveRepairService.instance(); int previousSize = activeRepairService.getRepairSessionSpaceInMiB(); try { diff --git a/test/unit/org/apache/cassandra/service/SerializationsTest.java b/test/unit/org/apache/cassandra/service/SerializationsTest.java index f9f4766f23..81d310932f 100644 --- a/test/unit/org/apache/cassandra/service/SerializationsTest.java +++ b/test/unit/org/apache/cassandra/service/SerializationsTest.java @@ -51,6 +51,7 @@ import org.apache.cassandra.schema.TableId; import org.apache.cassandra.streaming.PreviewKind; import org.apache.cassandra.streaming.SessionSummary; import org.apache.cassandra.streaming.StreamSummary; +import org.apache.cassandra.utils.Clock; import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.MerkleTrees; import org.apache.cassandra.utils.TimeUUID; @@ -120,7 +121,7 @@ public class SerializationsTest extends AbstractSerializationsTester // empty validation mts.addMerkleTree((int) Math.pow(2, 15), FULL_RANGE); - Validator v0 = new Validator(new ValidationState(DESC, FBUtilities.getBroadcastAddressAndPort()), -1, PreviewKind.NONE); + Validator v0 = new Validator(new ValidationState(Clock.Global.clock(), DESC, FBUtilities.getBroadcastAddressAndPort()), -1, PreviewKind.NONE); ValidationResponse c0 = new ValidationResponse(DESC, mts); // validation with a tree @@ -128,7 +129,7 @@ public class SerializationsTest extends AbstractSerializationsTester mts.addMerkleTree(Integer.MAX_VALUE, FULL_RANGE); for (int i = 0; i < 10; i++) mts.split(p.getRandomToken()); - Validator v1 = new Validator(new ValidationState(DESC, FBUtilities.getBroadcastAddressAndPort()), -1, PreviewKind.NONE); + Validator v1 = new Validator(new ValidationState(Clock.Global.clock(), DESC, FBUtilities.getBroadcastAddressAndPort()), -1, PreviewKind.NONE); ValidationResponse c1 = new ValidationResponse(DESC, mts); // validation failed diff --git a/test/unit/org/apache/cassandra/tools/JMXCompatabilityTest.java b/test/unit/org/apache/cassandra/tools/JMXCompatabilityTest.java index e37c574713..ee98545716 100644 --- a/test/unit/org/apache/cassandra/tools/JMXCompatabilityTest.java +++ b/test/unit/org/apache/cassandra/tools/JMXCompatabilityTest.java @@ -19,16 +19,20 @@ package org.apache.cassandra.tools; import java.util.List; +import java.util.Map; +import com.google.common.collect.ImmutableMap; import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.Test; import org.junit.rules.TemporaryFolder; import com.datastax.driver.core.SimpleStatement; +import org.apache.cassandra.config.CassandraRelevantProperties; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.cql3.CQLTester; import org.apache.cassandra.io.sstable.format.bti.BtiFormat; +import org.apache.cassandra.service.ActiveRepairService; import org.apache.cassandra.service.CassandraDaemon; import org.apache.cassandra.service.GCInspector; import org.apache.cassandra.tools.ToolRunner.ToolResult; @@ -58,6 +62,8 @@ import static com.google.common.collect.Lists.newArrayList; */ public class JMXCompatabilityTest extends CQLTester { + private static final Map ENV = ImmutableMap.of("JAVA_HOME", CassandraRelevantProperties.JAVA_HOME.getString()); + @ClassRule public static TemporaryFolder TMP = new TemporaryFolder(); @@ -70,6 +76,8 @@ public class JMXCompatabilityTest extends CQLTester DatabaseDescriptor.setColumnIndexSizeInKiB(0); // make sure the column index is created startJMXServer(); + // as of CASSANDRA-18816 the instance is lazy loaded only once instance() is called, which isn't true from this code path (but is from CassandraDaemon) + ActiveRepairService.instance(); } private void setupStandardTables() throws Throwable @@ -88,7 +96,7 @@ public class JMXCompatabilityTest extends CQLTester executeNet(ProtocolVersion.CURRENT, new SimpleStatement("SELECT * FROM " + name + " WHERE pk=?", 42)); String script = "tools/bin/jmxtool dump -f yaml --url service:jmx:rmi:///jndi/rmi://" + jmxHost + ":" + jmxPort + "/jmxrmi > " + TMP.getRoot().getAbsolutePath() + "/out.yaml"; - ToolRunner.invoke("bash", "-c", script).assertOnCleanExit(); + ToolRunner.invoke(ENV, "bash", "-c", script).assertOnCleanExit(); CREATED_TABLE = true; } @@ -222,7 +230,7 @@ public class JMXCompatabilityTest extends CQLTester args.add("--exclude-operation"); args.add(a); }); - ToolResult result = ToolRunner.invoke(args); + ToolResult result = ToolRunner.invoke(ENV, args); result.assertOnCleanExit(); Assertions.assertThat(result.getStdout()).isEmpty(); } diff --git a/test/unit/org/apache/cassandra/tools/ToolRunner.java b/test/unit/org/apache/cassandra/tools/ToolRunner.java index 7b69b39a1c..812377c53e 100644 --- a/test/unit/org/apache/cassandra/tools/ToolRunner.java +++ b/test/unit/org/apache/cassandra/tools/ToolRunner.java @@ -196,6 +196,11 @@ public class ToolRunner return invoke(args.toArray(new String[args.size()])); } + public static ToolResult invoke(Map env, List args) + { + return invoke(env, args.toArray(new String[args.size()])); + } + public static ToolResult invoke(String... args) { try (ObservableTool t = invokeAsync(args)) @@ -204,11 +209,24 @@ public class ToolRunner } } + public static ToolResult invoke(Map env, String... args) + { + try (ObservableTool t = invokeAsync(env, args)) + { + return t.waitComplete(); + } + } + public static ObservableTool invokeAsync(String... args) { return invokeAsync(Collections.emptyMap(), null, Arrays.asList(args)); } + public static ObservableTool invokeAsync(Map env, String... args) + { + return invokeAsync(env, null, Arrays.asList(args)); + } + public static ToolResult invoke(Map env, InputStream stdin, List args) { try (ObservableTool t = invokeAsync(env, stdin, args)) diff --git a/test/unit/org/apache/cassandra/utils/CassandraGenerators.java b/test/unit/org/apache/cassandra/utils/CassandraGenerators.java index 49701859b9..c6ab6b9ac4 100644 --- a/test/unit/org/apache/cassandra/utils/CassandraGenerators.java +++ b/test/unit/org/apache/cassandra/utils/CassandraGenerators.java @@ -62,6 +62,7 @@ import org.apache.cassandra.dht.LocalPartitioner; import org.apache.cassandra.dht.Murmur3Partitioner; import org.apache.cassandra.dht.OrderPreservingPartitioner; import org.apache.cassandra.dht.RandomPartitioner; +import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; import org.apache.cassandra.gms.ApplicationState; import org.apache.cassandra.gms.EndpointState; @@ -101,6 +102,11 @@ public final class CassandraGenerators private static final Gen NETWORK_PORT_GEN = SourceDSL.integers().between(0, 0xFFFF); private static final Gen BOOLEAN_GEN = SourceDSL.booleans().all(); + /** + * Similar to {@link Generators#IDENTIFIER_GEN} but uses a bound of 48 as keyspace has a smaller restriction than other identifiers + */ + public static final Gen KEYSPACE_NAME_GEN = Generators.regexWord(SourceDSL.integers().between(1, 48)); + public static final Gen INET_ADDRESS_AND_PORT_GEN = rnd -> { InetAddress address = Generators.INET_ADDRESS_GEN.generate(rnd); return InetAddressAndPort.getByAddressOverrideDefaults(address, NETWORK_PORT_GEN.generate(rnd)); @@ -113,6 +119,7 @@ public final class CassandraGenerators RandomPartitioner.instance); + public static final Gen TABLE_ID_GEN = Generators.UUID_RANDOM_GEN.map(TableId::fromUUID); private static final Gen TABLE_KIND_GEN = SourceDSL.arbitrary().pick(TableMetadata.Kind.REGULAR, TableMetadata.Kind.INDEX, TableMetadata.Kind.VIRTUAL); public static final Gen TABLE_METADATA_GEN = gen(rnd -> createTableMetadata(IDENTIFIER_GEN.generate(rnd), rnd)).describedAs(CassandraGenerators::toStringRecursive); @@ -180,12 +187,14 @@ public final class CassandraGenerators public static class TableMetadataBuilder { - private Gen ksNameGen = IDENTIFIER_GEN; + private Gen ksNameGen = CassandraGenerators.KEYSPACE_NAME_GEN; + private Gen tableNameGen = IDENTIFIER_GEN; private Gen> defaultTypeGen = AbstractTypeGenerators.builder() .withDefaultSetKey(AbstractTypeGenerators.withoutUnsafeEquality()) .withMaxDepth(1) .build(); private Gen> partitionColTypeGen, clusteringColTypeGen, staticColTypeGen, regularColTypeGen; + private Gen tableIdGen = TABLE_ID_GEN; private Gen tableKindGen = SourceDSL.arbitrary().constant(TableMetadata.Kind.REGULAR); private Gen numPartitionColumnsGen = SourceDSL.integers().between(1, 2); private Gen numClusteringColumnsGen = SourceDSL.integers().between(1, 2); @@ -214,6 +223,30 @@ public final class CassandraGenerators return this; } + public TableMetadataBuilder withTableName(Gen tableNameGen) + { + this.tableNameGen = tableNameGen; + return this; + } + + public TableMetadataBuilder withTableName(String name) + { + this.tableNameGen = SourceDSL.arbitrary().constant(name); + return this; + } + + public TableMetadataBuilder withTableId(Gen gen) + { + this.tableIdGen = gen; + return this; + } + + public TableMetadataBuilder withTableId(TableId id) + { + this.tableIdGen = SourceDSL.arbitrary().constant(id); + return this; + } + public TableMetadataBuilder withPartitionColumnsCount(int num) { this.numPartitionColumnsGen = SourceDSL.arbitrary().constant(num); @@ -316,11 +349,11 @@ public final class CassandraGenerators withPrimaryColumnTypeGen(Generators.filter(defaultTypeGen, t -> !AbstractTypeGenerators.UNSAFE_EQUALITY.contains(t.getClass()))); String ks = ksNameGen.generate(rnd); - String tableName = IDENTIFIER_GEN.generate(rnd); + String tableName = tableNameGen.generate(rnd); TableParams.Builder params = TableParams.builder(); if (memtableKeyGen != null) params.memtable(MemtableParams.get(memtableKeyGen.generate(rnd))); - TableMetadata.Builder builder = TableMetadata.builder(ks, tableName, TableId.fromUUID(Generators.UUID_RANDOM_GEN.generate(rnd))) + TableMetadata.Builder builder = TableMetadata.builder(ks, tableName, tableIdGen.generate(rnd)) .partitioner(PARTITIONER_GEN.generate(rnd)) .kind(tableKindGen.generate(rnd)) .isCounter(BOOLEAN_GEN.generate(rnd)) @@ -518,6 +551,28 @@ public final class CassandraGenerators return rs -> new Murmur3Partitioner.LongToken(rs.next(token)); } + public static Gen murmurTokenIn(Range range) + { + // left exclusive, right inclusive + if (range.isWrapAround()) + { + List> unwrap = range.unwrap(); + return rs -> { + Range subRange = unwrap.get(Math.toIntExact(rs.next(Constraint.between(0, unwrap.size() - 1)))); + long end = ((Murmur3Partitioner.LongToken) subRange.right).token; + if (end == Long.MIN_VALUE) + end = Long.MAX_VALUE; + Constraint token = Constraint.between(((Murmur3Partitioner.LongToken) subRange.left).token + 1, end); + return new Murmur3Partitioner.LongToken(rs.next(token)); + }; + } + else + { + Constraint token = Constraint.between(((Murmur3Partitioner.LongToken) range.left).token + 1, ((Murmur3Partitioner.LongToken) range.right).token); + return rs -> new Murmur3Partitioner.LongToken(rs.next(token)); + } + } + public static Gen byteOrderToken() { Constraint size = Constraint.between(0, 10); @@ -548,6 +603,13 @@ public final class CassandraGenerators return rs -> new OrderPreservingPartitioner.StringToken(string.generate(rs)); } + public static Gen tokensInRange(Range range) + { + IPartitioner partitioner = range.left.getPartitioner(); + if (partitioner instanceof Murmur3Partitioner) return murmurTokenIn(range); + throw new UnsupportedOperationException("Unsupported partitioner: " + partitioner.getClass()); + } + public static Gen token(IPartitioner partitioner) { if (partitioner instanceof Murmur3Partitioner) return murmurToken(); diff --git a/test/unit/org/quicktheories/impl/JavaRandom.java b/test/unit/org/quicktheories/impl/JavaRandom.java index b766033996..a1e4a780e8 100644 --- a/test/unit/org/quicktheories/impl/JavaRandom.java +++ b/test/unit/org/quicktheories/impl/JavaRandom.java @@ -18,6 +18,7 @@ package org.quicktheories.impl; +import java.util.Objects; import java.util.Random; import org.quicktheories.core.DetatchedRandomnessSource; @@ -36,6 +37,11 @@ public class JavaRandom implements RandomnessSource, DetatchedRandomnessSource this.random = new Random(seed); } + public JavaRandom(Random random) + { + this.random = Objects.requireNonNull(random); + } + public void setSeed(long seed) { this.random.setSeed(seed);