From d2923275e360a1ee9db498e748c269f701bb3a8b Mon Sep 17 00:00:00 2001 From: Benedict Elliott Smith Date: Mon, 30 Mar 2020 15:24:12 +0100 Subject: [PATCH] CEP-14: Paxos Improvements This work encompasses a range of improvements to Paxos, summarised as: - The introduction of Paxos Repair for ensuring consistency during range movements - The reduction of network round-trips by a factor of 2x for reads and writes patch by Benedict Elliott Smith and Blake Eggleston; reviewed by Alex Petrov, Aleksey Yeschenko and Sam Tunnicliffe for CASSANDRA-17164 --- .build/build-rat.xml | 1 + .circleci/config.yml | 6 +- .circleci/config.yml.HIGHRES | 6 +- .circleci/config.yml.LOWRES | 6 +- .circleci/config.yml.MIDRES | 6 +- NEWS.txt | 24 + build.xml | 42 +- .../cassandra/batchlog/BatchlogManager.java | 13 +- .../cassandra/concurrent/ExecutorFactory.java | 26 +- .../cassandra/concurrent/FutureTask.java | 4 +- .../concurrent/InfiniteLoopExecutor.java | 69 +- .../apache/cassandra/concurrent/Stage.java | 14 +- .../cassandra/concurrent/SyncFutureTask.java | 3 +- .../config/CassandraRelevantProperties.java | 15 +- .../org/apache/cassandra/config/Config.java | 202 ++- .../cassandra/config/DataStorageSpec.java | 2 +- .../cassandra/config/DatabaseDescriptor.java | 199 ++- .../apache/cassandra/config/DurationSpec.java | 4 +- .../cassandra/config/EncryptionOptions.java | 17 +- .../apache/cassandra/cql3/QueryOptions.java | 16 + .../apache/cassandra/cql3/QueryProcessor.java | 40 +- .../cassandra/cql3/UntypedResultSet.java | 6 + .../cassandra/cql3/functions/TimeFcts.java | 4 +- .../cql3/statements/CQL3CasRequest.java | 13 +- .../statements/ModificationStatement.java | 10 +- .../cql3/statements/SelectStatement.java | 50 +- .../cassandra/db/ColumnFamilyStore.java | 64 +- .../apache/cassandra/db/ConsistencyLevel.java | 6 +- .../apache/cassandra/db/CounterMutation.java | 7 + .../org/apache/cassandra/db/IMutation.java | 2 + .../org/apache/cassandra/db/Keyspace.java | 5 + .../org/apache/cassandra/db/Mutation.java | 25 +- .../cassandra/db/MutationVerbHandler.java | 16 +- .../db/PartitionRangeReadCommand.java | 4 +- .../org/apache/cassandra/db/ReadQuery.java | 8 +- .../org/apache/cassandra/db/ReadResponse.java | 12 + .../apache/cassandra/db/SimpleBuilders.java | 1 - .../db/SinglePartitionReadCommand.java | 17 +- .../apache/cassandra/db/SystemKeyspace.java | 234 ++- .../db/commitlog/CommitLogReplayer.java | 1 - .../AbstractCompactionStrategy.java | 1 - .../CompactionInterruptedException.java | 10 +- .../db/compaction/CompactionIterator.java | 96 +- .../db/marshal/ByteArrayAccessor.java | 7 + .../db/marshal/ByteBufferAccessor.java | 7 + .../cassandra/db/marshal/TupleType.java | 14 +- .../cassandra/db/marshal/ValueAccessor.java | 4 + .../db/partitions/AbstractBTreePartition.java | 44 +- .../db/partitions/PartitionUpdate.java | 49 +- .../cassandra/db/rows/AbstractCell.java | 5 + .../apache/cassandra/db/rows/BTreeRow.java | 12 + .../apache/cassandra/db/rows/BufferCell.java | 9 - .../org/apache/cassandra/db/rows/Cell.java | 4 + .../apache/cassandra/db/rows/CellPath.java | 2 + .../apache/cassandra/db/rows/ColumnData.java | 3 + .../cassandra/db/rows/ComplexColumnData.java | 13 +- .../apache/cassandra/db/rows/NativeCell.java | 1 - .../org/apache/cassandra/db/rows/Row.java | 8 +- .../rows/UnfilteredRowIteratorSerializer.java | 2 +- .../apache/cassandra/db/view/ViewUtils.java | 1 - .../db/virtual/StreamingVirtualTable.java | 6 +- .../cassandra/db/virtual/VirtualMutation.java | 8 + .../cassandra/dht/Murmur3Partitioner.java | 11 +- src/java/org/apache/cassandra/dht/Range.java | 2 +- .../exceptions/CasWriteTimeoutException.java | 3 +- .../exceptions/RequestFailureException.java | 2 +- .../exceptions/RequestFailureReason.java | 5 +- .../apache/cassandra/gms/EndpointState.java | 18 +- .../cassandra/gms/GossipDigestAck2.java | 5 - .../gms/GossipDigestAck2VerbHandler.java | 2 +- .../org/apache/cassandra/gms/Gossiper.java | 15 +- .../apache/cassandra/gms/VersionedValue.java | 8 + .../cassandra/hadoop/cql3/CqlInputFormat.java | 2 +- .../org/apache/cassandra/index/Index.java | 22 +- .../index/SecondaryIndexManager.java | 59 +- .../cassandra/io/FSDiskFullWriteError.java | 6 - src/java/org/apache/cassandra/io/FSError.java | 23 +- .../io/FSNoDiskAvailableForWriteError.java | 6 - .../org/apache/cassandra/io/FSReadError.java | 20 +- .../org/apache/cassandra/io/FSWriteError.java | 20 +- .../io/sstable/CQLSSTableWriter.java | 2 +- .../io/sstable/IndexSummaryManager.java | 7 +- .../io/sstable/format/SSTableReader.java | 1 - .../io/sstable/format/big/BigFormat.java | 1 - .../io/sstable/format/big/BigTableWriter.java | 2 +- .../io/sstable/metadata/StatsMetadata.java | 1 - .../io/util/ChecksummedSequentialWriter.java | 2 - .../org/apache/cassandra/io/util/File.java | 187 ++- .../apache/cassandra/io/util/PathUtils.java | 40 +- .../io/util/SequentialWriterOption.java | 1 + .../locator/AbstractReplicaCollection.java | 133 +- .../locator/AbstractReplicationStrategy.java | 18 +- .../apache/cassandra/locator/Endpoints.java | 24 +- .../cassandra/locator/EndpointsForToken.java | 39 + .../{InOurDcTester.java => InOurDc.java} | 31 +- .../cassandra/locator/ReplicaCollection.java | 12 +- .../cassandra/locator/ReplicaLayout.java | 12 +- .../apache/cassandra/locator/ReplicaPlan.java | 171 +-- .../cassandra/locator/ReplicaPlans.java | 36 +- .../apache/cassandra/locator/Replicas.java | 2 +- .../cassandra/locator/TokenMetadata.java | 2 +- .../metrics/ClientRequestsMetricsHolder.java | 4 +- .../DecayingEstimatedHistogramReservoir.java | 51 +- .../metrics/HintedHandoffMetrics.java | 2 +- .../cassandra/metrics/PaxosMetrics.java | 30 + .../org/apache/cassandra/net/Message.java | 31 +- .../cassandra/net/MessagingService.java | 18 + .../cassandra/net/OutboundMessageQueue.java | 1 - .../apache/cassandra/net/RequestCallback.java | 7 + .../RequestCallbackWithFailure.java} | 26 +- .../cassandra/net/RequestCallbacks.java | 22 +- src/java/org/apache/cassandra/net/Verb.java | 47 +- .../cassandra/repair/AbstractRepairTask.java | 2 + .../apache/cassandra/repair/RepairJob.java | 75 +- .../cassandra/repair/RepairRunnable.java | 4 +- .../cassandra/repair/RepairSession.java | 15 +- .../repair/consistent/LocalSessions.java | 1 - .../consistent/admin/CleanupSummary.java | 1 - .../repair/messages/PrepareMessage.java | 1 - .../repair/messages/RepairOption.java | 33 +- .../schema/MigrationCoordinator.java | 3 + .../security/DisableSslContextFactory.java | 55 + .../service/AbstractWriteResponseHandler.java | 25 +- .../service/ActiveRepairService.java | 94 +- .../service/ActiveRepairServiceMBean.java | 2 + .../service/BatchlogResponseHandler.java | 2 +- .../apache/cassandra/service/CASRequest.java | 8 +- .../cassandra/service/CassandraDaemon.java | 13 + .../apache/cassandra/service/ClientState.java | 19 +- .../DatacenterSyncWriteResponseHandler.java | 7 +- .../DatacenterWriteResponseHandler.java | 11 +- .../service/FailureRecordingCallback.java | 153 ++ .../PendingRangeCalculatorService.java | 6 + .../cassandra/service/StorageProxy.java | 175 ++- .../cassandra/service/StorageProxyMBean.java | 6 + .../cassandra/service/StorageService.java | 332 ++++- .../service/StorageServiceMBean.java | 42 + .../service/WriteResponseHandler.java | 11 +- .../service/paxos/AbstractPaxosRepair.java | 290 ++++ .../cassandra/service/paxos/Ballot.java | 151 ++ .../service/paxos/BallotGenerator.java | 41 +- .../cassandra/service/paxos/Commit.java | 399 +++++- .../service/paxos/CommitVerbHandler.java | 39 - .../service/paxos/ContentionStrategy.java | 651 +++++++++ .../apache/cassandra/service/paxos/Paxos.java | 1199 +++++++++++++++- .../apache/cassandra/service/paxos/Paxos.md | 345 +++++ .../cassandra/service/paxos/PaxosCommit.java | 325 +++++ .../service/paxos/PaxosCommitAndPrepare.java | 145 ++ .../service/paxos/PaxosOperationLock.java | 15 +- .../cassandra/service/paxos/PaxosPrepare.java | 1253 +++++++++++++++++ .../service/paxos/PaxosPrepareRefresh.java | 245 ++++ .../cassandra/service/paxos/PaxosPropose.java | 479 +++++++ .../cassandra/service/paxos/PaxosRepair.java | 706 ++++++++++ .../service/paxos/PaxosRepairHistory.java | 480 +++++++ .../service/paxos/PaxosRequestCallback.java | 76 + .../cassandra/service/paxos/PaxosState.java | 877 ++++++++++-- .../paxos/TablePaxosRepairHistory.java | 80 ++ .../service/paxos/cleanup/PaxosCleanup.java | 156 ++ .../paxos/cleanup/PaxosCleanupComplete.java | 144 ++ .../paxos/cleanup/PaxosCleanupException.java | 27 + .../paxos/cleanup/PaxosCleanupHistory.java | 68 + .../cleanup/PaxosCleanupLocalCoordinator.java | 189 +++ .../paxos/cleanup/PaxosCleanupRequest.java | 142 ++ .../paxos/cleanup/PaxosCleanupResponse.java | 87 ++ .../paxos/cleanup/PaxosCleanupSession.java | 267 ++++ .../cleanup/PaxosFinishPrepareCleanup.java | 169 +++ .../cleanup/PaxosStartPrepareCleanup.java | 192 +++ .../paxos/cleanup/PaxosTableRepairs.java | 235 ++++ .../paxos/uncommitted/PaxosBallotTracker.java | 207 +++ .../paxos/uncommitted/PaxosKeyState.java | 168 +++ .../service/paxos/uncommitted/PaxosRows.java | 359 +++++ .../paxos/uncommitted/PaxosStateTracker.java | 322 +++++ .../uncommitted/PaxosUncommittedIndex.java | 264 ++++ .../uncommitted/PaxosUncommittedTracker.java | 376 +++++ .../uncommitted/UncommittedDataFile.java | 384 +++++ .../uncommitted/UncommittedPaxosKey.java | 30 + .../uncommitted/UncommittedTableData.java | 617 ++++++++ .../paxos/{ => v1}/AbstractPaxosCallback.java | 23 +- .../paxos/{ => v1}/PrepareCallback.java | 54 +- .../paxos/{ => v1}/PrepareVerbHandler.java | 27 +- .../paxos/{ => v1}/ProposeCallback.java | 22 +- .../paxos/{ => v1}/ProposeVerbHandler.java | 29 +- .../service/reads/AbstractReadExecutor.java | 6 +- .../cassandra/service/reads/DataResolver.java | 30 +- .../service/reads/DigestResolver.java | 2 +- .../cassandra/service/reads/ReadCallback.java | 4 +- .../service/reads/ResponseResolver.java | 11 +- .../reads/ShortReadPartitionsProtection.java | 2 +- .../reads/repair/AbstractReadRepair.java | 5 +- .../reads/repair/BlockingPartitionRepair.java | 14 +- .../reads/repair/BlockingReadRepair.java | 4 +- .../service/reads/repair/NoopReadRepair.java | 4 +- .../PartitionIteratorMergeListener.java | 4 +- .../reads/repair/ReadOnlyReadRepair.java | 4 +- .../service/reads/repair/ReadRepair.java | 8 +- .../reads/repair/ReadRepairDiagnostics.java | 8 +- .../reads/repair/ReadRepairStrategy.java | 4 +- .../repair/RowIteratorMergeListener.java | 6 +- .../cassandra/streaming/StreamManager.java | 2 +- .../streaming/StreamTransferTask.java | 6 +- .../async/StreamCompressionSerializer.java | 1 - .../cassandra/tools/nodetool/Repair.java | 8 + .../cassandra/tracing/TraceStateImpl.java | 2 - .../cassandra/utils/BiLongAccumulator.java | 2 +- .../cassandra/utils/CassandraVersion.java | 26 +- .../cassandra/utils/CloseableIterator.java | 45 + .../cassandra/utils/CollectionSerializer.java | 125 ++ .../cassandra/utils/EstimatedHistogram.java | 9 +- .../cassandra/utils/MonotonicClock.java | 4 +- .../org/apache/cassandra/utils/Nemesis.java | 2 +- .../cassandra/utils/NullableSerializer.java | 70 + .../org/apache/cassandra/utils/TimeUUID.java | 35 +- .../org/apache/cassandra/utils/UUIDGen.java | 1 + .../cassandra/utils/VoidSerializer.java | 34 + .../utils/binlog/ExternalArchiver.java | 2 +- .../utils/concurrent/AbstractFuture.java | 1 - .../utils/concurrent/ConditionAsConsumer.java | 34 + .../utils/concurrent/IntrusiveStack.java | 58 +- .../cassandra/utils/concurrent/Ref.java | 3 +- .../utils/concurrent/SyncFuture.java | 2 +- .../utils/concurrent/SyncPromise.java | 269 ++++ .../cassandra/utils/memory/BufferPool.java | 3 +- .../utils/memory/LongBufferPoolTest.java | 1 + test/conf/cassandra.yaml | 2 +- test/conf/logback-simulator.xml | 4 + .../serialization/4.1/gms.EndpointState.bin | Bin 0 -> 73 bytes test/data/serialization/4.1/gms.Gossip.bin | Bin 0 -> 166 bytes .../4.1/service.SyncComplete.bin | Bin 0 -> 256 bytes .../serialization/4.1/service.SyncRequest.bin | Bin 0 -> 111 bytes .../4.1/service.ValidationComplete.bin | Bin 0 -> 597 bytes .../4.1/service.ValidationRequest.bin | Bin 0 -> 74 bytes .../4.1/utils.EstimatedHistogram.bin | Bin 0 -> 97500 bytes .../distributed/api/ConsistencyLevel.java | 41 + .../distributed/api/IClassTransformer.java | 30 + .../distributed/api/ICoordinator.java | 72 + .../cassandra/distributed/api/IMessage.java | 43 + .../distributed/api/QueryResult.java | 86 ++ .../distributed/api/QueryResults.java | 224 +++ .../distributed/api/SimpleQueryResult.java | 163 +++ .../distributed/impl/AbstractCluster.java | 38 +- .../distributed/impl/Coordinator.java | 38 +- .../impl/DelegatingInvokableInstance.java | 2 +- .../DirectStreamingConnectionFactory.java | 38 +- .../cassandra/distributed/impl/Instance.java | 57 +- .../distributed/impl/IsolatedExecutor.java | 85 +- .../distributed/impl/MessageFilters.java | 207 +++ .../distributed/impl/MessageImpl.java | 10 +- .../distributed/impl/ShutdownExecutor.java | 31 + .../distributed/impl/UnsafeGossipHelper.java | 24 +- .../distributed/test/CASCommonTestCases.java | 204 +++ .../distributed/test/CASContentionTest.java | 110 ++ .../distributed/test/CASMultiDCTest.java | 150 ++ .../cassandra/distributed/test/CASTest.java | 884 ++++++------ .../distributed/test/CASTestBase.java | 221 +++ .../test/CasCriticalSectionTest.java | 218 +++ .../distributed/test/LegacyCASTest.java | 118 ++ .../cassandra/distributed/test/MoveTest.java | 90 ++ .../distributed/test/PaxosRepairTest.java | 620 ++++++++ .../distributed/test/PaxosRepairTest2.java | 612 ++++++++ .../test/PaxosUncommittedIndexTest.java | 54 + .../test/ReadDigestConsistencyTest.java | 3 +- .../distributed/test/ReadRepairTest.java | 2 +- .../distributed/test/SecondaryIndexTest.java | 3 +- .../test/microbench/MessageOutBench.java | 6 +- .../test/microbench/TimedMonitorBench.java | 198 +++ .../simulator/asm/ClassTransformer.java | 81 +- .../apache/cassandra/simulator/asm/Flag.java | 2 +- .../asm/GlobalMethodTransformer.java | 8 + .../simulator/asm/InterceptAgent.java | 15 +- .../simulator/asm/InterceptClasses.java | 216 ++- .../cassandra/simulator/asm/MethodLogger.java | 10 +- .../simulator/asm/ShadowingTransformer.java | 2 +- .../ThreadLocalRandomCheckTransformer.java | 56 + .../apache/cassandra/simulator/asm/Utils.java | 25 + .../systems/InterceptorOfSystemMethods.java | 46 +- .../apache/cassandra/simulator/Action.java | 158 ++- .../cassandra/simulator/ActionList.java | 6 + .../cassandra/simulator/ActionPlan.java | 13 +- .../cassandra/simulator/ActionSchedule.java | 86 +- .../simulator/ClusterSimulation.java | 144 +- .../org/apache/cassandra/simulator/Debug.java | 3 +- .../simulator/FutureActionScheduler.java | 4 +- .../apache/cassandra/simulator/OrderOns.java | 5 + .../apache/cassandra/simulator/Ordered.java | 7 +- .../simulator/RunnableActionScheduler.java | 2 +- .../cassandra/simulator/SimulationRunner.java | 54 +- .../asm/InterceptAsClassTransformer.java | 49 + .../simulator/cluster/ClusterActions.java | 14 +- .../simulator/cluster/KeyspaceActions.java | 10 +- .../cluster/OnClusterChangeTopology.java | 3 +- .../simulator/cluster/OnClusterJoin.java | 1 + .../simulator/cluster/OnClusterLeave.java | 4 +- .../simulator/cluster/OnClusterReplace.java | 1 + .../simulator/cluster/OnInstanceRepair.java | 2 +- .../OnInstanceTopologyChangePaxosRepair.java | 71 + .../cassandra/simulator/debug/Reconcile.java | 134 +- .../cassandra/simulator/debug/Record.java | 121 +- .../simulator/debug/SelfReconcile.java | 76 +- .../cassandra/simulator/package-info.java | 1 + .../cassandra/simulator/paxos/Ballots.java | 62 +- .../paxos/PairOfSequencesPaxosSimulation.java | 20 +- .../paxos/PaxosClusterSimulation.java | 7 +- .../simulator/paxos/PaxosRepairValidator.java | 5 +- .../simulator/paxos/PaxosSimulation.java | 78 +- .../paxos/PaxosSimulationRunner.java | 9 +- .../paxos/PaxosTopologyChangeVerifier.java | 9 +- .../systems/InterceptedExecution.java | 3 +- .../simulator/systems/InterceptedWait.java | 127 +- .../systems/InterceptibleThread.java | 110 +- .../systems/InterceptingAwaitable.java | 45 +- .../systems/InterceptingExecutor.java | 326 ++++- .../systems/InterceptingExecutorFactory.java | 35 +- .../systems/InterceptingGlobalMethods.java | 157 ++- .../systems/InterceptingMonitors.java | 275 +++- .../systems/InterceptingWaitQueue.java | 15 +- .../systems/InterceptorOfConsequences.java | 43 +- .../systems/InterceptorOfExecution.java | 5 +- .../systems/InterceptorOfGlobalMethods.java | 96 ++ .../simulator/systems/InterceptorOfWaits.java | 46 - .../simulator/systems/NonInterceptible.java | 46 +- .../simulator/systems/SimulatedAction.java | 134 +- .../systems/SimulatedActionCallable.java | 1 - .../systems/SimulatedActionTask.java | 27 +- .../simulator/systems/SimulatedBallots.java | 31 +- .../simulator/systems/SimulatedExecution.java | 1 + .../SimulatedFutureActionScheduler.java | 12 +- .../simulator/systems/SimulatedSystems.java | 10 +- .../simulator/systems/SimulatedTime.java | 273 +++- .../simulator/systems/SimulatedWaits.java | 156 -- .../test/ShortPaxosSimulationTest.java | 42 + .../simulator/test/SimulationTestBase.java | 32 +- ...onTest.java => TrivialSimulationTest.java} | 2 +- .../AbstractSerializationsTester.java | 3 +- test/unit/org/apache/cassandra/Util.java | 1 + .../config/DatabaseDescriptorRefTest.java | 2 + .../LoadOldYAMLBackwardCompatibilityTest.java | 2 +- .../config/ParseAndConvertUnitsTest.java | 2 +- .../cassandra/cql3/PstmtPersistenceTest.java | 1 - .../cassandra/db/filter/ColumnFilterTest.java | 1 + .../db/partition/PartitionUpdateTest.java | 2 +- .../index/internal/CassandraIndexTest.java | 8 +- .../metadata/MetadataSerializerTest.java | 3 - .../cassandra/locator/ReplicaPlansTest.java | 4 +- ...cayingEstimatedHistogramReservoirTest.java | 50 +- .../cassandra/net/WriteCallbackInfoTest.java | 5 +- .../cassandra/repair/RepairJobTest.java | 91 +- .../cassandra/repair/RepairSessionTest.java | 2 +- .../messages/RepairMessageSerializerTest.java | 1 - .../service/ActiveRepairServiceTest.java | 1 - .../cassandra/service/PaxosStateTest.java | 177 +++ .../service/WriteResponseHandlerTest.java | 2 +- .../WriteResponseHandlerTransientTest.java | 12 +- .../paxos/AbstractPaxosRepairTest.java | 174 +++ .../service/paxos/ContentionStrategyTest.java | 430 ++++++ .../service/paxos/PaxosProposeTest.java | 97 ++ .../service/paxos/PaxosRepairHistoryTest.java | 531 +++++++ .../service/paxos/PaxosRepairTest.java | 172 +++ .../service/paxos/PaxosStateTest.java | 323 +++++ .../paxos/cleanup/PaxosTableRepairsTest.java | 262 ++++ .../uncommitted/PaxosBallotTrackerTest.java | 258 ++++ .../uncommitted/PaxosMockUpdateSupplier.java | 113 ++ .../paxos/uncommitted/PaxosRowsTest.java | 165 +++ .../uncommitted/PaxosStateTrackerTest.java | 242 ++++ .../uncommitted/PaxosUncommittedTests.java | 94 ++ ...axosUncommittedTrackerIntegrationTest.java | 141 ++ .../PaxosUncommittedTrackerTest.java | 248 ++++ .../uncommitted/UncommittedTableDataTest.java | 650 +++++++++ .../reads/repair/AbstractReadRepairTest.java | 10 +- .../reads/repair/BlockingReadRepairTest.java | 10 +- .../DiagEventsBlockingReadRepairTest.java | 8 +- .../reads/repair/InstrumentedReadRepair.java | 4 +- .../reads/repair/ReadOnlyReadRepairTest.java | 4 +- .../service/reads/repair/ReadRepairTest.java | 6 +- .../reads/repair/TestableReadRepair.java | 4 +- .../cassandra/transport/ErrorMessageTest.java | 6 +- .../cassandra/utils/CassandraVersionTest.java | 12 +- .../concurrent/AbstractTestAsyncPromise.java | 56 +- .../concurrent/AbstractTestAwaitable.java | 40 +- .../utils/concurrent/SemaphoreTest.java | 21 +- 379 files changed, 27090 insertions(+), 2728 deletions(-) rename src/java/org/apache/cassandra/locator/{InOurDcTester.java => InOurDc.java} (71%) create mode 100644 src/java/org/apache/cassandra/metrics/PaxosMetrics.java rename src/java/org/apache/cassandra/{serializers/TimeUUIDSerializer.java => net/RequestCallbackWithFailure.java} (55%) create mode 100644 src/java/org/apache/cassandra/security/DisableSslContextFactory.java create mode 100644 src/java/org/apache/cassandra/service/FailureRecordingCallback.java create mode 100644 src/java/org/apache/cassandra/service/paxos/AbstractPaxosRepair.java create mode 100644 src/java/org/apache/cassandra/service/paxos/Ballot.java delete mode 100644 src/java/org/apache/cassandra/service/paxos/CommitVerbHandler.java create mode 100644 src/java/org/apache/cassandra/service/paxos/ContentionStrategy.java create mode 100644 src/java/org/apache/cassandra/service/paxos/Paxos.md create mode 100644 src/java/org/apache/cassandra/service/paxos/PaxosCommit.java create mode 100644 src/java/org/apache/cassandra/service/paxos/PaxosCommitAndPrepare.java rename test/simulator/main/org/apache/cassandra/simulator/debug/Capture.java => src/java/org/apache/cassandra/service/paxos/PaxosOperationLock.java (69%) create mode 100644 src/java/org/apache/cassandra/service/paxos/PaxosPrepare.java create mode 100644 src/java/org/apache/cassandra/service/paxos/PaxosPrepareRefresh.java create mode 100644 src/java/org/apache/cassandra/service/paxos/PaxosPropose.java create mode 100644 src/java/org/apache/cassandra/service/paxos/PaxosRepair.java create mode 100644 src/java/org/apache/cassandra/service/paxos/PaxosRepairHistory.java create mode 100644 src/java/org/apache/cassandra/service/paxos/PaxosRequestCallback.java create mode 100644 src/java/org/apache/cassandra/service/paxos/TablePaxosRepairHistory.java create mode 100644 src/java/org/apache/cassandra/service/paxos/cleanup/PaxosCleanup.java create mode 100644 src/java/org/apache/cassandra/service/paxos/cleanup/PaxosCleanupComplete.java create mode 100644 src/java/org/apache/cassandra/service/paxos/cleanup/PaxosCleanupException.java create mode 100644 src/java/org/apache/cassandra/service/paxos/cleanup/PaxosCleanupHistory.java create mode 100644 src/java/org/apache/cassandra/service/paxos/cleanup/PaxosCleanupLocalCoordinator.java create mode 100644 src/java/org/apache/cassandra/service/paxos/cleanup/PaxosCleanupRequest.java create mode 100644 src/java/org/apache/cassandra/service/paxos/cleanup/PaxosCleanupResponse.java create mode 100644 src/java/org/apache/cassandra/service/paxos/cleanup/PaxosCleanupSession.java create mode 100644 src/java/org/apache/cassandra/service/paxos/cleanup/PaxosFinishPrepareCleanup.java create mode 100644 src/java/org/apache/cassandra/service/paxos/cleanup/PaxosStartPrepareCleanup.java create mode 100644 src/java/org/apache/cassandra/service/paxos/cleanup/PaxosTableRepairs.java create mode 100644 src/java/org/apache/cassandra/service/paxos/uncommitted/PaxosBallotTracker.java create mode 100644 src/java/org/apache/cassandra/service/paxos/uncommitted/PaxosKeyState.java create mode 100644 src/java/org/apache/cassandra/service/paxos/uncommitted/PaxosRows.java create mode 100644 src/java/org/apache/cassandra/service/paxos/uncommitted/PaxosStateTracker.java create mode 100644 src/java/org/apache/cassandra/service/paxos/uncommitted/PaxosUncommittedIndex.java create mode 100644 src/java/org/apache/cassandra/service/paxos/uncommitted/PaxosUncommittedTracker.java create mode 100644 src/java/org/apache/cassandra/service/paxos/uncommitted/UncommittedDataFile.java create mode 100644 src/java/org/apache/cassandra/service/paxos/uncommitted/UncommittedPaxosKey.java create mode 100644 src/java/org/apache/cassandra/service/paxos/uncommitted/UncommittedTableData.java rename src/java/org/apache/cassandra/service/paxos/{ => v1}/AbstractPaxosCallback.java (81%) rename src/java/org/apache/cassandra/service/paxos/{ => v1}/PrepareCallback.java (56%) rename src/java/org/apache/cassandra/service/paxos/{ => v1}/PrepareVerbHandler.java (60%) rename src/java/org/apache/cassandra/service/paxos/{ => v1}/ProposeCallback.java (86%) rename src/java/org/apache/cassandra/service/paxos/{ => v1}/ProposeVerbHandler.java (56%) create mode 100644 src/java/org/apache/cassandra/utils/CollectionSerializer.java create mode 100644 src/java/org/apache/cassandra/utils/NullableSerializer.java create mode 100644 src/java/org/apache/cassandra/utils/VoidSerializer.java create mode 100644 src/java/org/apache/cassandra/utils/concurrent/ConditionAsConsumer.java create mode 100644 src/java/org/apache/cassandra/utils/concurrent/SyncPromise.java create mode 100644 test/data/serialization/4.1/gms.EndpointState.bin create mode 100644 test/data/serialization/4.1/gms.Gossip.bin create mode 100644 test/data/serialization/4.1/service.SyncComplete.bin create mode 100644 test/data/serialization/4.1/service.SyncRequest.bin create mode 100644 test/data/serialization/4.1/service.ValidationComplete.bin create mode 100644 test/data/serialization/4.1/service.ValidationRequest.bin create mode 100644 test/data/serialization/4.1/utils.EstimatedHistogram.bin create mode 100644 test/distributed/org/apache/cassandra/distributed/api/ConsistencyLevel.java create mode 100644 test/distributed/org/apache/cassandra/distributed/api/IClassTransformer.java create mode 100644 test/distributed/org/apache/cassandra/distributed/api/ICoordinator.java create mode 100644 test/distributed/org/apache/cassandra/distributed/api/IMessage.java create mode 100644 test/distributed/org/apache/cassandra/distributed/api/QueryResult.java create mode 100644 test/distributed/org/apache/cassandra/distributed/api/QueryResults.java create mode 100644 test/distributed/org/apache/cassandra/distributed/api/SimpleQueryResult.java create mode 100644 test/distributed/org/apache/cassandra/distributed/impl/MessageFilters.java create mode 100644 test/distributed/org/apache/cassandra/distributed/impl/ShutdownExecutor.java create mode 100644 test/distributed/org/apache/cassandra/distributed/test/CASCommonTestCases.java create mode 100644 test/distributed/org/apache/cassandra/distributed/test/CASContentionTest.java create mode 100644 test/distributed/org/apache/cassandra/distributed/test/CASMultiDCTest.java create mode 100644 test/distributed/org/apache/cassandra/distributed/test/CASTestBase.java create mode 100644 test/distributed/org/apache/cassandra/distributed/test/CasCriticalSectionTest.java create mode 100644 test/distributed/org/apache/cassandra/distributed/test/LegacyCASTest.java create mode 100644 test/distributed/org/apache/cassandra/distributed/test/MoveTest.java create mode 100644 test/distributed/org/apache/cassandra/distributed/test/PaxosRepairTest.java create mode 100644 test/distributed/org/apache/cassandra/distributed/test/PaxosRepairTest2.java create mode 100644 test/distributed/org/apache/cassandra/distributed/test/PaxosUncommittedIndexTest.java create mode 100644 test/microbench/org/apache/cassandra/test/microbench/TimedMonitorBench.java create mode 100644 test/simulator/asm/org/apache/cassandra/simulator/asm/ThreadLocalRandomCheckTransformer.java create mode 100644 test/simulator/main/org/apache/cassandra/simulator/asm/InterceptAsClassTransformer.java create mode 100644 test/simulator/main/org/apache/cassandra/simulator/cluster/OnInstanceTopologyChangePaxosRepair.java delete mode 100644 test/simulator/main/org/apache/cassandra/simulator/systems/InterceptorOfWaits.java delete mode 100644 test/simulator/main/org/apache/cassandra/simulator/systems/SimulatedWaits.java create mode 100644 test/simulator/test/org/apache/cassandra/simulator/test/ShortPaxosSimulationTest.java rename test/simulator/test/org/apache/cassandra/simulator/test/{SimulationTest.java => TrivialSimulationTest.java} (98%) create mode 100644 test/unit/org/apache/cassandra/service/PaxosStateTest.java create mode 100644 test/unit/org/apache/cassandra/service/paxos/AbstractPaxosRepairTest.java create mode 100644 test/unit/org/apache/cassandra/service/paxos/ContentionStrategyTest.java create mode 100644 test/unit/org/apache/cassandra/service/paxos/PaxosProposeTest.java create mode 100644 test/unit/org/apache/cassandra/service/paxos/PaxosRepairHistoryTest.java create mode 100644 test/unit/org/apache/cassandra/service/paxos/PaxosRepairTest.java create mode 100644 test/unit/org/apache/cassandra/service/paxos/PaxosStateTest.java create mode 100644 test/unit/org/apache/cassandra/service/paxos/cleanup/PaxosTableRepairsTest.java create mode 100644 test/unit/org/apache/cassandra/service/paxos/uncommitted/PaxosBallotTrackerTest.java create mode 100644 test/unit/org/apache/cassandra/service/paxos/uncommitted/PaxosMockUpdateSupplier.java create mode 100644 test/unit/org/apache/cassandra/service/paxos/uncommitted/PaxosRowsTest.java create mode 100644 test/unit/org/apache/cassandra/service/paxos/uncommitted/PaxosStateTrackerTest.java create mode 100644 test/unit/org/apache/cassandra/service/paxos/uncommitted/PaxosUncommittedTests.java create mode 100644 test/unit/org/apache/cassandra/service/paxos/uncommitted/PaxosUncommittedTrackerIntegrationTest.java create mode 100644 test/unit/org/apache/cassandra/service/paxos/uncommitted/PaxosUncommittedTrackerTest.java create mode 100644 test/unit/org/apache/cassandra/service/paxos/uncommitted/UncommittedTableDataTest.java diff --git a/.build/build-rat.xml b/.build/build-rat.xml index 13c4169abb..b30e5cd4d8 100644 --- a/.build/build-rat.xml +++ b/.build/build-rat.xml @@ -74,6 +74,7 @@ + diff --git a/.circleci/config.yml b/.circleci/config.yml index ee3f52c57b..0e42a3d212 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -39,7 +39,7 @@ jobs: echo "***java tests***" # get all of our unit test filenames - set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt + set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" "$HOME/cassandra/test/simulator/test/**/*.java" > /tmp/all_java_unit_tests.txt # split up the unit tests into groups based on the number of containers we have set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt @@ -1723,7 +1723,7 @@ jobs: echo "***java tests***" # get all of our unit test filenames - set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt + set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" "$HOME/cassandra/test/simulator/test/**/*.java" > /tmp/all_java_unit_tests.txt # split up the unit tests into groups based on the number of containers we have set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt @@ -2335,7 +2335,7 @@ jobs: echo "***java tests***" # get all of our unit test filenames - set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt + set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" "$HOME/cassandra/test/simulator/test/**/*.java" > /tmp/all_java_unit_tests.txt # split up the unit tests into groups based on the number of containers we have set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt diff --git a/.circleci/config.yml.HIGHRES b/.circleci/config.yml.HIGHRES index 3d15e4b958..8eb0f1f4bb 100644 --- a/.circleci/config.yml.HIGHRES +++ b/.circleci/config.yml.HIGHRES @@ -39,7 +39,7 @@ jobs: echo "***java tests***" # get all of our unit test filenames - set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt + set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" "$HOME/cassandra/test/simulator/test/**/*.java" > /tmp/all_java_unit_tests.txt # split up the unit tests into groups based on the number of containers we have set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt @@ -1723,7 +1723,7 @@ jobs: echo "***java tests***" # get all of our unit test filenames - set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt + set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" "$HOME/cassandra/test/simulator/test/**/*.java" > /tmp/all_java_unit_tests.txt # split up the unit tests into groups based on the number of containers we have set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt @@ -2335,7 +2335,7 @@ jobs: echo "***java tests***" # get all of our unit test filenames - set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt + set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" "$HOME/cassandra/test/simulator/test/**/*.java" > /tmp/all_java_unit_tests.txt # split up the unit tests into groups based on the number of containers we have set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt diff --git a/.circleci/config.yml.LOWRES b/.circleci/config.yml.LOWRES index ee3f52c57b..1715910e8d 100644 --- a/.circleci/config.yml.LOWRES +++ b/.circleci/config.yml.LOWRES @@ -39,7 +39,7 @@ jobs: echo "***java tests***" # get all of our unit test filenames - set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt + set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" "$HOME/cassandra/test/simulator/test/**/*.java" > /tmp/all_java_unit_tests.txt # split up the unit tests into groups based on the number of containers we have set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt @@ -1723,7 +1723,7 @@ jobs: echo "***java tests***" # get all of our unit test filenames - set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt + set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" "$HOME/cassandra/test/simulator/test/**/*.java" > /tmp/all_java_unit_tests.txt # split up the unit tests into groups based on the number of containers we have set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt @@ -2335,7 +2335,7 @@ jobs: echo "***java tests***" # get all of our unit test filenames - set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt + set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" "$HOME/cassandra/test/simulator/test/**/*.java" > /tmp/all_java_unit_tests.txt # split up the unit tests into groups based on the number of containers we have set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt diff --git a/.circleci/config.yml.MIDRES b/.circleci/config.yml.MIDRES index 981dccfdaf..a750d21f23 100644 --- a/.circleci/config.yml.MIDRES +++ b/.circleci/config.yml.MIDRES @@ -39,7 +39,7 @@ jobs: echo "***java tests***" # get all of our unit test filenames - set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt + set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" "$HOME/cassandra/test/simulator/test/**/*.java" > /tmp/all_java_unit_tests.txt # split up the unit tests into groups based on the number of containers we have set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt @@ -1723,7 +1723,7 @@ jobs: echo "***java tests***" # get all of our unit test filenames - set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt + set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" "$HOME/cassandra/test/simulator/test/**/*.java" > /tmp/all_java_unit_tests.txt # split up the unit tests into groups based on the number of containers we have set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt @@ -2335,7 +2335,7 @@ jobs: echo "***java tests***" # get all of our unit test filenames - set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt + set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" "$HOME/cassandra/test/simulator/test/**/*.java" > /tmp/all_java_unit_tests.txt # split up the unit tests into groups based on the number of containers we have set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt diff --git a/NEWS.txt b/NEWS.txt index 65cce3cbf4..3ed36a7a7e 100644 --- a/NEWS.txt +++ b/NEWS.txt @@ -74,6 +74,18 @@ New features - A new ALL TABLES IN KEYSPACE resource has been added. It allows to grant permissions for all tables and user types in a keyspace while preventing the user to use those permissions on the keyspace itself. - Added support for type casting in the WHERE clause components and in the values of INSERT and UPDATE statements. + - A new implementation of Paxos (named v2) has been included that improves the safety and performance of LWT operations. + Importantly, v2 guarantees linearizability across safe range movements, so users are encouraged to enable v2. + v2 also halves the number of WAN messages required to be exchanged if used on conjunction with the new Paxos Repair + mechanism (see below) and with some minor modifications to applications using LWTs. + The new implementation may be enabled at any time by setting paxos_variant: v2, and disabled by setting to v1, + and this alone will reduce the number of WAN round-trips by between one and two for reads, and one for writes. + - A new Paxos Repair mechanism has been introduced as part of Repair, that permits further reducing the number of WAN + round-trips for write LWTs. This process may be manually executed for v1 and is run automatically alongside normal + repairs for v2. Once users are running regular repairs that include paxos repairs they are encouraged to set + paxos_state_purging: repaired. Once this has been set across the cluster, users are encouraged to set their + applications to supply a Commit consistency level of ANY with their LWT write operations, saving one additional WAN + round-trip. See upgrade notes below. - Warn/abort thresholds added to read queries notifying clients when these thresholds trigger (by emitting a client warning or aborting the query). This feature is disabled by default, scheduled to be enabled in 4.2; it is controlled with the configuration track_warnings.enabled, @@ -122,6 +134,18 @@ Upgrading what was actually applied in previous versions. This also affects the setters and getters for these properties in the JMX MBean `org.apache.cassandra.db:type=StorageService` and the nodetool commands `set/getstreamthroughput` and `set/getinterdcstreamthroughput`. + - Steps for upgrading Paxos + - Set paxos_variant: v2 across the cluster. This may be set via JMX, but should also be written + persistently to any yaml. + - Ensure paxos repairs are running regularly, either as part of normal incremental repair workflows or on their + own separate schedule. These operations are cheap and better to run frequently (e.g. once per hour) + - Set paxos_state_purging: repaired across the cluster. This may be set via JMX, but should also be written + persistently to any yaml. NOTE: once this has been set, you must not restore paxos_state_purging: legacy. If + this setting must be disabled you must instead set paxos_state_purging: gc_grace. This may be necessary if + paxos repairs must be disabled for some reason on an extended basis, but in this case your applications must + restore default commit consistency to ensure correctness. + - Applications may now safely be updated to use ANY commit consistency level (or LOCAL_QUORUM, as preferred). + Uncontended writes should now take 2 round-trips, and uncontended reads should typically take one round-trip. Deprecation ----------- diff --git a/build.xml b/build.xml index 46e8f62488..30097acae7 100644 --- a/build.xml +++ b/build.xml @@ -66,6 +66,7 @@ + @@ -1070,6 +1071,7 @@ + @@ -1154,10 +1156,10 @@ - - + + - + @@ -1167,10 +1169,10 @@ - - + + - + @@ -1367,7 +1369,7 @@ - @@ -1397,6 +1399,7 @@ + @@ -1504,6 +1507,8 @@ + + @@ -1960,23 +1965,36 @@ - - - - + + + + - + + + + + + + + + + + + + + diff --git a/src/java/org/apache/cassandra/batchlog/BatchlogManager.java b/src/java/org/apache/cassandra/batchlog/BatchlogManager.java index 34f532678d..8897e374c2 100644 --- a/src/java/org/apache/cassandra/batchlog/BatchlogManager.java +++ b/src/java/org/apache/cassandra/batchlog/BatchlogManager.java @@ -30,6 +30,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import java.util.function.Supplier; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Iterables; @@ -50,7 +51,6 @@ import org.apache.cassandra.db.Mutation; import org.apache.cassandra.db.SystemKeyspace; import org.apache.cassandra.db.WriteType; import org.apache.cassandra.db.marshal.BytesType; -import org.apache.cassandra.db.marshal.UUIDType; import org.apache.cassandra.db.partitions.PartitionUpdate; import org.apache.cassandra.dht.Token; import org.apache.cassandra.exceptions.WriteFailureException; @@ -501,9 +501,9 @@ public class BatchlogManager implements BatchlogManagerMBean } } - ReplicaPlan.ForTokenWrite replicaPlan = new ReplicaPlan.ForTokenWrite(keyspace, liveAndDown.replicationStrategy(), - ConsistencyLevel.ONE, liveRemoteOnly.pending(), liveRemoteOnly.all(), liveRemoteOnly.all(), liveRemoteOnly.all()); - ReplayWriteResponseHandler handler = new ReplayWriteResponseHandler<>(replicaPlan, nanoTime()); + ReplicaPlan.ForWrite replicaPlan = new ReplicaPlan.ForWrite(keyspace, liveAndDown.replicationStrategy(), + ConsistencyLevel.ONE, liveRemoteOnly.pending(), liveRemoteOnly.all(), liveRemoteOnly.all(), liveRemoteOnly.all()); + ReplayWriteResponseHandler handler = new ReplayWriteResponseHandler<>(replicaPlan, mutation, nanoTime()); Message message = Message.outWithFlag(MUTATION_REQ, mutation, MessageFlag.CALL_BACK_ON_FAILURE); for (Replica replica : liveRemoteOnly.all()) MessagingService.instance().sendWriteWithCallback(message, replica, handler, false); @@ -526,9 +526,10 @@ public class BatchlogManager implements BatchlogManagerMBean { private final Set undelivered = Collections.newSetFromMap(new ConcurrentHashMap<>()); - ReplayWriteResponseHandler(ReplicaPlan.ForTokenWrite replicaPlan, long queryStartNanoTime) + // TODO: should we be hinting here, since presumably batch log will retry? Maintaining historical behaviour for the moment. + ReplayWriteResponseHandler(ReplicaPlan.ForWrite replicaPlan, Supplier hintOnFailure, long queryStartNanoTime) { - super(replicaPlan, null, WriteType.UNLOGGED_BATCH, queryStartNanoTime); + super(replicaPlan, null, WriteType.UNLOGGED_BATCH, hintOnFailure, queryStartNanoTime); Iterables.addAll(undelivered, replicaPlan.contacts().endpoints()); } diff --git a/src/java/org/apache/cassandra/concurrent/ExecutorFactory.java b/src/java/org/apache/cassandra/concurrent/ExecutorFactory.java index 83f48d1a56..f7d93e8379 100644 --- a/src/java/org/apache/cassandra/concurrent/ExecutorFactory.java +++ b/src/java/org/apache/cassandra/concurrent/ExecutorFactory.java @@ -25,6 +25,7 @@ import org.apache.cassandra.utils.JVMStabilityInspector; import org.apache.cassandra.utils.Shared; import static java.lang.Thread.*; +import static org.apache.cassandra.concurrent.ExecutorFactory.SimulatorSemantics.NORMAL; import static org.apache.cassandra.concurrent.InfiniteLoopExecutor.Daemon.DAEMON; import static org.apache.cassandra.concurrent.InfiniteLoopExecutor.Interrupts.UNSYNCHRONIZED; import static org.apache.cassandra.concurrent.NamedThreadFactory.createThread; @@ -72,6 +73,11 @@ public interface ExecutorFactory extends ExecutorBuilderFactory.Jmxable extends AsyncFuture implements RunnableFuture { - private Callable call; + private Callable call; - public FutureTask(Callable call) + public FutureTask(Callable call) { this.call = call; } diff --git a/src/java/org/apache/cassandra/concurrent/InfiniteLoopExecutor.java b/src/java/org/apache/cassandra/concurrent/InfiniteLoopExecutor.java index e1d25e5bcd..51c5f9f69e 100644 --- a/src/java/org/apache/cassandra/concurrent/InfiniteLoopExecutor.java +++ b/src/java/org/apache/cassandra/concurrent/InfiniteLoopExecutor.java @@ -29,21 +29,25 @@ import java.util.function.BiFunction; import java.util.function.Consumer; import org.apache.cassandra.utils.Shared; +import org.apache.cassandra.utils.concurrent.Condition; import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException; +import static org.apache.cassandra.concurrent.InfiniteLoopExecutor.InternalState.SHUTTING_DOWN_NOW; import static org.apache.cassandra.concurrent.InfiniteLoopExecutor.InternalState.TERMINATED; import static org.apache.cassandra.concurrent.InfiniteLoopExecutor.Interrupts.SYNCHRONIZED; import static org.apache.cassandra.concurrent.InfiniteLoopExecutor.Interrupts.UNSYNCHRONIZED; import static org.apache.cassandra.concurrent.Interruptible.State.INTERRUPTED; import static org.apache.cassandra.concurrent.Interruptible.State.NORMAL; import static org.apache.cassandra.concurrent.Interruptible.State.SHUTTING_DOWN; +import static org.apache.cassandra.utils.Clock.Global.nanoTime; +import static org.apache.cassandra.utils.concurrent.Condition.newOneTimeCondition; public class InfiniteLoopExecutor implements Interruptible { private static final Logger logger = LoggerFactory.getLogger(InfiniteLoopExecutor.class); @Shared(scope = Shared.Scope.SIMULATION) - public enum InternalState { TERMINATED } + public enum InternalState { SHUTTING_DOWN_NOW, TERMINATED } @Shared(scope = Shared.Scope.SIMULATION) public enum SimulatorSafe { SAFE, UNSAFE } @@ -59,6 +63,7 @@ public class InfiniteLoopExecutor implements Interruptible private final Task task; private volatile Object state = NORMAL; private final Consumer interruptHandler; + private final Condition isTerminated = newOneTimeCondition(); public InfiniteLoopExecutor(String name, Task task, Daemon daemon) { @@ -102,33 +107,41 @@ public class InfiniteLoopExecutor implements Interruptible private void loop() { boolean interrupted = false; - while (true) + try { - try + while (true) { - Object cur = state; - if (cur == TERMINATED) break; + try + { + Object cur = state; + if (cur == SHUTTING_DOWN_NOW) break; - interrupted |= Thread.interrupted(); - if (cur == NORMAL && interrupted) cur = INTERRUPTED; - task.run((State) cur); + interrupted |= Thread.interrupted(); + if (cur == NORMAL && interrupted) cur = INTERRUPTED; + task.run((State) cur); - interrupted = false; - if (cur == SHUTTING_DOWN) state = TERMINATED; - } - catch (TerminateException ignore) - { - state = TERMINATED; - } - catch (UncheckedInterruptedException | InterruptedException ignore) - { - interrupted = true; - } - catch (Throwable t) - { - logger.error("Exception thrown by runnable, continuing with loop", t); + interrupted = false; + if (cur == SHUTTING_DOWN) break; + } + catch (TerminateException ignore) + { + break; + } + catch (UncheckedInterruptedException | InterruptedException ignore) + { + interrupted = true; + } + catch (Throwable t) + { + logger.error("Exception thrown by runnable, continuing with loop", t); + } } } + finally + { + state = TERMINATED; + isTerminated.signal(); + } } public void interrupt() @@ -138,13 +151,13 @@ public class InfiniteLoopExecutor implements Interruptible public void shutdown() { - stateUpdater.updateAndGet(this, cur -> cur != TERMINATED ? SHUTTING_DOWN : TERMINATED); + stateUpdater.updateAndGet(this, cur -> cur != TERMINATED && cur != SHUTTING_DOWN_NOW ? SHUTTING_DOWN : cur); interruptHandler.accept(thread); } public Object shutdownNow() { - state = TERMINATED; + stateUpdater.updateAndGet(this, cur -> cur != TERMINATED ? SHUTTING_DOWN_NOW : TERMINATED); interruptHandler.accept(thread); return null; } @@ -152,12 +165,16 @@ public class InfiniteLoopExecutor implements Interruptible @Override public boolean isTerminated() { - return state == TERMINATED && !thread.isAlive(); + return state == TERMINATED; } public boolean awaitTermination(long time, TimeUnit unit) throws InterruptedException { - thread.join(unit.toMillis(time)); + if (isTerminated()) + return true; + + long deadlineNanos = nanoTime() + unit.toNanos(time); + isTerminated.awaitUntil(deadlineNanos); return isTerminated(); } diff --git a/src/java/org/apache/cassandra/concurrent/Stage.java b/src/java/org/apache/cassandra/concurrent/Stage.java index d8a5e548e4..1c15e4505a 100644 --- a/src/java/org/apache/cassandra/concurrent/Stage.java +++ b/src/java/org/apache/cassandra/concurrent/Stage.java @@ -53,16 +53,18 @@ public enum Stage MISC ("MiscStage", "internal", () -> 1, null, Stage::singleThreadedStage), TRACING ("TracingStage", "internal", () -> 1, null, Stage::tracingStage), INTERNAL_RESPONSE ("InternalResponseStage", "internal", FBUtilities::getAvailableProcessors, null, Stage::multiThreadedStage), - IMMEDIATE ("ImmediateStage", "internal", () -> 0, null, Stage::immediateExecutor); + IMMEDIATE ("ImmediateStage", "internal", () -> 0, null, Stage::immediateExecutor), + PAXOS_REPAIR ("PaxosRepairStage", "internal", FBUtilities::getAvailableProcessors, null, Stage::multiThreadedStage), + ; public final String jmxName; - private final Supplier initialiser; - private volatile ExecutorPlus executor = null; + private final Supplier executorSupplier; + private volatile ExecutorPlus executor; - Stage(String jmxName, String jmxType, IntSupplier numThreads, LocalAwareExecutorPlus.MaximumPoolSizeListener onSetMaximumPoolSize, ExecutorServiceInitialiser initialiser) + Stage(String jmxName, String jmxType, IntSupplier numThreads, LocalAwareExecutorPlus.MaximumPoolSizeListener onSetMaximumPoolSize, ExecutorServiceInitialiser executorSupplier) { this.jmxName = jmxName; - this.initialiser = () -> initialiser.init(jmxName, jmxType, numThreads.getAsInt(), onSetMaximumPoolSize); + this.executorSupplier = () -> executorSupplier.init(jmxName, jmxType, numThreads.getAsInt(), onSetMaximumPoolSize); } private static String normalizeName(String stageName) @@ -128,7 +130,7 @@ public enum Stage { if (executor == null) { - executor = initialiser.get(); + executor = executorSupplier.get(); } } } diff --git a/src/java/org/apache/cassandra/concurrent/SyncFutureTask.java b/src/java/org/apache/cassandra/concurrent/SyncFutureTask.java index 4885821aed..422da99fb8 100644 --- a/src/java/org/apache/cassandra/concurrent/SyncFutureTask.java +++ b/src/java/org/apache/cassandra/concurrent/SyncFutureTask.java @@ -71,8 +71,7 @@ public class SyncFutureTask extends SyncFuture implements RunnableFuture T convert(PropertyConverter converter) + { + String value = System.getProperty(key); + if (value == null) + value = defaultVal; + + return converter.convert(value); + } + /** * Sets the value into system properties. * @param value to set @@ -415,7 +426,7 @@ public enum CassandraRelevantProperties System.setProperty(key, Long.toString(value)); } - private interface PropertyConverter + public interface PropertyConverter { T convert(String value); } diff --git a/src/java/org/apache/cassandra/config/Config.java b/src/java/org/apache/cassandra/config/Config.java index 09da88c810..4c0f29a07f 100644 --- a/src/java/org/apache/cassandra/config/Config.java +++ b/src/java/org/apache/cassandra/config/Config.java @@ -29,6 +29,7 @@ import java.util.TreeMap; import java.util.function.Supplier; import com.google.common.base.Joiner; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.Sets; import org.slf4j.Logger; @@ -48,6 +49,20 @@ public class Config { private static final Logger logger = LoggerFactory.getLogger(Config.class); + public static Set splitCommaDelimited(String src) + { + if (src == null) + return ImmutableSet.of(); + String[] split = src.split(",\\s*"); + ImmutableSet.Builder builder = ImmutableSet.builder(); + for (String s : split) + { + s = s.trim(); + if (!s.isEmpty()) + builder.add(s); + } + return builder.build(); + } /* * Prefix for Java properties for internal Cassandra configuration options */ @@ -125,11 +140,13 @@ public class Config public volatile SmallestDurationMilliseconds counter_write_request_timeout = new SmallestDurationMilliseconds("5000ms"); @Replaces(oldName = "cas_contention_timeout_in_ms", converter = Converters.MILLIS_DURATION, deprecated = true) - public volatile SmallestDurationMilliseconds cas_contention_timeout = new SmallestDurationMilliseconds("1000ms"); + public volatile SmallestDurationMilliseconds cas_contention_timeout = new SmallestDurationMilliseconds("1800ms"); @Replaces(oldName = "truncate_request_timeout_in_ms", converter = Converters.MILLIS_DURATION, deprecated = true) public volatile SmallestDurationMilliseconds truncate_request_timeout = new SmallestDurationMilliseconds("60000ms"); + public volatile Long repair_request_timeout_in_ms = 120000L; + public Integer streaming_connections_per_host = 1; @Replaces(oldName = "streaming_keep_alive_period_in_secs", converter = Converters.SECONDS_DURATION, deprecated = true) public SmallestDurationSeconds streaming_keep_alive_period = new SmallestDurationSeconds("300s"); @@ -406,6 +423,8 @@ public class Config public volatile SmallestDurationSeconds counter_cache_save_period = new SmallestDurationSeconds("7200s"); public volatile int counter_cache_keys_to_save = Integer.MAX_VALUE; + public SmallestDataStorageMebibytes paxos_cache_size = null; + @Replaces(oldName = "cache_load_timeout_seconds ", converter = Converters.SECONDS_DURATION, deprecated = true) public SmallestDurationSeconds cache_load_timeout = new SmallestDurationSeconds("30s"); @@ -780,13 +799,190 @@ public class Config /** The configuration of startup checks. */ public volatile Map> startup_checks = new HashMap<>(); + /** + * The variants of paxos implementation and semantics supported by Cassandra. + */ public enum PaxosVariant { - v1_without_linearizable_reads, // with legacy semantics for read/read linearizability (i.e. not guaranteed) - v1 + /** + * v1 Paxos lacks most optimisations. Expect 4RTs for a write and 2RTs for a read. + * + * With legacy semantics for read/read and rejected write linearizability, i.e. not guaranteed. + */ + v1_without_linearizable_reads_or_rejected_writes, + + /** + * v1 Paxos lacks most optimisations. Expect 4RTs for a write and 3RTs for a read. + */ + v1, + + /** + * v2 Paxos. With PaxosStatePurging.repaired safe to use ANY Commit consistency. + * Expect 2RTs for a write and 1RT for a read. + * + * With legacy semantics for read/read linearizability, i.e. not guaranteed. + */ + v2_without_linearizable_reads, + + /** + * v2 Paxos. With PaxosStatePurging.repaired safe to use ANY Commit consistency. + * Expect 2RTs for a write and 1RT for a read. + * + * With legacy semantics for read/read and rejected write linearizability, i.e. not guaranteed. + */ + v2_without_linearizable_reads_or_rejected_writes, + + /** + * v2 Paxos. With PaxosStatePurging.repaired safe to use ANY Commit consistency. + * Expect 2RTs for a write, and either 1RT or 2RT for a read. + */ + v2 } + + /** + * Select the kind of paxos state purging to use. Migration to repaired is recommended, but requires that + * regular paxos repairs are performed (which by default run as part of incremental repair). + * + * Once migrated from legacy it is unsafe to return to legacy, but gc_grace mode may be used in its place + * and performs a very similar cleanup process. + * + * Should only be modified once paxos_variant = v2. + */ + public enum PaxosStatePurging + { + /** + * system.paxos records are written and garbage collected with TTLs. Unsafe to use with Commit consistency ANY. + * Once migrated from, cannot be migrated back to safely. Must use gc_grace or repaired instead, as TTLs + * will not have been set. + */ + legacy, + + /** + * Functionally similar to legacy, but the gc_grace expiry is applied at compaction and read time rather than + * using TTLs, so may be safely enabled at any point. + */ + gc_grace, + + /** + * Clears system.paxos records only once they are known to be persisted to a quorum of replica's base tables + * through the use of paxos repair. Requires that regular paxos repairs are performed on the cluster + * (which by default are included in incremental repairs if paxos_variant = v2). + * + * This setting permits the use of Commit consistency ANY or LOCAL_QUORUM without any loss of durability or consistency, + * saving 1 RT. + */ + repaired; + + public static PaxosStatePurging fromBoolean(boolean enabled) + { + return enabled ? repaired : gc_grace; + } + } + + /** + * See {@link PaxosVariant}. Defaults to v1, recommend upgrading to v2 at earliest opportunity. + */ public volatile PaxosVariant paxos_variant = PaxosVariant.v1; + /** + * If true, paxos topology change repair will not run on a topology change - this option should only be used in + * rare operation circumstances e.g. where for some reason the repair is impossible to perform (e.g. too few replicas) + * and an unsafe topology change must be made + */ + public volatile boolean skip_paxos_repair_on_topology_change = Boolean.getBoolean("cassandra.skip_paxos_repair_on_topology_change"); + + /** + * A safety margin when purging paxos state information that has been safely replicated to a quorum. + * Data for transactions initiated within this grace period will not be expunged. + */ + public volatile DurationSpec paxos_purge_grace_period = new SmallestDurationSeconds("60s"); + + /** + * A safety mechanism for detecting incorrect paxos state, that may be down either to a bug or incorrect usage of LWTs + * (most likely due to unsafe mixing of SERIAL and LOCAL_SERIAL operations), and rejecting + */ + public enum PaxosOnLinearizabilityViolation + { + // reject an operation when a linearizability violation is detected. + // note this does not guarantee a violation has been averted, + // as it may be a prior operation that invalidated the state. + fail, + // log any detected linearizability violation + log, + // ignore any detected linearizability violation + ignore + } + + /** + * See {@link PaxosOnLinearizabilityViolation}. + * + * Default is to ignore, as applications may readily mix SERIAL and LOCAL_SERIAL and this is the most likely source + * of linearizability violations. this facility should be activated only for debugging Cassandra or by power users + * who are investigating their own application behaviour. + */ + public volatile PaxosOnLinearizabilityViolation paxos_on_linearizability_violations = PaxosOnLinearizabilityViolation.ignore; + + /** + * See {@link PaxosStatePurging} default is legacy. + */ + public volatile PaxosStatePurging paxos_state_purging; + + /** + * Enable/disable paxos repair. This is a global flag that not only determines default behaviour but overrides + * explicit paxos repair requests, paxos repair on topology changes and paxos auto repairs. + */ + public volatile boolean paxos_repair_enabled = true; + + /** + * If true, paxos topology change repair only requires a global quorum of live nodes. If false, + * it requires a global quorum as well as a local quorum for each dc (EACH_QUORUM), with the + * exception explained in paxos_topology_repair_strict_each_quorum + */ + public boolean paxos_topology_repair_no_dc_checks = false; + + /** + * If true, a quorum will be required for the global and local quorum checks. If false, we will + * accept a quorum OR n - 1 live nodes. This is to allow for topologies like 2:2:2, where paxos queries + * always use SERIAL, and a single node down in a dc should not preclude a paxos repair + */ + public boolean paxos_topology_repair_strict_each_quorum = false; + + /** + * If necessary for operational purposes, permit certain keyspaces to be ignored for paxos topology repairs + */ + public volatile Set skip_paxos_repair_on_topology_change_keyspaces = splitCommaDelimited(System.getProperty("cassandra.skip_paxos_repair_on_topology_change_keyspaces")); + + /** + * See {@link org.apache.cassandra.service.paxos.ContentionStrategy} + */ + public String paxos_contention_wait_randomizer; + + /** + * See {@link org.apache.cassandra.service.paxos.ContentionStrategy} + */ + public String paxos_contention_min_wait; + + /** + * See {@link org.apache.cassandra.service.paxos.ContentionStrategy} + */ + public String paxos_contention_max_wait; + + /** + * See {@link org.apache.cassandra.service.paxos.ContentionStrategy} + */ + public String paxos_contention_min_delta; + + /** + * The amount of disk space paxos uncommitted key files can consume before we begin automatically scheduling paxos repairs. + * Note that these repairs are uncoordinated and so do not contribute to expunging system.paxos records. + */ + public volatile int paxos_auto_repair_threshold_mb = 32; + + /** + * The number of keys we may simultaneously attempt to finish incomplete paxos operations for. + */ + public volatile int paxos_repair_parallelism = -1; + public static Supplier getOverrideLoadConfig() { return overrideLoadConfig; diff --git a/src/java/org/apache/cassandra/config/DataStorageSpec.java b/src/java/org/apache/cassandra/config/DataStorageSpec.java index baa6d4a3a3..8b5a0e4be0 100644 --- a/src/java/org/apache/cassandra/config/DataStorageSpec.java +++ b/src/java/org/apache/cassandra/config/DataStorageSpec.java @@ -88,7 +88,7 @@ public class DataStorageSpec public DataStorageSpec (String value, DataStorageUnit minUnit) { - if (value == null || value.equals("null")) + if (value == null || value.equals("null") || value.equals("0")) { quantity = 0; unit = minUnit; diff --git a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java index 65cb3d62be..05547e5437 100644 --- a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java +++ b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java @@ -32,6 +32,15 @@ import com.google.common.primitives.Ints; import com.google.common.primitives.Longs; import com.google.common.util.concurrent.RateLimiter; +import org.apache.cassandra.config.Config.PaxosOnLinearizabilityViolation; +import org.apache.cassandra.db.ConsistencyLevel; +import org.apache.cassandra.gms.IFailureDetector; +import org.apache.cassandra.io.util.File; +import org.apache.cassandra.config.Config.PaxosStatePurging; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import org.apache.cassandra.audit.AuditLogOptions; import org.apache.cassandra.fql.FullQueryLoggerOptions; import org.apache.cassandra.auth.AllowAllInternodeAuthenticator; @@ -42,15 +51,12 @@ import org.apache.cassandra.auth.IInternodeAuthenticator; import org.apache.cassandra.auth.INetworkAuthorizer; import org.apache.cassandra.auth.IRoleManager; import org.apache.cassandra.config.Config.CommitLogSync; -import org.apache.cassandra.db.ConsistencyLevel; import org.apache.cassandra.db.commitlog.AbstractCommitLogSegmentManager; import org.apache.cassandra.db.commitlog.CommitLog; import org.apache.cassandra.db.commitlog.CommitLogSegmentManagerCDC; import org.apache.cassandra.db.commitlog.CommitLogSegmentManagerStandard; import org.apache.cassandra.dht.IPartitioner; import org.apache.cassandra.exceptions.ConfigurationException; -import org.apache.cassandra.gms.IFailureDetector; -import org.apache.cassandra.io.util.File; import org.apache.cassandra.io.FSWriteError; import org.apache.cassandra.io.util.DiskOptimizationStrategy; import org.apache.cassandra.io.util.FileUtils; @@ -81,6 +87,7 @@ import static org.apache.cassandra.config.CassandraRelevantProperties.OS_ARCH; import static org.apache.cassandra.config.CassandraRelevantProperties.SUN_ARCH_DATA_MODEL; import static org.apache.cassandra.io.util.FileUtils.ONE_GIB; import static org.apache.cassandra.io.util.FileUtils.ONE_MIB; +import static org.apache.cassandra.config.CassandraRelevantProperties.TEST_JVM_DTEST_DISABLE_SSL; import static org.apache.cassandra.utils.Clock.Global.logInitializationOutcome; public class DatabaseDescriptor @@ -132,6 +139,7 @@ public class DatabaseDescriptor private static long preparedStatementsCacheSizeInMiB; private static long keyCacheSizeInMiB; + private static long paxosCacheSizeInMiB; private static long counterCacheSizeInMiB; private static long indexSummaryCapacityInMiB; @@ -730,6 +738,28 @@ public class DatabaseDescriptor + (conf.counter_cache_size !=null ?conf.counter_cache_size.toString() : null) + "', supported values are >= 0.", false); } + try + { + // if paxosCacheSizeInMiB option was set to "auto" then size of the cache should be "min(1% of Heap (in MB), 50MB) + paxosCacheSizeInMiB = (conf.paxos_cache_size == null) + ? Math.min(Math.max(1, (int) (Runtime.getRuntime().totalMemory() * 0.01 / 1024 / 1024)), 50) + : conf.paxos_cache_size.toMebibytes(); + + if (paxosCacheSizeInMiB < 0) + throw new NumberFormatException(); // to escape duplicating error message + } + catch (NumberFormatException e) + { + throw new ConfigurationException("paxos_cache_size option was set incorrectly to '" + + conf.paxos_cache_size + "', supported values are >= 0.", false); + } + + if (conf.paxos_auto_repair_threshold_mb < 0) + { + throw new ConfigurationException("paxos_auto_repair_threshold_mb option was set incorrectly to '" + + conf.paxos_auto_repair_threshold_mb + "', supported values are >= 0.", false); + } + // if set to empty/"auto" then use 5% of Heap size indexSummaryCapacityInMiB = (conf.index_summary_capacity == null) ? Math.max(1, (int) (Runtime.getRuntime().totalMemory() * 0.05 / 1024 / 1024)) @@ -844,18 +874,12 @@ public class DatabaseDescriptor conf.default_keyspace_rf, conf.minimum_keyspace_rf)); } - if (conf.paxos_variant == Config.PaxosVariant.v1_without_linearizable_reads) - { - logger.warn("This node was started with paxos_variant config option set to v1_norrl. " + - "SERIAL (and LOCAL_SERIAL) reads coordinated by this node " + - "will not offer linearizability (see CASSANDRA-12126 for details on what this mean) with " + - "respect to other SERIAL operations. Please note that, with this option, SERIAL reads will be " + - "slower than QUORUM reads, yet offer no more guarantee. This flag should only be used in " + - "the restricted case of upgrading from a pre-CASSANDRA-12126 version, and only if you " + - "understand the tradeoff."); - } + if (conf.paxos_repair_parallelism <= 0) + conf.paxos_repair_parallelism = Math.max(1, conf.concurrent_writes / 8); Paxos.setPaxosVariant(conf.paxos_variant); + if (conf.paxos_state_purging == null) + conf.paxos_state_purging = PaxosStatePurging.legacy; logInitializationOutcome(logger); } @@ -1053,6 +1077,9 @@ public class DatabaseDescriptor public static void applySslContext() { + if (TEST_JVM_DTEST_DISABLE_SSL.getBoolean()) + return; + try { SSLFactory.validateSslContext("Internode messaging", conf.server_encryption_options, true, true); @@ -1756,6 +1783,16 @@ public class DatabaseDescriptor conf.truncate_request_timeout = SmallestDurationMilliseconds.inMilliseconds(timeOutInMillis); } + public static long getRepairRpcTimeout() + { + return conf.repair_request_timeout_in_ms; + } + + public static void setRepairRpcTimeout(Long timeOutInMillis) + { + conf.repair_request_timeout_in_ms = timeOutInMillis; + } + public static boolean hasCrossNodeTimeout() { return conf.internode_timeout; @@ -2462,6 +2499,126 @@ public class DatabaseDescriptor conf.paxos_variant = variant; } + public static String getPaxosContentionWaitRandomizer() + { + return conf.paxos_contention_wait_randomizer; + } + + public static String getPaxosContentionMinWait() + { + return conf.paxos_contention_min_wait; + } + + public static String getPaxosContentionMaxWait() + { + return conf.paxos_contention_max_wait; + } + + public static String getPaxosContentionMinDelta() + { + return conf.paxos_contention_min_delta; + } + + public static void setPaxosContentionWaitRandomizer(String waitRandomizer) + { + conf.paxos_contention_wait_randomizer = waitRandomizer; + } + + public static void setPaxosContentionMinWait(String minWait) + { + conf.paxos_contention_min_wait = minWait; + } + + public static void setPaxosContentionMaxWait(String maxWait) + { + conf.paxos_contention_max_wait = maxWait; + } + + public static void setPaxosContentionMinDelta(String minDelta) + { + conf.paxos_contention_min_delta = minDelta; + } + + public static boolean skipPaxosRepairOnTopologyChange() + { + return conf.skip_paxos_repair_on_topology_change; + } + + public static void setSkipPaxosRepairOnTopologyChange(boolean value) + { + conf.skip_paxos_repair_on_topology_change = value; + } + + public static long getPaxosPurgeGrace(TimeUnit units) + { + return conf.paxos_purge_grace_period.to(units); + } + + public static void setPaxosPurgeGrace(long value, TimeUnit units) + { + conf.paxos_purge_grace_period = new DurationSpec(value, units); + } + + public static PaxosOnLinearizabilityViolation paxosOnLinearizabilityViolations() + { + return conf.paxos_on_linearizability_violations; + } + + public static void setPaxosOnLinearizabilityViolations(PaxosOnLinearizabilityViolation v) + { + conf.paxos_on_linearizability_violations = v; + } + + public static PaxosStatePurging paxosStatePurging() + { + return conf.paxos_state_purging; + } + + public static void setPaxosStatePurging(PaxosStatePurging v) + { + conf.paxos_state_purging = v; + } + + public static boolean paxosRepairEnabled() + { + return conf.paxos_repair_enabled; + } + + public static void setPaxosRepairEnabled(boolean v) + { + conf.paxos_repair_enabled = v; + } + + public static Set skipPaxosRepairOnTopologyChangeKeyspaces() + { + return conf.skip_paxos_repair_on_topology_change_keyspaces; + } + + public static void setSkipPaxosRepairOnTopologyChangeKeyspaces(String keyspaces) + { + conf.skip_paxos_repair_on_topology_change_keyspaces = Config.splitCommaDelimited(keyspaces); + } + + public static boolean paxoTopologyRepairNoDcChecks() + { + return conf.paxos_topology_repair_no_dc_checks; + } + + public static boolean paxoTopologyRepairStrictEachQuorum() + { + return conf.paxos_topology_repair_strict_each_quorum; + } + + public static int getPaxosAutoRepairThresholdMB() + { + return conf.paxos_auto_repair_threshold_mb; + } + + public static void setPaxosAutoRepairThresholdMB(int threshold) + { + conf.paxos_auto_repair_threshold_mb = threshold; + } + public static void setNativeTransportMaxConcurrentRequestsInBytesPerIp(long maxConcurrentRequestsInBytes) { conf.native_transport_max_concurrent_requests_in_bytes_per_ip = maxConcurrentRequestsInBytes; @@ -2952,6 +3109,11 @@ public class DatabaseDescriptor return conf.row_cache_keys_to_save; } + public static long getPaxosCacheSizeInMiB() + { + return paxosCacheSizeInMiB; + } + public static long getCounterCacheSizeInMiB() { return counterCacheSizeInMiB; @@ -3081,6 +3243,17 @@ public class DatabaseDescriptor conf.repair_session_space = SmallestDataStorageMebibytes.inMebibytes(sizeInMiB); } + public static int getPaxosRepairParallelism() + { + return conf.paxos_repair_parallelism; + } + + public static void setPaxosRepairParallelism(int v) + { + Preconditions.checkArgument(v > 0); + conf.paxos_repair_parallelism = v; + } + public static Float getMemtableCleanupThreshold() { return conf.memtable_cleanup_threshold; diff --git a/src/java/org/apache/cassandra/config/DurationSpec.java b/src/java/org/apache/cassandra/config/DurationSpec.java index e5edb3558a..a192e35abb 100644 --- a/src/java/org/apache/cassandra/config/DurationSpec.java +++ b/src/java/org/apache/cassandra/config/DurationSpec.java @@ -64,7 +64,7 @@ public class DurationSpec public DurationSpec(String value) { - if (value == null || value.equals("null") || value.toLowerCase(Locale.ROOT).equals("nan")) + if (value == null || value.equals("null") || value.toLowerCase(Locale.ROOT).equals("nan") || value.equals("0")) { quantity = 0; unit = MILLISECONDS; @@ -74,7 +74,7 @@ public class DurationSpec //parse the string field value Matcher matcher = TIME_UNITS_PATTERN.matcher(value); - if(matcher.find()) + if (matcher.find()) { quantity = Long.parseLong(matcher.group(1)); unit = fromSymbol(matcher.group(2)); diff --git a/src/java/org/apache/cassandra/config/EncryptionOptions.java b/src/java/org/apache/cassandra/config/EncryptionOptions.java index 6c26e8d3c6..246d13ffe1 100644 --- a/src/java/org/apache/cassandra/config/EncryptionOptions.java +++ b/src/java/org/apache/cassandra/config/EncryptionOptions.java @@ -25,6 +25,10 @@ import java.util.Map; import java.util.Objects; import java.util.Set; +import javax.net.ssl.KeyManagerFactory; +import javax.net.ssl.SSLException; +import javax.net.ssl.TrustManagerFactory; + import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableList; @@ -33,6 +37,8 @@ import org.slf4j.LoggerFactory; import org.apache.cassandra.locator.IEndpointSnitch; import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.security.AbstractSslContextFactory; +import org.apache.cassandra.security.DisableSslContextFactory; import org.apache.cassandra.security.ISslContextFactory; import org.apache.cassandra.utils.FBUtilities; @@ -277,8 +283,15 @@ public class EncryptionOptions putSslContextFactoryParameter(sslContextFactoryParameters, ConfigKey.ENABLED, this.enabled); putSslContextFactoryParameter(sslContextFactoryParameters, ConfigKey.OPTIONAL, this.optional); - sslContextFactoryInstance = FBUtilities.newSslContextFactory(ssl_context_factory.class_name, - sslContextFactoryParameters); + if (CassandraRelevantProperties.TEST_JVM_DTEST_DISABLE_SSL.getBoolean()) + { + sslContextFactoryInstance = new DisableSslContextFactory(); + } + else + { + sslContextFactoryInstance = FBUtilities.newSslContextFactory(ssl_context_factory.class_name, + sslContextFactoryParameters); + } } private void putSslContextFactoryParameter(Map existingParameters, ConfigKey configKey, diff --git a/src/java/org/apache/cassandra/cql3/QueryOptions.java b/src/java/org/apache/cassandra/cql3/QueryOptions.java index 7e3d267ed7..7b93616c1c 100644 --- a/src/java/org/apache/cassandra/cql3/QueryOptions.java +++ b/src/java/org/apache/cassandra/cql3/QueryOptions.java @@ -60,6 +60,11 @@ public abstract class QueryOptions return new DefaultQueryOptions(consistency, values, false, SpecificOptions.DEFAULT, ProtocolVersion.V3); } + public static QueryOptions forInternalCallsWithNowInSec(int nowInSec, ConsistencyLevel consistency, List values) + { + return new DefaultQueryOptions(consistency, values, false, SpecificOptions.DEFAULT.withNowInSec(nowInSec), ProtocolVersion.CURRENT); + } + public static QueryOptions forInternalCalls(List values) { return new DefaultQueryOptions(ConsistencyLevel.ONE, values, false, SpecificOptions.DEFAULT, ProtocolVersion.V3); @@ -209,6 +214,12 @@ public abstract class QueryOptions /** The keyspace that this query is bound to, or null if not relevant. */ public String getKeyspace() { return getSpecificOptions().keyspace; } + public int getNowInSec(int ifNotSet) + { + int nowInSec = getSpecificOptions().nowInSeconds; + return nowInSec != Integer.MIN_VALUE ? nowInSec : ifNotSet; + } + /** * The protocol version for the query. */ @@ -506,6 +517,11 @@ public abstract class QueryOptions this.keyspace = keyspace; this.nowInSeconds = nowInSeconds; } + + public SpecificOptions withNowInSec(int nowInSec) + { + return new SpecificOptions(pageSize, state, serialConsistency, timestamp, keyspace, nowInSec); + } } private static class Codec implements CBCodec diff --git a/src/java/org/apache/cassandra/cql3/QueryProcessor.java b/src/java/org/apache/cassandra/cql3/QueryProcessor.java index 1b8e94db85..bf03773a3b 100644 --- a/src/java/org/apache/cassandra/cql3/QueryProcessor.java +++ b/src/java/org/apache/cassandra/cql3/QueryProcessor.java @@ -52,6 +52,7 @@ import org.apache.cassandra.cql3.functions.Function; import org.apache.cassandra.cql3.functions.FunctionName; import org.apache.cassandra.cql3.statements.*; import org.apache.cassandra.db.*; +import org.apache.cassandra.db.rows.Row; import org.apache.cassandra.db.rows.RowIterator; import org.apache.cassandra.db.partitions.PartitionIterator; import org.apache.cassandra.db.partitions.PartitionIterators; @@ -375,6 +376,16 @@ public class QueryProcessor implements QueryHandler } private static QueryOptions makeInternalOptions(CQLStatement prepared, Object[] values, ConsistencyLevel cl) + { + return makeInternalOptionsWithNowInSec(prepared, FBUtilities.nowInSeconds(), values, cl); + } + + public static QueryOptions makeInternalOptionsWithNowInSec(CQLStatement prepared, int nowInSec, Object[] values) + { + return makeInternalOptionsWithNowInSec(prepared, nowInSec, values, ConsistencyLevel.ONE); + } + + private static QueryOptions makeInternalOptionsWithNowInSec(CQLStatement prepared, int nowInSec, Object[] values, ConsistencyLevel cl) { if (prepared.getBindVariables().size() != values.length) throw new IllegalArgumentException(String.format("Invalid number of values. Expecting %d but got %d", prepared.getBindVariables().size(), values.length)); @@ -383,10 +394,10 @@ public class QueryProcessor implements QueryHandler for (int i = 0; i < values.length; i++) { Object value = values[i]; - AbstractType type = prepared.getBindVariables().get(i).type; - boundValues.add(value instanceof ByteBuffer || value == null ? (ByteBuffer)value : type.decompose(value)); + AbstractType type = prepared.getBindVariables().get(i).type; + boundValues.add(value instanceof ByteBuffer || value == null ? (ByteBuffer)value : type.decomposeUntyped(value)); } - return QueryOptions.forInternalCalls(cl, boundValues); + return QueryOptions.forInternalCallsWithNowInSec(nowInSec, cl, boundValues); } public static Prepared prepareInternal(String query) throws RequestValidationException @@ -497,6 +508,16 @@ public class QueryProcessor implements QueryHandler return execute(query, cl, internalQueryState(), values); } + public static UntypedResultSet executeInternalWithNowInSec(String query, int nowInSec, Object... values) + { + Prepared prepared = prepareInternal(query); + ResultMessage result = prepared.statement.executeLocally(internalQueryState(), makeInternalOptionsWithNowInSec(prepared.statement, nowInSec, values)); + if (result instanceof ResultMessage.Rows) + return UntypedResultSet.create(((ResultMessage.Rows)result).result); + else + return null; + } + public static UntypedResultSet execute(String query, ConsistencyLevel cl, QueryState state, Object... values) throws RequestExecutionException { @@ -572,6 +593,19 @@ public class QueryProcessor implements QueryHandler return UntypedResultSet.create(((ResultMessage.Rows)result).result); } + /** + * A special version of executeInternal that takes the time used as "now" for the query in argument. + * Note that this only make sense for Selects so this only accept SELECT statements and is only useful in rare + * cases. + */ + public static Map> executeInternalRawWithNow(int nowInSec, String query, Object... values) + { + Prepared prepared = prepareInternal(query); + assert prepared.statement instanceof SelectStatement; + SelectStatement select = (SelectStatement)prepared.statement; + return select.executeRawInternal(makeInternalOptions(prepared.statement, values), internalQueryState().getClientState(), nowInSec); + } + public static UntypedResultSet resultify(String query, RowIterator partition) { return resultify(query, PartitionIterators.singletonIterator(partition)); diff --git a/src/java/org/apache/cassandra/cql3/UntypedResultSet.java b/src/java/org/apache/cassandra/cql3/UntypedResultSet.java index 020d27f5c9..f00c137d8c 100644 --- a/src/java/org/apache/cassandra/cql3/UntypedResultSet.java +++ b/src/java/org/apache/cassandra/cql3/UntypedResultSet.java @@ -374,6 +374,12 @@ public abstract class UntypedResultSet implements Iterable return Int32Type.instance.compose(data.get(column)); } + public int getInt(String column, int ifNull) + { + ByteBuffer bytes = data.get(column); + return bytes == null ? ifNull : Int32Type.instance.compose(bytes); + } + public double getDouble(String column) { return DoubleType.instance.compose(data.get(column)); diff --git a/src/java/org/apache/cassandra/cql3/functions/TimeFcts.java b/src/java/org/apache/cassandra/cql3/functions/TimeFcts.java index d9c706a31c..1bf03ff3fc 100644 --- a/src/java/org/apache/cassandra/cql3/functions/TimeFcts.java +++ b/src/java/org/apache/cassandra/cql3/functions/TimeFcts.java @@ -94,7 +94,7 @@ public abstract class TimeFcts /** * Function that convert a value of TIMEUUID into a value of type TIMESTAMP. - * @deprecated Replaced by the {@link #timeUuidToTimestamp} function + * @deprecated Replaced by the {@link #toTimestamp} function */ public static final NativeScalarFunction dateOfFct = new NativeScalarFunction("dateof", TimestampType.instance, TimeUUIDType.instance) { @@ -120,7 +120,7 @@ public abstract class TimeFcts /** * Function that convert a value of type TIMEUUID into an UNIX timestamp. - * @deprecated Replaced by the {@link #timeUuidToUnixTimestamp} function + * @deprecated Replaced by the {@link #toUnixTimestamp} function */ public static final NativeScalarFunction unixTimestampOfFct = new NativeScalarFunction("unixtimestampof", LongType.instance, TimeUUIDType.instance) { diff --git a/src/java/org/apache/cassandra/cql3/statements/CQL3CasRequest.java b/src/java/org/apache/cassandra/cql3/statements/CQL3CasRequest.java index 376a65f4bd..3db4793c56 100644 --- a/src/java/org/apache/cassandra/cql3/statements/CQL3CasRequest.java +++ b/src/java/org/apache/cassandra/cql3/statements/CQL3CasRequest.java @@ -36,6 +36,7 @@ import org.apache.cassandra.db.partitions.PartitionUpdate; import org.apache.cassandra.exceptions.InvalidRequestException; import org.apache.cassandra.service.CASRequest; import org.apache.cassandra.service.ClientState; +import org.apache.cassandra.service.paxos.Ballot; import org.apache.cassandra.utils.Pair; import org.apache.cassandra.utils.TimeUUID; @@ -180,7 +181,7 @@ public class CQL3CasRequest implements CASRequest return new RegularAndStaticColumns(statics, regulars); } - public SinglePartitionReadQuery readCommand(int nowInSec) + public SinglePartitionReadCommand readCommand(int nowInSec) { assert staticConditions != null || !conditions.isEmpty(); @@ -190,7 +191,7 @@ public class CQL3CasRequest implements CASRequest // With only a static condition, we still want to make the distinction between a non-existing partition and one // that exists (has some live data) but has not static content. So we query the first live row of the partition. if (conditions.isEmpty()) - return SinglePartitionReadQuery.create(metadata, + return SinglePartitionReadCommand.create(metadata, nowInSec, columnFilter, RowFilter.NONE, @@ -199,7 +200,7 @@ public class CQL3CasRequest implements CASRequest new ClusteringIndexSliceFilter(Slices.ALL, false)); ClusteringIndexNamesFilter filter = new ClusteringIndexNamesFilter(conditions.navigableKeySet(), false); - return SinglePartitionReadQuery.create(metadata, nowInSec, key, columnFilter, filter); + return SinglePartitionReadCommand.create(metadata, nowInSec, key, columnFilter, filter); } /** @@ -232,14 +233,14 @@ public class CQL3CasRequest implements CASRequest return builder.build(); } - public PartitionUpdate makeUpdates(FilteredPartition current, ClientState state, TimeUUID ballot) throws InvalidRequestException + public PartitionUpdate makeUpdates(FilteredPartition current, ClientState clientState, Ballot ballot) throws InvalidRequestException { PartitionUpdate.Builder updateBuilder = new PartitionUpdate.Builder(metadata, key, updatedColumns(), conditions.size()); long timeUuidNanos = 0; for (RowUpdate upd : updates) - timeUuidNanos = upd.applyUpdates(current, updateBuilder, state, ballot.msb(), timeUuidNanos); + timeUuidNanos = upd.applyUpdates(current, updateBuilder, clientState, ballot.msb(), timeUuidNanos); for (RangeDeletion upd : rangeDeletions) - upd.applyUpdates(current, updateBuilder, state); + upd.applyUpdates(current, updateBuilder, clientState); PartitionUpdate partitionUpdate = updateBuilder.build(); IndexRegistry.obtain(metadata).validate(partitionUpdate); diff --git a/src/java/org/apache/cassandra/cql3/statements/ModificationStatement.java b/src/java/org/apache/cassandra/cql3/statements/ModificationStatement.java index fd2f0441be..52578a6318 100644 --- a/src/java/org/apache/cassandra/cql3/statements/ModificationStatement.java +++ b/src/java/org/apache/cassandra/cql3/statements/ModificationStatement.java @@ -51,18 +51,18 @@ import org.apache.cassandra.exceptions.*; import org.apache.cassandra.service.ClientState; import org.apache.cassandra.service.QueryState; import org.apache.cassandra.service.StorageProxy; +import org.apache.cassandra.service.paxos.Ballot; import org.apache.cassandra.service.paxos.BallotGenerator; -import org.apache.cassandra.service.paxos.Commit; +import org.apache.cassandra.service.paxos.Commit.Proposal; import org.apache.cassandra.transport.messages.ResultMessage; import org.apache.cassandra.triggers.TriggerExecutor; import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.MD5Digest; import org.apache.cassandra.utils.Pair; -import org.apache.cassandra.utils.TimeUUID; -import org.apache.cassandra.utils.UUIDGen; import static org.apache.cassandra.cql3.statements.RequestValidations.checkFalse; import static org.apache.cassandra.cql3.statements.RequestValidations.checkNull; +import static org.apache.cassandra.service.paxos.Ballot.Flag.NONE; import static org.apache.cassandra.utils.Clock.Global.nanoTime; /* @@ -660,7 +660,7 @@ public abstract class ModificationStatement implements CQLStatement.SingleKeyspa static RowIterator casInternal(ClientState state, CQL3CasRequest request, long timestamp, int nowInSeconds) { - TimeUUID ballot = BallotGenerator.Global.randomBallot(timestamp, false); + Ballot ballot = BallotGenerator.Global.atUnixMicros(timestamp, NONE); SinglePartitionReadQuery readCommand = request.readCommand(nowInSeconds); FilteredPartition current; @@ -676,7 +676,7 @@ public abstract class ModificationStatement implements CQLStatement.SingleKeyspa PartitionUpdate updates = request.makeUpdates(current, state, ballot); updates = TriggerExecutor.instance.execute(updates); - Commit proposal = Commit.newProposal(ballot, updates); + Proposal proposal = Proposal.of(ballot, updates); proposal.makeMutation().apply(); return null; } diff --git a/src/java/org/apache/cassandra/cql3/statements/SelectStatement.java b/src/java/org/apache/cassandra/cql3/statements/SelectStatement.java index 4d6bf00b75..e5f4d75334 100644 --- a/src/java/org/apache/cassandra/cql3/statements/SelectStatement.java +++ b/src/java/org/apache/cassandra/cql3/statements/SelectStatement.java @@ -254,7 +254,7 @@ public class SelectStatement implements CQLStatement.SingleKeyspaceCqlStatement query.trackWarnings(); if (aggregationSpec == null && (pageSize <= 0 || (query.limits().count() <= pageSize))) - return execute(query, options, state, selectors, nowInSec, userLimit, queryStartNanoTime); + return execute(query, options, state.getClientState(), selectors, nowInSec, userLimit, queryStartNanoTime); QueryPager pager = getPager(query, options); @@ -300,12 +300,12 @@ public class SelectStatement implements CQLStatement.SingleKeyspaceCqlStatement private ResultMessage.Rows execute(ReadQuery query, QueryOptions options, - QueryState state, + ClientState state, Selectors selectors, int nowInSec, int userLimit, long queryStartNanoTime) throws RequestValidationException, RequestExecutionException { - try (PartitionIterator data = query.execute(options.getConsistency(), state.getClientState(), queryStartNanoTime)) + try (PartitionIterator data = query.execute(options.getConsistency(), state, queryStartNanoTime)) { return processResults(data, options, selectors, nowInSec, userLimit); } @@ -491,6 +491,50 @@ public class SelectStatement implements CQLStatement.SingleKeyspaceCqlStatement return new AggregationQueryPager(pager, query.limits()); } + public Map> executeRawInternal(QueryOptions options, ClientState state, int nowInSec) throws RequestExecutionException, RequestValidationException + { + int userLimit = getLimit(options); + int userPerPartitionLimit = getPerPartitionLimit(options); + if (options.getPageSize() > 0) + throw new IllegalStateException(); + if (aggregationSpec != null) + throw new IllegalStateException(); + + Selectors selectors = selection.newSelectors(options); + ReadQuery query = getQuery(options, state, selectors.getColumnFilter(), nowInSec, userLimit, userPerPartitionLimit, Integer.MAX_VALUE); + + Map> result = Collections.emptyMap(); + try (ReadExecutionController executionController = query.executionController()) + { + try (PartitionIterator data = query.executeInternal(executionController)) + { + while (data.hasNext()) + { + try (RowIterator in = data.next()) + { + List out = Collections.emptyList(); + while (in.hasNext()) + { + switch (out.size()) + { + case 0: out = Collections.singletonList(in.next()); break; + case 1: out = new ArrayList<>(out); + default: out.add(in.next()); + } + } + switch (result.size()) + { + case 0: result = Collections.singletonMap(in.partitionKey(), out); break; + case 1: result = new TreeMap<>(result); + default: result.put(in.partitionKey(), out); + } + } + } + return result; + } + } + } + public ResultSet process(PartitionIterator partitions, int nowInSec) throws InvalidRequestException { QueryOptions options = QueryOptions.DEFAULT; diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java index 9e2338ad9e..542bf0bfa4 100644 --- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java +++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java @@ -40,10 +40,10 @@ import java.util.stream.Collectors; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.*; -import com.google.common.base.Throwables; import com.google.common.collect.*; import com.google.common.util.concurrent.*; +import org.apache.cassandra.service.paxos.Ballot; import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.TimeUUID; import org.apache.cassandra.utils.concurrent.AsyncPromise; @@ -99,7 +99,15 @@ import org.apache.cassandra.service.StorageService; import org.apache.cassandra.service.snapshot.SnapshotManifest; import org.apache.cassandra.service.snapshot.TableSnapshot; import org.apache.cassandra.streaming.TableStreamManager; -import org.apache.cassandra.utils.*; +import org.apache.cassandra.service.paxos.PaxosRepairHistory; +import org.apache.cassandra.service.paxos.TablePaxosRepairHistory; +import org.apache.cassandra.utils.DefaultValue; +import org.apache.cassandra.utils.ExecutorUtils; +import org.apache.cassandra.utils.FBUtilities; +import org.apache.cassandra.utils.JVMStabilityInspector; +import org.apache.cassandra.utils.MBeanWrapper; +import org.apache.cassandra.utils.NoSpamLogger; +import org.apache.cassandra.utils.WrappedRunnable; import org.apache.cassandra.utils.concurrent.OpOrder; import org.apache.cassandra.utils.concurrent.Promise; import org.apache.cassandra.utils.concurrent.Refs; @@ -228,6 +236,29 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean private volatile boolean neverPurgeTombstones = false; + private class PaxosRepairHistoryLoader + { + private TablePaxosRepairHistory history; + + TablePaxosRepairHistory get() + { + if (history != null) + return history; + + synchronized (this) + { + if (history != null) + return history; + + history = TablePaxosRepairHistory.load(keyspace.getName(), name); + return history; + } + } + + } + + private final PaxosRepairHistoryLoader paxosRepairHistory = new PaxosRepairHistoryLoader(); + public static void shutdownPostFlushExecutor() throws InterruptedException { postFlushExecutor.shutdown(); @@ -604,7 +635,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean return createColumnFamilyStore(keyspace, metadata.name, metadata, loadSSTables); } - public static synchronized ColumnFamilyStore createColumnFamilyStore(Keyspace keyspace, + public static ColumnFamilyStore createColumnFamilyStore(Keyspace keyspace, String columnFamily, TableMetadataRef metadata, boolean loadSSTables) @@ -1143,7 +1174,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean * with CL as we do with memtables/CFS-backed SecondaryIndexes. */ if (flushNonCf2i) - indexManager.flushAllNonCFSBackedIndexesBlocking(); + indexManager.flushAllNonCFSBackedIndexesBlocking(memtable); flushResults = Lists.newArrayList(FBUtilities.waitOnFutures(futures)); } @@ -1731,6 +1762,31 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean || filter.isFullyCoveredBy(cached); } + public PaxosRepairHistory getPaxosRepairHistory() + { + return paxosRepairHistory.get().getHistory(); + } + + public PaxosRepairHistory getPaxosRepairHistoryForRanges(Collection> ranges) + { + return paxosRepairHistory.get().getHistoryForRanges(ranges); + } + + public void syncPaxosRepairHistory(PaxosRepairHistory sync, boolean flush) + { + paxosRepairHistory.get().merge(sync, flush); + } + + public void onPaxosRepairComplete(Collection> ranges, Ballot highBallot) + { + paxosRepairHistory.get().add(ranges, highBallot, true); + } + + public Ballot getPaxosRepairLowBound(DecoratedKey key) + { + return paxosRepairHistory.get().getBallotForToken(key.getToken()); + } + public int gcBefore(int nowInSec) { return nowInSec - metadata().params.gcGraceSeconds; diff --git a/src/java/org/apache/cassandra/db/ConsistencyLevel.java b/src/java/org/apache/cassandra/db/ConsistencyLevel.java index fbaf3fd4d6..ccbd9181a6 100644 --- a/src/java/org/apache/cassandra/db/ConsistencyLevel.java +++ b/src/java/org/apache/cassandra/db/ConsistencyLevel.java @@ -20,6 +20,7 @@ package org.apache.cassandra.db; import com.carrotsearch.hppc.ObjectIntHashMap; import org.apache.cassandra.locator.Endpoints; +import org.apache.cassandra.locator.InOurDc; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.exceptions.InvalidRequestException; @@ -28,7 +29,6 @@ import org.apache.cassandra.locator.NetworkTopologyStrategy; import org.apache.cassandra.transport.ProtocolException; import static org.apache.cassandra.locator.Replicas.addToCountPerDc; -import static org.apache.cassandra.locator.Replicas.countInOurDc; public enum ConsistencyLevel { @@ -41,7 +41,7 @@ public enum ConsistencyLevel LOCAL_QUORUM(6, true), EACH_QUORUM (7), SERIAL (8), - LOCAL_SERIAL(9), + LOCAL_SERIAL(9, true), LOCAL_ONE (10, true), NODE_LOCAL (11, true); @@ -173,7 +173,7 @@ public enum ConsistencyLevel break; case LOCAL_ONE: case LOCAL_QUORUM: case LOCAL_SERIAL: // we will only count local replicas towards our response count, as these queries only care about local guarantees - blockFor += countInOurDc(pending).allReplicas(); + blockFor += pending.count(InOurDc.replicas()); break; case ONE: case TWO: case THREE: case QUORUM: case EACH_QUORUM: diff --git a/src/java/org/apache/cassandra/db/CounterMutation.java b/src/java/org/apache/cassandra/db/CounterMutation.java index 3ac1c5d2c0..4f91b83ca2 100644 --- a/src/java/org/apache/cassandra/db/CounterMutation.java +++ b/src/java/org/apache/cassandra/db/CounterMutation.java @@ -21,6 +21,7 @@ import java.io.IOException; import java.util.*; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.Lock; +import java.util.function.Supplier; import com.google.common.base.Function; import com.google.common.base.Objects; @@ -82,6 +83,12 @@ public class CounterMutation implements IMutation return mutation.getPartitionUpdates(); } + @Override + public Supplier hintOnFailure() + { + return null; + } + public void validateSize(int version, int overhead) { long totalSize = serializedSize(version) + overhead; diff --git a/src/java/org/apache/cassandra/db/IMutation.java b/src/java/org/apache/cassandra/db/IMutation.java index 10472c110f..831652bf29 100644 --- a/src/java/org/apache/cassandra/db/IMutation.java +++ b/src/java/org/apache/cassandra/db/IMutation.java @@ -19,6 +19,7 @@ package org.apache.cassandra.db; import java.util.Collection; import java.util.concurrent.TimeUnit; +import java.util.function.Supplier; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.partitions.PartitionUpdate; @@ -35,6 +36,7 @@ public interface IMutation public long getTimeout(TimeUnit unit); public String toString(boolean shallow); public Collection getPartitionUpdates(); + public Supplier hintOnFailure(); public default void validateIndexedColumns() { diff --git a/src/java/org/apache/cassandra/db/Keyspace.java b/src/java/org/apache/cassandra/db/Keyspace.java index 4ef3d022b0..1457b89a69 100644 --- a/src/java/org/apache/cassandra/db/Keyspace.java +++ b/src/java/org/apache/cassandra/db/Keyspace.java @@ -119,6 +119,11 @@ public class Keyspace private static volatile boolean initialized = false; + public static boolean isInitialized() + { + return initialized; + } + public static void setInitialized() { initialized = true; diff --git a/src/java/org/apache/cassandra/db/Mutation.java b/src/java/org/apache/cassandra/db/Mutation.java index 6350082c41..7b6a68670f 100644 --- a/src/java/org/apache/cassandra/db/Mutation.java +++ b/src/java/org/apache/cassandra/db/Mutation.java @@ -21,6 +21,7 @@ import java.io.IOException; import java.util.*; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Supplier; import com.google.common.collect.ImmutableCollection; import com.google.common.collect.ImmutableMap; @@ -44,7 +45,7 @@ import static org.apache.cassandra.net.MessagingService.VERSION_3014; import static org.apache.cassandra.net.MessagingService.VERSION_40; import static org.apache.cassandra.utils.MonotonicClock.Global.approxTime; -public class Mutation implements IMutation +public class Mutation implements IMutation, Supplier { public static final MutationSerializer serializer = new MutationSerializer(); @@ -132,6 +133,18 @@ public class Mutation implements IMutation return modifications.values(); } + @Override + public Supplier hintOnFailure() + { + return this; + } + + @Override + public Mutation get() + { + return this; + } + public void validateSize(int version, int overhead) { long totalSize = serializedSize(version) + overhead; @@ -210,9 +223,14 @@ public class Mutation implements IMutation return ks.applyFuture(this, Keyspace.open(keyspaceName).getMetadata().params.durableWrites, true); } + private void apply(Keyspace keyspace, boolean durableWrites, boolean isDroppable) + { + keyspace.apply(this, durableWrites, true, isDroppable); + } + public void apply(boolean durableWrites, boolean isDroppable) { - Keyspace.open(keyspaceName).apply(this, durableWrites, true, isDroppable); + apply(Keyspace.open(keyspaceName), durableWrites, isDroppable); } public void apply(boolean durableWrites) @@ -226,7 +244,8 @@ public class Mutation implements IMutation */ public void apply() { - apply(Keyspace.open(keyspaceName).getMetadata().params.durableWrites); + Keyspace keyspace = Keyspace.open(keyspaceName); + apply(keyspace, keyspace.getMetadata().params.durableWrites, true); } public void applyUnsafe() diff --git a/src/java/org/apache/cassandra/db/MutationVerbHandler.java b/src/java/org/apache/cassandra/db/MutationVerbHandler.java index 9c0335edff..3077b24f40 100644 --- a/src/java/org/apache/cassandra/db/MutationVerbHandler.java +++ b/src/java/org/apache/cassandra/db/MutationVerbHandler.java @@ -40,19 +40,11 @@ public class MutationVerbHandler implements IVerbHandler public void doVerb(Message message) { // Check if there were any forwarding headers in this message - InetAddressAndPort from = message.respondTo(); - InetAddressAndPort respondToAddress; - if (from == null) - { - respondToAddress = message.from(); - ForwardingInfo forwardTo = message.forwardTo(); - if (forwardTo != null) forwardToLocalNodes(message, forwardTo); - } - else - { - respondToAddress = from; - } + ForwardingInfo forwardTo = message.forwardTo(); + if (forwardTo != null) + forwardToLocalNodes(message, forwardTo); + InetAddressAndPort respondToAddress = message.respondTo(); try { message.payload.applyFuture().addCallback(o -> respond(message, respondToAddress), wto -> failed()); diff --git a/src/java/org/apache/cassandra/db/PartitionRangeReadCommand.java b/src/java/org/apache/cassandra/db/PartitionRangeReadCommand.java index ef128538c6..48bacf6aeb 100644 --- a/src/java/org/apache/cassandra/db/PartitionRangeReadCommand.java +++ b/src/java/org/apache/cassandra/db/PartitionRangeReadCommand.java @@ -293,7 +293,7 @@ public class PartitionRangeReadCommand extends ReadCommand implements PartitionR return dataRange.isReversed(); } - public PartitionIterator execute(ConsistencyLevel consistency, ClientState clientState, long queryStartNanoTime) throws RequestExecutionException + public PartitionIterator execute(ConsistencyLevel consistency, ClientState state, long queryStartNanoTime) throws RequestExecutionException { return StorageProxy.getRangeSlice(this, consistency, queryStartNanoTime); } @@ -512,7 +512,7 @@ public class PartitionRangeReadCommand extends ReadCommand implements PartitionR } @Override - public PartitionIterator execute(ConsistencyLevel consistency, ClientState clientState, long queryStartNanoTime) throws RequestExecutionException + public PartitionIterator execute(ConsistencyLevel consistency, ClientState state, long queryStartNanoTime) throws RequestExecutionException { return executeInternal(executionController()); } diff --git a/src/java/org/apache/cassandra/db/ReadQuery.java b/src/java/org/apache/cassandra/db/ReadQuery.java index f9695d6f61..55a2cf6969 100644 --- a/src/java/org/apache/cassandra/db/ReadQuery.java +++ b/src/java/org/apache/cassandra/db/ReadQuery.java @@ -48,7 +48,7 @@ public interface ReadQuery return ReadExecutionController.empty(); } - public PartitionIterator execute(ConsistencyLevel consistency, ClientState clientState, long queryStartNanoTime) throws RequestExecutionException + public PartitionIterator execute(ConsistencyLevel consistency, ClientState state, long queryStartNanoTime) throws RequestExecutionException { return EmptyIterators.partition(); } @@ -140,12 +140,10 @@ public interface ReadQuery * Executes the query at the provided consistency level. * * @param consistency the consistency level to achieve for the query. - * @param clientState the {@code ClientState} for the query. In practice, this can be null unless - * {@code consistency} is a serial consistency. - * + * @param state client state * @return the result of the query. */ - public PartitionIterator execute(ConsistencyLevel consistency, ClientState clientState, long queryStartNanoTime) throws RequestExecutionException; + public PartitionIterator execute(ConsistencyLevel consistency, ClientState state, long queryStartNanoTime) throws RequestExecutionException; /** * Execute the query for internal queries (that is, it basically executes the query locally). diff --git a/src/java/org/apache/cassandra/db/ReadResponse.java b/src/java/org/apache/cassandra/db/ReadResponse.java index 52e6fd5711..4d34326c8d 100644 --- a/src/java/org/apache/cassandra/db/ReadResponse.java +++ b/src/java/org/apache/cassandra/db/ReadResponse.java @@ -34,6 +34,8 @@ import org.apache.cassandra.net.MessagingService; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.utils.ByteBufferUtil; +import static org.apache.cassandra.db.RepairedDataInfo.NO_OP_REPAIRED_DATA_INFO; + public abstract class ReadResponse { // Serializer for single partition read response @@ -48,6 +50,11 @@ public abstract class ReadResponse return new LocalDataResponse(data, command, rdi); } + public static ReadResponse createSimpleDataResponse(UnfilteredPartitionIterator data, ColumnFilter selection) + { + return new LocalDataResponse(data, selection); + } + @VisibleForTesting public static ReadResponse createRemoteDataResponse(UnfilteredPartitionIterator data, ByteBuffer repairedDataDigest, @@ -184,6 +191,11 @@ public abstract class ReadResponse DeserializationHelper.Flag.LOCAL); } + private LocalDataResponse(UnfilteredPartitionIterator iter, ColumnFilter selection) + { + super(build(iter, selection), null, false, MessagingService.current_version, DeserializationHelper.Flag.LOCAL); + } + private static ByteBuffer build(UnfilteredPartitionIterator iter, ColumnFilter selection) { try (DataOutputBuffer buffer = new DataOutputBuffer()) diff --git a/src/java/org/apache/cassandra/db/SimpleBuilders.java b/src/java/org/apache/cassandra/db/SimpleBuilders.java index c63703a400..3167f8d17e 100644 --- a/src/java/org/apache/cassandra/db/SimpleBuilders.java +++ b/src/java/org/apache/cassandra/db/SimpleBuilders.java @@ -36,7 +36,6 @@ import org.apache.cassandra.db.marshal.*; import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.CounterId; import org.apache.cassandra.utils.FBUtilities; -import org.apache.cassandra.utils.UUIDGen; import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUIDAsBytes; diff --git a/src/java/org/apache/cassandra/db/SinglePartitionReadCommand.java b/src/java/org/apache/cassandra/db/SinglePartitionReadCommand.java index 54a2524145..ccd97681e6 100644 --- a/src/java/org/apache/cassandra/db/SinglePartitionReadCommand.java +++ b/src/java/org/apache/cassandra/db/SinglePartitionReadCommand.java @@ -434,12 +434,13 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar return cmd; } - public PartitionIterator execute(ConsistencyLevel consistency, ClientState clientState, long queryStartNanoTime) throws RequestExecutionException + @Override + public PartitionIterator execute(ConsistencyLevel consistency, ClientState state, long queryStartNanoTime) throws RequestExecutionException { if (clusteringIndexFilter.isEmpty(metadata().comparator)) return EmptyIterators.partition(); - return StorageProxy.read(Group.one(this), consistency, clientState, queryStartNanoTime); + return StorageProxy.read(Group.one(this), consistency, queryStartNanoTime); } protected void recordLatency(TableMetrics metric, long latencyNanos) @@ -1221,9 +1222,9 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar new Group(commands, limits); } - public PartitionIterator execute(ConsistencyLevel consistency, ClientState clientState, long queryStartNanoTime) throws RequestExecutionException + public PartitionIterator execute(ConsistencyLevel consistency, ClientState state, long queryStartNanoTime) throws RequestExecutionException { - return StorageProxy.read(this, consistency, clientState, queryStartNanoTime); + return StorageProxy.read(this, consistency, queryStartNanoTime); } } @@ -1235,13 +1236,13 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar } @Override - public PartitionIterator execute(ConsistencyLevel consistency, ClientState clientState, long queryStartNanoTime) throws RequestExecutionException + public PartitionIterator execute(ConsistencyLevel consistency, ClientState state, long queryStartNanoTime) throws RequestExecutionException { if (queries.size() == 1) - return queries.get(0).execute(consistency, clientState, queryStartNanoTime); + return queries.get(0).execute(consistency, state, queryStartNanoTime); return PartitionIterators.concat(queries.stream() - .map(q -> q.execute(consistency, clientState, queryStartNanoTime)) + .map(q -> q.execute(consistency, state, queryStartNanoTime)) .collect(Collectors.toList())); } } @@ -1314,7 +1315,7 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar } @Override - public PartitionIterator execute(ConsistencyLevel consistency, ClientState clientState, long queryStartNanoTime) throws RequestExecutionException + public PartitionIterator execute(ConsistencyLevel consistency, ClientState state, long queryStartNanoTime) throws RequestExecutionException { return executeInternal(executionController()); } diff --git a/src/java/org/apache/cassandra/db/SystemKeyspace.java b/src/java/org/apache/cassandra/db/SystemKeyspace.java index 69ed5e2431..e837aa51f3 100644 --- a/src/java/org/apache/cassandra/db/SystemKeyspace.java +++ b/src/java/org/apache/cassandra/db/SystemKeyspace.java @@ -34,6 +34,7 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.HashMultimap; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; import com.google.common.collect.SetMultimap; import com.google.common.collect.Sets; import com.google.common.io.ByteStreams; @@ -51,6 +52,7 @@ import org.apache.cassandra.db.compaction.CompactionHistoryTabularData; import org.apache.cassandra.db.marshal.*; import org.apache.cassandra.db.partitions.PartitionUpdate; import org.apache.cassandra.db.rows.Rows; +import org.apache.cassandra.db.rows.Row; import org.apache.cassandra.dht.*; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.io.util.*; @@ -60,10 +62,14 @@ import org.apache.cassandra.metrics.RestorableMeter; import org.apache.cassandra.net.MessagingService; import org.apache.cassandra.schema.*; import org.apache.cassandra.service.StorageService; -import org.apache.cassandra.service.paxos.Commit; -import org.apache.cassandra.service.paxos.PaxosState; import org.apache.cassandra.streaming.StreamOperation; import org.apache.cassandra.transport.ProtocolVersion; +import org.apache.cassandra.service.paxos.*; +import org.apache.cassandra.service.paxos.Commit.Accepted; +import org.apache.cassandra.service.paxos.Commit.AcceptedWithTTL; +import org.apache.cassandra.service.paxos.Commit.Committed; +import org.apache.cassandra.service.paxos.uncommitted.PaxosRows; +import org.apache.cassandra.service.paxos.uncommitted.PaxosUncommittedIndex; import org.apache.cassandra.utils.*; import org.apache.cassandra.utils.concurrent.Future; @@ -71,8 +77,12 @@ import static java.lang.String.format; import static java.util.Collections.emptyMap; import static java.util.Collections.singletonMap; +import static org.apache.cassandra.config.Config.PaxosStatePurging.*; +import static org.apache.cassandra.config.DatabaseDescriptor.paxosStatePurging; import static org.apache.cassandra.cql3.QueryProcessor.executeInternal; +import static org.apache.cassandra.cql3.QueryProcessor.executeInternalWithNowInSec; import static org.apache.cassandra.cql3.QueryProcessor.executeOnceInternal; +import static org.apache.cassandra.service.paxos.Commit.latest; import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis; import static org.apache.cassandra.utils.Clock.Global.nanoTime; import static org.apache.cassandra.utils.FBUtilities.now; @@ -100,6 +110,7 @@ public final class SystemKeyspace public static final String BATCHES = "batches"; public static final String PAXOS = "paxos"; + public static final String PAXOS_REPAIR_HISTORY = "paxos_repair_history"; public static final String BUILT_INDEXES = "IndexInfo"; public static final String LOCAL = "local"; public static final String PEERS_V2 = "peers_v2"; @@ -153,6 +164,7 @@ public final class SystemKeyspace + "row_key blob," + "cf_id UUID," + "in_progress_ballot timeuuid," + + "in_progress_read_ballot timeuuid," + "most_recent_commit blob," + "most_recent_commit_at timeuuid," + "most_recent_commit_version int," @@ -161,6 +173,7 @@ public final class SystemKeyspace + "proposal_version int," + "PRIMARY KEY ((row_key), cf_id))") .compaction(CompactionParams.lcs(emptyMap())) + .indexes(PaxosUncommittedIndex.indexes()) .build(); private static final TableMetadata BuiltIndexes = @@ -173,6 +186,17 @@ public final class SystemKeyspace + "PRIMARY KEY ((table_name), index_name)) ") .build(); + private static final TableMetadata PaxosRepairHistoryTable = + parse(PAXOS_REPAIR_HISTORY, + "paxos repair history", + "CREATE TABLE %s (" + + "keyspace_name text," + + "table_name text," + + "points frozen>>, " + + "PRIMARY KEY (keyspace_name, table_name))" + + "WITH COMMENT='Last successful paxos repairs by range'") + .build(); + private static final TableMetadata Local = parse(LOCAL, "information about the local node", @@ -425,6 +449,7 @@ public final class SystemKeyspace return Tables.of(BuiltIndexes, Batches, Paxos, + PaxosRepairHistoryTable, Local, PeersV2, LegacyPeers, @@ -1179,72 +1204,177 @@ public final class SystemKeyspace return null; } - public static PaxosState loadPaxosState(DecoratedKey key, TableMetadata metadata, int nowInSec) + /** + * Load the current paxos state for the table and key + */ + public static PaxosState.Snapshot loadPaxosState(DecoratedKey partitionKey, TableMetadata metadata, int nowInSec) { - String req = "SELECT * FROM system.%s WHERE row_key = ? AND cf_id = ?"; - UntypedResultSet results = QueryProcessor.executeInternalWithNow(nowInSec, nanoTime(), format(req, PAXOS), key.getKey(), metadata.id.asUUID()); - if (results.isEmpty()) - return new PaxosState(key, metadata); - UntypedResultSet.Row row = results.one(); + String cql = "SELECT * FROM system." + PAXOS + " WHERE row_key = ? AND cf_id = ?"; + List results = QueryProcessor.executeInternalRawWithNow(nowInSec, cql, partitionKey.getKey(), metadata.id.asUUID()).get(partitionKey); + if (results == null || results.isEmpty()) + { + Committed noneCommitted = Committed.none(partitionKey, metadata); + return new PaxosState.Snapshot(Ballot.none(), Ballot.none(), null, noneCommitted); + } + + Row row = results.get(0); + + Ballot promisedWrite = PaxosRows.getWritePromise(row); + Ballot promised = latest(promisedWrite, PaxosRows.getPromise(row)); - Commit promised = row.has("in_progress_ballot") - ? new Commit(row.getTimeUUID("in_progress_ballot"), new PartitionUpdate.Builder(metadata, key, metadata.regularAndStaticColumns(), 1).build()) - : Commit.emptyCommit(key, metadata); // either we have both a recently accepted ballot and update or we have neither - Commit accepted = row.has("proposal_version") && row.has("proposal") - ? new Commit(row.getTimeUUID("proposal_ballot"), - PartitionUpdate.fromBytes(row.getBytes("proposal"), row.getInt("proposal_version"))) - : Commit.emptyCommit(key, metadata); - // either most_recent_commit and most_recent_commit_at will both be set, or neither - Commit mostRecent = row.has("most_recent_commit_version") && row.has("most_recent_commit") - ? new Commit(row.getTimeUUID("most_recent_commit_at"), - PartitionUpdate.fromBytes(row.getBytes("most_recent_commit"), row.getInt("most_recent_commit_version"))) - : Commit.emptyCommit(key, metadata); - return new PaxosState(promised, accepted, mostRecent); + Accepted accepted = PaxosRows.getAccepted(row); + Committed committed = PaxosRows.getCommitted(metadata, partitionKey, row); + // fix a race with TTL/deletion resolution, where TTL expires after equal deletion is inserted; TTL wins the resolution, and is read using an old ballot's nowInSec + if (accepted != null && !accepted.isAfter(committed)) + accepted = null; + + return new PaxosState.Snapshot(promised, promisedWrite, accepted, committed); } - public static void savePaxosPromise(Commit promise) + public static int legacyPaxosTtlSec(TableMetadata metadata) { - String req = "UPDATE system.%s USING TIMESTAMP ? AND TTL ? SET in_progress_ballot = ? WHERE row_key = ? AND cf_id = ?"; - executeInternal(format(req, PAXOS), - promise.ballot.unixMicros(), - paxosTtlSec(promise.update.metadata()), - promise.ballot, - promise.update.partitionKey().getKey(), - promise.update.metadata().id.asUUID()); + // keep paxos state around for at least 3h + return Math.max(3 * 3600, metadata.params.gcGraceSeconds); + } + + public static void savePaxosWritePromise(DecoratedKey key, TableMetadata metadata, Ballot ballot) + { + if (paxosStatePurging() == legacy) + { + String cql = "UPDATE system." + PAXOS + " USING TIMESTAMP ? AND TTL ? SET in_progress_ballot = ? WHERE row_key = ? AND cf_id = ?"; + executeInternal(cql, + ballot.unixMicros(), + legacyPaxosTtlSec(metadata), + ballot, + key.getKey(), + metadata.id.asUUID()); + } + else + { + String cql = "UPDATE system." + PAXOS + " USING TIMESTAMP ? SET in_progress_ballot = ? WHERE row_key = ? AND cf_id = ?"; + executeInternal(cql, + ballot.unixMicros(), + ballot, + key.getKey(), + metadata.id.asUUID()); + } + } + + public static void savePaxosReadPromise(DecoratedKey key, TableMetadata metadata, Ballot ballot) + { + if (paxosStatePurging() == legacy) + { + String cql = "UPDATE system." + PAXOS + " USING TIMESTAMP ? AND TTL ? SET in_progress_read_ballot = ? WHERE row_key = ? AND cf_id = ?"; + executeInternal(cql, + ballot.unixMicros(), + legacyPaxosTtlSec(metadata), + ballot, + key.getKey(), + metadata.id.asUUID()); + } + else + { + String cql = "UPDATE system." + PAXOS + " USING TIMESTAMP ? SET in_progress_read_ballot = ? WHERE row_key = ? AND cf_id = ?"; + executeInternal(cql, + ballot.unixMicros(), + ballot, + key.getKey(), + metadata.id.asUUID()); + } } public static void savePaxosProposal(Commit proposal) { - executeInternal(format("UPDATE system.%s USING TIMESTAMP ? AND TTL ? SET proposal_ballot = ?, proposal = ?, proposal_version = ? WHERE row_key = ? AND cf_id = ?", PAXOS), - proposal.ballot.unixMicros(), - paxosTtlSec(proposal.update.metadata()), - proposal.ballot, - PartitionUpdate.toBytes(proposal.update, MessagingService.current_version), - MessagingService.current_version, - proposal.update.partitionKey().getKey(), - proposal.update.metadata().id.asUUID()); - } - - public static int paxosTtlSec(TableMetadata metadata) - { - // keep paxos state around for at least 3h - return Math.max(3 * 3600, metadata.params.gcGraceSeconds); + if (proposal instanceof AcceptedWithTTL) + { + int localDeletionTime = ((Commit.AcceptedWithTTL) proposal).localDeletionTime; + int ttlInSec = legacyPaxosTtlSec(proposal.update.metadata()); + int nowInSec = localDeletionTime - ttlInSec; + String cql = "UPDATE system." + PAXOS + " USING TIMESTAMP ? AND TTL ? SET proposal_ballot = ?, proposal = ?, proposal_version = ? WHERE row_key = ? AND cf_id = ?"; + executeInternalWithNowInSec(cql, + nowInSec, + proposal.ballot.unixMicros(), + ttlInSec, + proposal.ballot, + PartitionUpdate.toBytes(proposal.update, MessagingService.current_version), + MessagingService.current_version, + proposal.update.partitionKey().getKey(), + proposal.update.metadata().id.asUUID()); + } + else + { + String cql = "UPDATE system." + PAXOS + " USING TIMESTAMP ? SET proposal_ballot = ?, proposal = ?, proposal_version = ? WHERE row_key = ? AND cf_id = ?"; + executeInternal(cql, + proposal.ballot.unixMicros(), + proposal.ballot, + PartitionUpdate.toBytes(proposal.update, MessagingService.current_version), + MessagingService.current_version, + proposal.update.partitionKey().getKey(), + proposal.update.metadata().id.asUUID()); + } } public static void savePaxosCommit(Commit commit) { // We always erase the last proposal (with the commit timestamp to no erase more recent proposal in case the commit is old) // even though that's really just an optimization since SP.beginAndRepairPaxos will exclude accepted proposal older than the mrc. - String cql = "UPDATE system.%s USING TIMESTAMP ? AND TTL ? SET proposal_ballot = null, proposal = null, most_recent_commit_at = ?, most_recent_commit = ?, most_recent_commit_version = ? WHERE row_key = ? AND cf_id = ?"; - executeInternal(format(cql, PAXOS), - commit.ballot.unixMicros(), - paxosTtlSec(commit.update.metadata()), - commit.ballot, - PartitionUpdate.toBytes(commit.update, MessagingService.current_version), - MessagingService.current_version, - commit.update.partitionKey().getKey(), - commit.update.metadata().id.asUUID()); + if (commit instanceof Commit.CommittedWithTTL) + { + int localDeletionTime = ((Commit.CommittedWithTTL) commit).localDeletionTime; + int ttlInSec = legacyPaxosTtlSec(commit.update.metadata()); + int nowInSec = localDeletionTime - ttlInSec; + String cql = "UPDATE system." + PAXOS + " USING TIMESTAMP ? AND TTL ? SET proposal_ballot = null, proposal = null, proposal_version = null, most_recent_commit_at = ?, most_recent_commit = ?, most_recent_commit_version = ? WHERE row_key = ? AND cf_id = ?"; + executeInternalWithNowInSec(cql, + nowInSec, + commit.ballot.unixMicros(), + ttlInSec, + commit.ballot, + PartitionUpdate.toBytes(commit.update, MessagingService.current_version), + MessagingService.current_version, + commit.update.partitionKey().getKey(), + commit.update.metadata().id.asUUID()); + } + else + { + String cql = "UPDATE system." + PAXOS + " USING TIMESTAMP ? SET proposal_ballot = null, proposal = null, proposal_version = null, most_recent_commit_at = ?, most_recent_commit = ?, most_recent_commit_version = ? WHERE row_key = ? AND cf_id = ?"; + executeInternal(cql, + commit.ballot.unixMicros(), + commit.ballot, + PartitionUpdate.toBytes(commit.update, MessagingService.current_version), + MessagingService.current_version, + commit.update.partitionKey().getKey(), + commit.update.metadata().id.asUUID()); + } + } + + @VisibleForTesting + public static void savePaxosRepairHistory(String keyspace, String table, PaxosRepairHistory history, boolean flush) + { + String cql = "INSERT INTO system.%s (keyspace_name, table_name, points) VALUES (?, ?, ?)"; + executeInternal(String.format(cql, PAXOS_REPAIR_HISTORY), keyspace, table, history.toTupleBufferList()); + if (flush) + flushPaxosRepairHistory(); + } + + public static void flushPaxosRepairHistory() + { + Schema.instance.getColumnFamilyStoreInstance(PaxosRepairHistoryTable.id).forceBlockingFlush(); + } + + public static PaxosRepairHistory loadPaxosRepairHistory(String keyspace, String table) + { + if (SchemaConstants.LOCAL_SYSTEM_KEYSPACE_NAMES.contains(keyspace)) + return PaxosRepairHistory.EMPTY; + + UntypedResultSet results = executeInternal(String.format("SELECT * FROM system.%s WHERE keyspace_name=? AND table_name=?", PAXOS_REPAIR_HISTORY), keyspace, table); + if (results.isEmpty()) + return PaxosRepairHistory.EMPTY; + + UntypedResultSet.Row row = Iterables.getOnlyElement(results); + List points = row.getList("points", BytesType.instance); + + return PaxosRepairHistory.fromTupleBufferList(points); } /** diff --git a/src/java/org/apache/cassandra/db/commitlog/CommitLogReplayer.java b/src/java/org/apache/cassandra/db/commitlog/CommitLogReplayer.java index b59480e05a..cbed01885b 100644 --- a/src/java/org/apache/cassandra/db/commitlog/CommitLogReplayer.java +++ b/src/java/org/apache/cassandra/db/commitlog/CommitLogReplayer.java @@ -52,7 +52,6 @@ import org.apache.cassandra.schema.Schema; import org.apache.cassandra.schema.SchemaConstants; import org.apache.cassandra.schema.TableId; import org.apache.cassandra.schema.TableMetadataRef; -import org.apache.cassandra.service.StorageService; import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.WrappedRunnable; diff --git a/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java b/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java index 4aec19ff3c..5fe1df70bd 100644 --- a/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java +++ b/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java @@ -38,7 +38,6 @@ import org.apache.cassandra.db.lifecycle.LifecycleTransaction; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; import org.apache.cassandra.exceptions.ConfigurationException; -import org.apache.cassandra.io.sstable.Component; import org.apache.cassandra.io.sstable.ISSTableScanner; import org.apache.cassandra.io.sstable.metadata.MetadataCollector; import org.apache.cassandra.io.sstable.metadata.StatsMetadata; diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionInterruptedException.java b/src/java/org/apache/cassandra/db/compaction/CompactionInterruptedException.java index 809c10c8cf..b9174ec262 100644 --- a/src/java/org/apache/cassandra/db/compaction/CompactionInterruptedException.java +++ b/src/java/org/apache/cassandra/db/compaction/CompactionInterruptedException.java @@ -17,16 +17,14 @@ */ package org.apache.cassandra.db.compaction; +import org.apache.cassandra.utils.Shared; + +@Shared public class CompactionInterruptedException extends RuntimeException { private static final long serialVersionUID = -8651427062512310398L; - public CompactionInterruptedException(CompactionInfo info) - { - super("Compaction interrupted: " + info); - } - - public CompactionInterruptedException(String info) + public CompactionInterruptedException(Object info) { super("Compaction interrupted: " + info); } diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionIterator.java b/src/java/org/apache/cassandra/db/compaction/CompactionIterator.java index fc4dc9c721..86ab5ffc6c 100644 --- a/src/java/org/apache/cassandra/db/compaction/CompactionIterator.java +++ b/src/java/org/apache/cassandra/db/compaction/CompactionIterator.java @@ -19,11 +19,17 @@ package org.apache.cassandra.db.compaction; import java.util.*; import java.util.function.LongPredicate; +import java.util.concurrent.TimeUnit; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Ordering; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.dht.Token; import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.schema.Schema; +import org.apache.cassandra.schema.SchemaConstants; +import org.apache.cassandra.schema.TableId; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.db.transform.DuplicateRowChecker; @@ -37,8 +43,14 @@ import org.apache.cassandra.db.transform.Transformation; import org.apache.cassandra.index.transactions.CompactionTransaction; import org.apache.cassandra.io.sstable.ISSTableScanner; import org.apache.cassandra.schema.CompactionParams.TombstoneOption; +import org.apache.cassandra.service.paxos.PaxosRepairHistory; +import org.apache.cassandra.service.paxos.uncommitted.PaxosRows; import org.apache.cassandra.utils.TimeUUID; +import static java.util.concurrent.TimeUnit.MICROSECONDS; +import static org.apache.cassandra.config.Config.PaxosStatePurging.legacy; +import static org.apache.cassandra.config.DatabaseDescriptor.paxosStatePurging; + /** * Merge multiple iterators over the content of sstable into a "compacted" iterator. *

@@ -110,7 +122,10 @@ public class CompactionIterator extends CompactionInfo.Holder implements Unfilte ? EmptyIterators.unfilteredPartition(controller.cfs.metadata()) : UnfilteredPartitionIterators.merge(scanners, listener()); merged = Transformation.apply(merged, new GarbageSkipper(controller)); - merged = Transformation.apply(merged, new Purger(controller, nowInSec)); + Transformation purger = isPaxos(controller.cfs) && paxosStatePurging() != legacy + ? new PaxosPurger(nowInSec) + : new Purger(controller, nowInSec); + merged = Transformation.apply(merged, purger); merged = DuplicateRowChecker.duringCompaction(merged, type); compacted = Transformation.apply(merged, new AbortableUnfilteredPartitionTransformation(this)); } @@ -556,6 +571,78 @@ public class CompactionIterator extends CompactionInfo.Holder implements Unfilte } } + private class PaxosPurger extends Transformation + { + private final long nowInSec; + private final long paxosPurgeGraceMicros = DatabaseDescriptor.getPaxosPurgeGrace(MICROSECONDS); + private final Map tableIdToHistory = new HashMap<>(); + private Token currentToken; + private int compactedUnfiltered; + + private PaxosPurger(long nowInSec) + { + this.nowInSec = nowInSec; + } + + protected void onEmptyPartitionPostPurge(DecoratedKey key) + { + if (type == OperationType.COMPACTION) + controller.cfs.invalidateCachedPartition(key); + } + + protected void updateProgress() + { + if ((++compactedUnfiltered) % UNFILTERED_TO_UPDATE_PROGRESS == 0) + updateBytesRead(); + } + + @Override + @SuppressWarnings("resource") + protected UnfilteredRowIterator applyToPartition(UnfilteredRowIterator partition) + { + currentToken = partition.partitionKey().getToken(); + UnfilteredRowIterator purged = Transformation.apply(partition, this); + if (purged.isEmpty()) + { + onEmptyPartitionPostPurge(purged.partitionKey()); + purged.close(); + return null; + } + + return purged; + } + + @Override + protected Row applyToRow(Row row) + { + updateProgress(); + TableId tableId = PaxosRows.getTableId(row); + + switch (paxosStatePurging()) + { + default: throw new AssertionError(); + case legacy: + case gc_grace: + { + TableMetadata metadata = Schema.instance.getTableMetadata(tableId); + return row.purgeDataOlderThan(TimeUnit.SECONDS.toMicros(nowInSec - (metadata == null ? (3 * 3600) : metadata.params.gcGraceSeconds)), false); + } + case repaired: + { + PaxosRepairHistory.Searcher history = tableIdToHistory.computeIfAbsent(tableId, find -> { + TableMetadata metadata = Schema.instance.getTableMetadata(find); + if (metadata == null) + return null; + return Keyspace.openAndGetStore(metadata).getPaxosRepairHistory().searcher(); + }); + + return history == null ? row : + row.purgeDataOlderThan(history.ballotForToken(currentToken).unixMicros() - paxosPurgeGraceMicros, false); + } + } + } + } + private static class AbortableUnfilteredPartitionTransformation extends Transformation { private final AbortableUnfilteredRowTransformation abortableIter; @@ -574,7 +661,7 @@ public class CompactionIterator extends CompactionInfo.Holder implements Unfilte } } - private static class AbortableUnfilteredRowTransformation extends Transformation + private static class AbortableUnfilteredRowTransformation extends Transformation { private final CompactionIterator iter; @@ -590,4 +677,9 @@ public class CompactionIterator extends CompactionInfo.Holder implements Unfilte return row; } } + + private static boolean isPaxos(ColumnFamilyStore cfs) + { + return cfs.name.equals(SystemKeyspace.PAXOS) && cfs.keyspace.getName().equals(SchemaConstants.SYSTEM_KEYSPACE_NAME); + } } diff --git a/src/java/org/apache/cassandra/db/marshal/ByteArrayAccessor.java b/src/java/org/apache/cassandra/db/marshal/ByteArrayAccessor.java index bb26cba460..df24a627a4 100644 --- a/src/java/org/apache/cassandra/db/marshal/ByteArrayAccessor.java +++ b/src/java/org/apache/cassandra/db/marshal/ByteArrayAccessor.java @@ -29,6 +29,7 @@ import org.apache.cassandra.db.Digest; import org.apache.cassandra.db.TypeSizes; import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.service.paxos.Ballot; import org.apache.cassandra.utils.TimeUUID; import org.apache.cassandra.utils.ByteArrayUtil; import org.apache.cassandra.utils.ByteBufferUtil; @@ -241,6 +242,12 @@ public class ByteArrayAccessor implements ValueAccessor return TimeUUID.fromBytes(getLong(value, 0), getLong(value, 8)); } + @Override + public Ballot toBallot(byte[] value) + { + return Ballot.deserialize(value); + } + @Override public int putShort(byte[] dst, int offset, short value) { diff --git a/src/java/org/apache/cassandra/db/marshal/ByteBufferAccessor.java b/src/java/org/apache/cassandra/db/marshal/ByteBufferAccessor.java index e428b7cd04..40a3bf4b34 100644 --- a/src/java/org/apache/cassandra/db/marshal/ByteBufferAccessor.java +++ b/src/java/org/apache/cassandra/db/marshal/ByteBufferAccessor.java @@ -28,6 +28,7 @@ import org.apache.cassandra.db.Digest; import org.apache.cassandra.db.TypeSizes; import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.service.paxos.Ballot; import org.apache.cassandra.utils.TimeUUID; import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.FastByteOperations; @@ -245,6 +246,12 @@ public class ByteBufferAccessor implements ValueAccessor return TimeUUID.fromBytes(value.getLong(value.position()), value.getLong(value.position() + 8)); } + @Override + public Ballot toBallot(ByteBuffer value) + { + return Ballot.deserialize(value); + } + @Override public int putShort(ByteBuffer dst, int offset, short value) { diff --git a/src/java/org/apache/cassandra/db/marshal/TupleType.java b/src/java/org/apache/cassandra/db/marshal/TupleType.java index 83fbb25d54..cc08487658 100644 --- a/src/java/org/apache/cassandra/db/marshal/TupleType.java +++ b/src/java/org/apache/cassandra/db/marshal/TupleType.java @@ -205,9 +205,17 @@ public class TupleType extends AbstractType */ public ByteBuffer[] split(ByteBuffer value) { - ByteBuffer[] components = new ByteBuffer[size()]; + return split(value, size(), this); + } + + /** + * Split a tuple value into its component values. + */ + public static ByteBuffer[] split(ByteBuffer value, int numberOfElements, TupleType type) + { + ByteBuffer[] components = new ByteBuffer[numberOfElements]; ByteBuffer input = value.duplicate(); - for (int i = 0; i < size(); i++) + for (int i = 0; i < numberOfElements; i++) { if (!input.hasRemaining()) return Arrays.copyOfRange(components, 0, i); @@ -226,7 +234,7 @@ public class TupleType extends AbstractType { throw new InvalidRequestException(String.format( "Expected %s %s for %s column, but got more", - size(), size() == 1 ? "value" : "values", this.asCQL3Type())); + numberOfElements, numberOfElements == 1 ? "value" : "values", type.asCQL3Type())); } return components; diff --git a/src/java/org/apache/cassandra/db/marshal/ValueAccessor.java b/src/java/org/apache/cassandra/db/marshal/ValueAccessor.java index 2f089a6c49..a51836e65a 100644 --- a/src/java/org/apache/cassandra/db/marshal/ValueAccessor.java +++ b/src/java/org/apache/cassandra/db/marshal/ValueAccessor.java @@ -37,6 +37,7 @@ import org.apache.cassandra.db.rows.CellPath; import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputPlus; import org.apache.cassandra.schema.ColumnMetadata; +import org.apache.cassandra.service.paxos.Ballot; import org.apache.cassandra.utils.TimeUUID; import static org.apache.cassandra.db.ClusteringPrefix.Kind.*; @@ -326,6 +327,9 @@ public interface ValueAccessor /** returns a TimeUUID from offset 0 */ TimeUUID toTimeUUID(V value); + /** returns a TimeUUID from offset 0 */ + Ballot toBallot(V value); + /** * writes the short value {@param value} to {@param dst} at offset {@param offset} * @return the number of bytes written to {@param value} diff --git a/src/java/org/apache/cassandra/db/partitions/AbstractBTreePartition.java b/src/java/org/apache/cassandra/db/partitions/AbstractBTreePartition.java index 1d6603eec2..b51a6787a0 100644 --- a/src/java/org/apache/cassandra/db/partitions/AbstractBTreePartition.java +++ b/src/java/org/apache/cassandra/db/partitions/AbstractBTreePartition.java @@ -375,26 +375,52 @@ public abstract class AbstractBTreePartition implements Partition, Iterable @Override public String toString() { - StringBuilder sb = new StringBuilder(); + return toString(true); + } - sb.append(String.format("[%s] key=%s partition_deletion=%s columns=%s", - metadata(), - metadata().partitionKeyType.getString(partitionKey().getKey()), - partitionLevelDeletion(), - columns())); + public String toString(boolean includeFullDetails) + { + StringBuilder sb = new StringBuilder(); + if (includeFullDetails) + { + sb.append(String.format("[%s.%s] key=%s partition_deletion=%s columns=%s", + metadata().keyspace, + metadata().name, + metadata().partitionKeyType.getString(partitionKey().getKey()), + partitionLevelDeletion(), + columns())); + } + else + { + sb.append("key=").append(metadata().partitionKeyType.getString(partitionKey().getKey())); + } if (staticRow() != Rows.EMPTY_STATIC_ROW) - sb.append("\n ").append(staticRow().toString(metadata(), true)); + sb.append("\n ").append(staticRow().toString(metadata(), includeFullDetails)); try (UnfilteredRowIterator iter = unfilteredIterator()) { while (iter.hasNext()) - sb.append("\n ").append(iter.next().toString(metadata(), true)); + sb.append("\n ").append(iter.next().toString(metadata(), includeFullDetails)); } - return sb.toString(); } + @Override + public boolean equals(Object obj) + { + if (!(obj instanceof PartitionUpdate)) + return false; + + PartitionUpdate that = (PartitionUpdate) obj; + Holder a = this.holder(), b = that.holder(); + return partitionKey.equals(that.partitionKey) + && metadata().id.equals(that.metadata().id) + && a.deletionInfo.equals(b.deletionInfo) + && a.staticRow.equals(b.staticRow) + && Iterators.elementsEqual(iterator(), that.iterator()); + } + public int rowCount() { return BTree.size(holder().tree); diff --git a/src/java/org/apache/cassandra/db/partitions/PartitionUpdate.java b/src/java/org/apache/cassandra/db/partitions/PartitionUpdate.java index 9530e10d3a..f6fd259e88 100644 --- a/src/java/org/apache/cassandra/db/partitions/PartitionUpdate.java +++ b/src/java/org/apache/cassandra/db/partitions/PartitionUpdate.java @@ -17,13 +17,16 @@ */ package org.apache.cassandra.db.partitions; +import java.io.EOFException; import java.io.IOException; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; import com.google.common.collect.Iterables; +import com.google.common.collect.Iterators; import com.google.common.collect.Lists; +import com.google.common.primitives.Ints; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -38,6 +41,9 @@ import org.apache.cassandra.schema.TableId; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.utils.btree.BTree; import org.apache.cassandra.utils.btree.UpdateFunction; +import org.apache.cassandra.utils.vint.VIntCoding; + +import static org.apache.cassandra.db.rows.UnfilteredRowIteratorSerializer.IS_EMPTY; /** * Stores updates made on a partition. @@ -322,23 +328,19 @@ public class PartitionUpdate extends AbstractBTreePartition */ public int dataSize() { - int size = 0; + return Ints.saturatedCast(BTree.accumulate(holder.tree, (row, value) -> row.dataSize() + value, 0L) + + holder.staticRow.dataSize() + holder.deletionInfo.dataSize()); + } - if (holder.staticRow != null) - { - for (ColumnData cd : holder.staticRow.columnData()) - { - size += cd.dataSize(); - } - } - - for (Row row : this) - { - size += row.clustering().dataSize(); - for (ColumnData cd : row) - size += cd.dataSize(); - } - return size; + /** + * The size of the data contained in this update. + * + * @return the size of the data contained in this update. + */ + public long unsharedHeapSize() + { + return BTree.accumulate(holder.tree, (row, value) -> row.unsharedHeapSize() + value, 0L) + + holder.staticRow.unsharedHeapSize() + holder.deletionInfo.unsharedHeapSize(); } public TableMetadata metadata() @@ -667,6 +669,21 @@ public class PartitionUpdate extends AbstractBTreePartition false); } + public static boolean isEmpty(ByteBuffer in, DeserializationHelper.Flag flag, DecoratedKey key) throws IOException + { + int position = in.position(); + position += 16; // CFMetaData.serializer.deserialize(in, version); + if (position >= in.limit()) + throw new EOFException(); + // DecoratedKey key = metadata.decorateKey(ByteBufferUtil.readWithVIntLength(in)); + int keyLength = (int) VIntCoding.getUnsignedVInt(in, position); + position += keyLength + VIntCoding.computeUnsignedVIntSize(keyLength); + if (position >= in.limit()) + throw new EOFException(); + int flags = in.get(position) & 0xff; + return (flags & IS_EMPTY) != 0; + } + public long serializedSize(PartitionUpdate update, int version) { try (UnfilteredRowIterator iter = update.unfilteredIterator()) diff --git a/src/java/org/apache/cassandra/db/rows/AbstractCell.java b/src/java/org/apache/cassandra/db/rows/AbstractCell.java index 5d4fc3c19e..21d2c10e98 100644 --- a/src/java/org/apache/cassandra/db/rows/AbstractCell.java +++ b/src/java/org/apache/cassandra/db/rows/AbstractCell.java @@ -98,6 +98,11 @@ public abstract class AbstractCell extends Cell return this; } + public Cell purgeDataOlderThan(long timestamp) + { + return this.timestamp() < timestamp ? null : this; + } + public Cell copy(AbstractAllocator allocator) { CellPath path = path(); diff --git a/src/java/org/apache/cassandra/db/rows/BTreeRow.java b/src/java/org/apache/cassandra/db/rows/BTreeRow.java index 2d3ee83de5..64b494f953 100644 --- a/src/java/org/apache/cassandra/db/rows/BTreeRow.java +++ b/src/java/org/apache/cassandra/db/rows/BTreeRow.java @@ -474,6 +474,18 @@ public class BTreeRow extends AbstractRow return transformAndFilter(newInfo, newDeletion, (cd) -> cd.purge(purger, nowInSec)); } + public Row purgeDataOlderThan(long timestamp, boolean enforceStrictLiveness) + { + LivenessInfo newInfo = primaryKeyLivenessInfo.timestamp() < timestamp ? LivenessInfo.EMPTY : primaryKeyLivenessInfo; + Deletion newDeletion = deletion.time().markedForDeleteAt() < timestamp ? Deletion.LIVE : deletion; + + // when enforceStrictLiveness is set, a row is considered dead when it's PK liveness info is not present + if (enforceStrictLiveness && newDeletion.isLive() && newInfo.isEmpty()) + return null; + + return transformAndFilter(newInfo, newDeletion, cd -> cd.purgeDataOlderThan(timestamp)); + } + private Row transformAndFilter(LivenessInfo info, Deletion deletion, Function function) { Object[] transformed = BTree.transformAndFilter(btree, function); diff --git a/src/java/org/apache/cassandra/db/rows/BufferCell.java b/src/java/org/apache/cassandra/db/rows/BufferCell.java index 85d28f8b46..171dfbb0d7 100644 --- a/src/java/org/apache/cassandra/db/rows/BufferCell.java +++ b/src/java/org/apache/cassandra/db/rows/BufferCell.java @@ -26,7 +26,6 @@ import org.apache.cassandra.schema.ColumnMetadata; import org.apache.cassandra.db.marshal.ByteType; import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.ObjectSizes; -import org.apache.cassandra.utils.memory.AbstractAllocator; import static java.lang.String.format; @@ -134,14 +133,6 @@ public class BufferCell extends AbstractCell return withUpdatedValue(ByteBufferUtil.EMPTY_BYTE_BUFFER); } - public Cell copy(AbstractAllocator allocator) - { - if (!value.hasRemaining()) - return this; - - return new BufferCell(column, timestamp, ttl, localDeletionTime, allocator.clone(value), path == null ? null : path.copy(allocator)); - } - @Override public long unsharedHeapSize() { diff --git a/src/java/org/apache/cassandra/db/rows/Cell.java b/src/java/org/apache/cassandra/db/rows/Cell.java index 38d1c84880..2a19a986b3 100644 --- a/src/java/org/apache/cassandra/db/rows/Cell.java +++ b/src/java/org/apache/cassandra/db/rows/Cell.java @@ -167,6 +167,10 @@ public abstract class Cell extends ColumnData // Overrides super type to provide a more precise return type. public abstract Cell purge(DeletionPurger purger, int nowInSec); + @Override + // Overrides super type to provide a more precise return type. + public abstract Cell purgeDataOlderThan(long timestamp); + /** * The serialization format for cell is: * [ flags ][ timestamp ][ deletion time ][ ttl ][ path size ][ path ][ value size ][ value ] diff --git a/src/java/org/apache/cassandra/db/rows/CellPath.java b/src/java/org/apache/cassandra/db/rows/CellPath.java index cee24840b5..817a2d0314 100644 --- a/src/java/org/apache/cassandra/db/rows/CellPath.java +++ b/src/java/org/apache/cassandra/db/rows/CellPath.java @@ -65,6 +65,8 @@ public abstract class CellPath implements IMeasurableMemory public abstract long unsharedHeapSizeExcludingData(); + public abstract long unsharedHeapSize(); + @Override public final int hashCode() { diff --git a/src/java/org/apache/cassandra/db/rows/ColumnData.java b/src/java/org/apache/cassandra/db/rows/ColumnData.java index 4146946616..8ac19cc496 100644 --- a/src/java/org/apache/cassandra/db/rows/ColumnData.java +++ b/src/java/org/apache/cassandra/db/rows/ColumnData.java @@ -58,6 +58,8 @@ public abstract class ColumnData implements IMeasurableMemory public abstract long unsharedHeapSizeExcludingData(); + public abstract long unsharedHeapSize(); + /** * Validate the column data. * @@ -95,6 +97,7 @@ public abstract class ColumnData implements IMeasurableMemory public abstract ColumnData markCounterLocalToBeCleared(); public abstract ColumnData purge(DeletionPurger purger, int nowInSec); + public abstract ColumnData purgeDataOlderThan(long timestamp); public abstract long maxTimestamp(); } diff --git a/src/java/org/apache/cassandra/db/rows/ComplexColumnData.java b/src/java/org/apache/cassandra/db/rows/ComplexColumnData.java index bf7714d9df..84f52e10bd 100644 --- a/src/java/org/apache/cassandra/db/rows/ComplexColumnData.java +++ b/src/java/org/apache/cassandra/db/rows/ComplexColumnData.java @@ -121,13 +121,10 @@ public class ComplexColumnData extends ColumnData implements Iterable> return size; } - @Override public long unsharedHeapSize() { - long heapSize = EMPTY_SIZE + ObjectSizes.sizeOfArray(cells); - for (Cell cell : this) - heapSize += cell.unsharedHeapSize(); - return heapSize; + long heapSize = EMPTY_SIZE + ObjectSizes.sizeOfArray(cells) + complexDeletion.unsharedHeapSize(); + return BTree.accumulate(cells, (cell, value) -> value + cell.unsharedHeapSize(), heapSize); } @Override @@ -205,6 +202,12 @@ public class ComplexColumnData extends ColumnData implements Iterable> return transformAndFilter(complexDeletion, (cell) -> filter.fetchedCellIsQueried(column, cell.path()) ? null : cell); } + public ComplexColumnData purgeDataOlderThan(long timestamp) + { + DeletionTime newDeletion = complexDeletion.markedForDeleteAt() < timestamp ? DeletionTime.LIVE : complexDeletion; + return transformAndFilter(newDeletion, (cell) -> cell.purgeDataOlderThan(timestamp)); + } + private ComplexColumnData transformAndFilter(DeletionTime newDeletion, Function, ? extends Cell> function) { Object[] transformed = BTree.transformAndFilter(cells, function); diff --git a/src/java/org/apache/cassandra/db/rows/NativeCell.java b/src/java/org/apache/cassandra/db/rows/NativeCell.java index 03dfc7090f..1267560aa2 100644 --- a/src/java/org/apache/cassandra/db/rows/NativeCell.java +++ b/src/java/org/apache/cassandra/db/rows/NativeCell.java @@ -177,5 +177,4 @@ public class NativeCell extends AbstractCell { return EMPTY_SIZE; } - } diff --git a/src/java/org/apache/cassandra/db/rows/Row.java b/src/java/org/apache/cassandra/db/rows/Row.java index 85d27a44eb..7575f06552 100644 --- a/src/java/org/apache/cassandra/db/rows/Row.java +++ b/src/java/org/apache/cassandra/db/rows/Row.java @@ -116,7 +116,7 @@ public interface Row extends Unfiltered, Iterable, IMeasurableMemory * * @param nowInSec the current time to decide what is deleted and what isn't * @param enforceStrictLiveness whether the row should be purged if there is no PK liveness info, - * normally retrieved from {@link CFMetaData#enforceStrictLiveness()} + * normally retrieved from {@link TableMetadata#enforceStrictLiveness()} * @return true if there is some live information */ public boolean hasLiveData(int nowInSec, boolean enforceStrictLiveness); @@ -249,6 +249,11 @@ public interface Row extends Unfiltered, Iterable, IMeasurableMemory */ public Row withOnlyQueriedData(ColumnFilter filter); + /* + * Returns a copy of this row without any data with a timestamp older than the one provided + */ + public Row purgeDataOlderThan(long timestamp, boolean enforceStrictLiveness); + /** * Returns a copy of this row where all counter cells have they "local" shard marked for clearing. */ @@ -281,6 +286,7 @@ public interface Row extends Unfiltered, Iterable, IMeasurableMemory public long unsharedHeapSizeExcludingData(); public String toString(TableMetadata metadata, boolean fullDetails); + public long unsharedHeapSize(); /** * Apply a function to every column in a row diff --git a/src/java/org/apache/cassandra/db/rows/UnfilteredRowIteratorSerializer.java b/src/java/org/apache/cassandra/db/rows/UnfilteredRowIteratorSerializer.java index 938a3eed11..11541ee358 100644 --- a/src/java/org/apache/cassandra/db/rows/UnfilteredRowIteratorSerializer.java +++ b/src/java/org/apache/cassandra/db/rows/UnfilteredRowIteratorSerializer.java @@ -66,7 +66,7 @@ public class UnfilteredRowIteratorSerializer { protected static final Logger logger = LoggerFactory.getLogger(UnfilteredRowIteratorSerializer.class); - private static final int IS_EMPTY = 0x01; + public static final int IS_EMPTY = 0x01; private static final int IS_REVERSED = 0x02; private static final int HAS_PARTITION_DELETION = 0x04; private static final int HAS_STATIC_ROW = 0x08; diff --git a/src/java/org/apache/cassandra/db/view/ViewUtils.java b/src/java/org/apache/cassandra/db/view/ViewUtils.java index c248ddcde7..55a462c31b 100644 --- a/src/java/org/apache/cassandra/db/view/ViewUtils.java +++ b/src/java/org/apache/cassandra/db/view/ViewUtils.java @@ -23,7 +23,6 @@ import java.util.function.Predicate; import com.google.common.collect.Iterables; import org.apache.cassandra.config.DatabaseDescriptor; -import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.dht.Token; import org.apache.cassandra.locator.AbstractReplicationStrategy; import org.apache.cassandra.locator.EndpointsForToken; diff --git a/src/java/org/apache/cassandra/db/virtual/StreamingVirtualTable.java b/src/java/org/apache/cassandra/db/virtual/StreamingVirtualTable.java index 1036f18426..f9fe0ef54d 100644 --- a/src/java/org/apache/cassandra/db/virtual/StreamingVirtualTable.java +++ b/src/java/org/apache/cassandra/db/virtual/StreamingVirtualTable.java @@ -23,10 +23,12 @@ import java.util.UUID; import java.util.stream.Collectors; import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.marshal.TimeUUIDType; import org.apache.cassandra.db.marshal.UUIDType; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.streaming.StreamManager; import org.apache.cassandra.streaming.StreamingState; +import org.apache.cassandra.utils.TimeUUID; import static org.apache.cassandra.cql3.statements.schema.CreateTableStatement.parse; @@ -35,7 +37,7 @@ public class StreamingVirtualTable extends AbstractVirtualTable public StreamingVirtualTable(String keyspace) { super(parse("CREATE TABLE streaming (" + - " id uuid,\n" + + " id timeuuid,\n" + " follower boolean,\n" + " operation text, \n" + " peers frozen>,\n" + @@ -76,7 +78,7 @@ public class StreamingVirtualTable extends AbstractVirtualTable @Override public DataSet data(DecoratedKey partitionKey) { - UUID id = UUIDType.instance.compose(partitionKey.getKey()); + TimeUUID id = TimeUUIDType.instance.compose(partitionKey.getKey()); SimpleDataSet result = new SimpleDataSet(metadata()); StreamingState state = StreamManager.instance.getStreamingState(id); if (state != null) diff --git a/src/java/org/apache/cassandra/db/virtual/VirtualMutation.java b/src/java/org/apache/cassandra/db/virtual/VirtualMutation.java index 09ac4a604b..3e26032383 100644 --- a/src/java/org/apache/cassandra/db/virtual/VirtualMutation.java +++ b/src/java/org/apache/cassandra/db/virtual/VirtualMutation.java @@ -19,6 +19,7 @@ package org.apache.cassandra.db.virtual; import java.util.Collection; import java.util.concurrent.TimeUnit; +import java.util.function.Supplier; import com.google.common.base.MoreObjects; import com.google.common.collect.ImmutableMap; @@ -26,6 +27,7 @@ import com.google.common.collect.ImmutableMap; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.db.IMutation; +import org.apache.cassandra.db.Mutation; import org.apache.cassandra.db.partitions.PartitionUpdate; import org.apache.cassandra.schema.TableId; @@ -104,6 +106,12 @@ public final class VirtualMutation implements IMutation return modifications.values(); } + @Override + public Supplier hintOnFailure() + { + return null; + } + @Override public void validateIndexedColumns() { diff --git a/src/java/org/apache/cassandra/dht/Murmur3Partitioner.java b/src/java/org/apache/cassandra/dht/Murmur3Partitioner.java index 2856f131f1..b0f8bf90e3 100644 --- a/src/java/org/apache/cassandra/dht/Murmur3Partitioner.java +++ b/src/java/org/apache/cassandra/dht/Murmur3Partitioner.java @@ -144,7 +144,7 @@ public class Murmur3Partitioner implements IPartitioner { static final long serialVersionUID = -5833580143318243006L; - final long token; + public final long token; public LongToken(long token) { @@ -204,11 +204,16 @@ public class Murmur3Partitioner implements IPartitioner } @Override - public Token increaseSlightly() + public LongToken increaseSlightly() { return new LongToken(token + 1); } + public LongToken decreaseSlightly() + { + return new LongToken(token - 1); + } + /** * Reverses murmur3 to find a possible 16 byte key that generates a given token */ @@ -216,7 +221,7 @@ public class Murmur3Partitioner implements IPartitioner public static ByteBuffer keyForToken(LongToken token) { ByteBuffer result = ByteBuffer.allocate(16); - long[] inv = MurmurHash.inv_hash3_x64_128(new long[] {token.token, 0L}); + long[] inv = MurmurHash.inv_hash3_x64_128(new long[]{ token.token, 0L }); result.putLong(inv[0]).putLong(inv[1]).position(0); return result; } diff --git a/src/java/org/apache/cassandra/dht/Range.java b/src/java/org/apache/cassandra/dht/Range.java index 5b2f3d9fbf..2c468990d2 100644 --- a/src/java/org/apache/cassandra/dht/Range.java +++ b/src/java/org/apache/cassandra/dht/Range.java @@ -483,7 +483,7 @@ public class Range> extends AbstractBounds implemen * Given a list of unwrapped ranges sorted by left position, return an * equivalent list of ranges but with no overlapping ranges. */ - private static > List> deoverlap(List> ranges) + public static > List> deoverlap(List> ranges) { if (ranges.isEmpty()) return ranges; diff --git a/src/java/org/apache/cassandra/exceptions/CasWriteTimeoutException.java b/src/java/org/apache/cassandra/exceptions/CasWriteTimeoutException.java index b1347645f1..32cc014da1 100644 --- a/src/java/org/apache/cassandra/exceptions/CasWriteTimeoutException.java +++ b/src/java/org/apache/cassandra/exceptions/CasWriteTimeoutException.java @@ -20,13 +20,14 @@ package org.apache.cassandra.exceptions; import org.apache.cassandra.db.ConsistencyLevel; import org.apache.cassandra.db.WriteType; + public class CasWriteTimeoutException extends WriteTimeoutException { public final int contentions; public CasWriteTimeoutException(WriteType writeType, ConsistencyLevel consistency, int received, int blockFor, int contentions) { - super(writeType, consistency, received, blockFor, String.format("CAS operation timed out - encountered contentions: %d", contentions)); + super(writeType, consistency, received, blockFor, String.format("CAS operation timed out: received %d of %d required responses after %d contention retries", received, blockFor, contentions)); this.contentions = contentions; } } diff --git a/src/java/org/apache/cassandra/exceptions/RequestFailureException.java b/src/java/org/apache/cassandra/exceptions/RequestFailureException.java index c75ff9c19d..a71534720b 100644 --- a/src/java/org/apache/cassandra/exceptions/RequestFailureException.java +++ b/src/java/org/apache/cassandra/exceptions/RequestFailureException.java @@ -35,7 +35,7 @@ public class RequestFailureException extends RequestExecutionException this(code, buildErrorMessage(received, failureReasonByEndpoint), consistency, received, blockFor, failureReasonByEndpoint); } - protected RequestFailureException(ExceptionCode code, String msg, ConsistencyLevel consistency, int received, int blockFor, Map failureReasonByEndpoint) + public RequestFailureException(ExceptionCode code, String msg, ConsistencyLevel consistency, int received, int blockFor, Map failureReasonByEndpoint) { super(code, buildErrorMessage(msg, failureReasonByEndpoint)); this.consistency = consistency; diff --git a/src/java/org/apache/cassandra/exceptions/RequestFailureReason.java b/src/java/org/apache/cassandra/exceptions/RequestFailureReason.java index f205900bf8..3d3476a139 100644 --- a/src/java/org/apache/cassandra/exceptions/RequestFailureReason.java +++ b/src/java/org/apache/cassandra/exceptions/RequestFailureReason.java @@ -36,8 +36,9 @@ public enum RequestFailureReason READ_TOO_MANY_TOMBSTONES (1), TIMEOUT (2), INCOMPATIBLE_SCHEMA (3), - READ_SIZE (4); - + READ_SIZE (4), + NODE_DOWN (5); + public static final Serializer serializer = new Serializer(); public final int code; diff --git a/src/java/org/apache/cassandra/gms/EndpointState.java b/src/java/org/apache/cassandra/gms/EndpointState.java index 2cc9c0debc..c60a4793bd 100644 --- a/src/java/org/apache/cassandra/gms/EndpointState.java +++ b/src/java/org/apache/cassandra/gms/EndpointState.java @@ -34,6 +34,7 @@ import org.apache.cassandra.io.IVersionedSerializer; import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputPlus; import org.apache.cassandra.utils.CassandraVersion; +import org.apache.cassandra.utils.NullableSerializer; import static org.apache.cassandra.utils.Clock.Global.nanoTime; @@ -41,13 +42,12 @@ import static org.apache.cassandra.utils.Clock.Global.nanoTime; * This abstraction represents both the HeartBeatState and the ApplicationState in an EndpointState * instance. Any state for a given endpoint can be retrieved from this instance. */ - - public class EndpointState { protected static final Logger logger = LoggerFactory.getLogger(EndpointState.class); public final static IVersionedSerializer serializer = new EndpointStateSerializer(); + public final static IVersionedSerializer nullableSerializer = NullableSerializer.wrap(serializer); private volatile HeartBeatState hbState; private final AtomicReference> applicationState; @@ -255,6 +255,20 @@ public class EndpointState { return "EndpointState: HeartBeatState = " + hbState + ", AppStateMap = " + applicationState.get(); } + + public boolean isSupersededBy(EndpointState that) + { + int thisGeneration = this.getHeartBeatState().getGeneration(); + int thatGeneration = that.getHeartBeatState().getGeneration(); + + if (thatGeneration > thisGeneration) + return true; + + if (thisGeneration > thatGeneration) + return false; + + return Gossiper.getMaxEndpointStateVersion(that) > Gossiper.getMaxEndpointStateVersion(this); + } } class EndpointStateSerializer implements IVersionedSerializer diff --git a/src/java/org/apache/cassandra/gms/GossipDigestAck2.java b/src/java/org/apache/cassandra/gms/GossipDigestAck2.java index 0e4062bb0f..ed959a54e5 100644 --- a/src/java/org/apache/cassandra/gms/GossipDigestAck2.java +++ b/src/java/org/apache/cassandra/gms/GossipDigestAck2.java @@ -43,11 +43,6 @@ public class GossipDigestAck2 { this.epStateMap = epStateMap; } - - Map getEndpointStateMap() - { - return epStateMap; - } } class GossipDigestAck2Serializer implements IVersionedSerializer diff --git a/src/java/org/apache/cassandra/gms/GossipDigestAck2VerbHandler.java b/src/java/org/apache/cassandra/gms/GossipDigestAck2VerbHandler.java index 58c1589eca..0f0199957f 100644 --- a/src/java/org/apache/cassandra/gms/GossipDigestAck2VerbHandler.java +++ b/src/java/org/apache/cassandra/gms/GossipDigestAck2VerbHandler.java @@ -44,7 +44,7 @@ public class GossipDigestAck2VerbHandler extends GossipVerbHandler remoteEpStateMap = message.payload.getEndpointStateMap(); + Map remoteEpStateMap = message.payload.epStateMap; /* Notify the Failure Detector */ Gossiper.instance.notifyFailureDetector(remoteEpStateMap); Gossiper.instance.applyStateLocally(remoteEpStateMap); diff --git a/src/java/org/apache/cassandra/gms/Gossiper.java b/src/java/org/apache/cassandra/gms/Gossiper.java index 4c68cfe8c6..7ee36a4083 100644 --- a/src/java/org/apache/cassandra/gms/Gossiper.java +++ b/src/java/org/apache/cassandra/gms/Gossiper.java @@ -610,7 +610,7 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean * @param epState * @return */ - int getMaxEndpointStateVersion(EndpointState epState) + static int getMaxEndpointStateVersion(EndpointState epState) { int maxVersion = epState.getHeartBeatState().getHeartBeatVersion(); for (Map.Entry state : epState.states()) @@ -1144,6 +1144,15 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean return ImmutableSet.copyOf(endpointStateMap.keySet()); } + public String getForEndpoint(InetAddressAndPort ep, ApplicationState state) + { + EndpointState epState = endpointStateMap.get(ep); + if (epState == null) + return null; + VersionedValue value = epState.getApplicationState(state); + return value == null ? null : value.value; + } + public int getEndpointCount() { return endpointStateMap.size(); @@ -1261,7 +1270,7 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean return ep1.getHeartBeatState().getGeneration() - ep2.getHeartBeatState().getGeneration(); } - void notifyFailureDetector(Map remoteEpStateMap) + public void notifyFailureDetector(Map remoteEpStateMap) { for (Entry entry : remoteEpStateMap.entrySet()) { @@ -2143,7 +2152,7 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean } } - protected boolean isInShadowRound() + public boolean isInShadowRound() { return inShadowRound; } diff --git a/src/java/org/apache/cassandra/gms/VersionedValue.java b/src/java/org/apache/cassandra/gms/VersionedValue.java index 880cb98e06..63e9487d7c 100644 --- a/src/java/org/apache/cassandra/gms/VersionedValue.java +++ b/src/java/org/apache/cassandra/gms/VersionedValue.java @@ -190,6 +190,14 @@ public class VersionedValue implements Comparable Long.toString(expireTime))); } + @VisibleForTesting + public VersionedValue left(Collection tokens, long expireTime, int generation) + { + return new VersionedValue(versionString(VersionedValue.STATUS_LEFT, + makeTokenString(tokens), + Long.toString(expireTime)), generation); + } + public VersionedValue moving(Token token) { return new VersionedValue(VersionedValue.STATUS_MOVING + VersionedValue.DELIMITER + partitioner.getTokenFactory().toString(token)); diff --git a/src/java/org/apache/cassandra/hadoop/cql3/CqlInputFormat.java b/src/java/org/apache/cassandra/hadoop/cql3/CqlInputFormat.java index 584a80c761..07556846b9 100644 --- a/src/java/org/apache/cassandra/hadoop/cql3/CqlInputFormat.java +++ b/src/java/org/apache/cassandra/hadoop/cql3/CqlInputFormat.java @@ -54,8 +54,8 @@ import org.apache.cassandra.dht.*; import org.apache.cassandra.hadoop.*; import org.apache.cassandra.utils.*; -import static org.apache.cassandra.utils.Clock.Global.nanoTime; import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory; +import static org.apache.cassandra.utils.Clock.Global.nanoTime; /** * Hadoop InputFormat allowing map/reduce against Cassandra rows within one ColumnFamily. diff --git a/src/java/org/apache/cassandra/index/Index.java b/src/java/org/apache/cassandra/index/Index.java index e9d3d3c3d3..63f619848e 100644 --- a/src/java/org/apache/cassandra/index/Index.java +++ b/src/java/org/apache/cassandra/index/Index.java @@ -267,7 +267,27 @@ public interface Index public Optional getBackingTable(); /** - * Return a task which performs a blocking flush of the index's data to persistent storage. + * Return a task which performs a blocking flush of the index's data corresponding to the provided + * base table's Memtable. This may extract any necessary data from the base table's Memtable as part of the flush. + * + * This version of the method is invoked whenever we flush the base table. If the index stores no in-memory data + * of its own, it is safe to only implement this method. + * + * @return task to be executed by the index manager to perform the flush. + */ + public default Callable getBlockingFlushTask(Memtable baseCfs) + { + return getBlockingFlushTask(); + } + + /** + * Return a task which performs a blocking flush of any in-memory index data to persistent storage, + * independent of any flush of the base table. + * + * Note that this method is only invoked outside of normal flushes: if there is no in-memory storage + * for this index, and it only extracts data on flush from the base table's Memtable, then it is safe to + * perform no work. + * * @return task to be executed by the index manager to perform the flush. */ public Callable getBlockingFlushTask(); diff --git a/src/java/org/apache/cassandra/index/SecondaryIndexManager.java b/src/java/org/apache/cassandra/index/SecondaryIndexManager.java index d75b488f5f..c296880586 100644 --- a/src/java/org/apache/cassandra/index/SecondaryIndexManager.java +++ b/src/java/org/apache/cassandra/index/SecondaryIndexManager.java @@ -38,7 +38,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.cassandra.concurrent.ExecutorPlus; +import org.apache.cassandra.concurrent.FutureTask; import org.apache.cassandra.concurrent.ImmediateExecutor; +import org.apache.cassandra.concurrent.ScheduledExecutors; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.cql3.statements.schema.IndexTarget; import org.apache.cassandra.db.*; @@ -63,6 +65,8 @@ import org.apache.cassandra.notifications.SSTableAddedNotification; import org.apache.cassandra.schema.ColumnMetadata; import org.apache.cassandra.schema.IndexMetadata; import org.apache.cassandra.schema.Indexes; +import org.apache.cassandra.schema.Schema; +import org.apache.cassandra.schema.SchemaConstants; import org.apache.cassandra.service.pager.SinglePartitionPager; import org.apache.cassandra.tracing.Tracing; import org.apache.cassandra.transport.ProtocolVersion; @@ -209,13 +213,15 @@ public class SecondaryIndexManager implements IndexRegistry, INotificationConsum markIndexesBuilding(ImmutableSet.of(index), true, isNewCF); - Callable initialBuildTask = null; + FutureTask initialBuildTask = null; // if the index didn't register itself, we can probably assume that no initialization needs to happen if (indexes.containsKey(indexDef.name)) { try { - initialBuildTask = index.getInitializationTask(); + Callable call = index.getInitializationTask(); + if (call != null) + initialBuildTask = new FutureTask<>(call); } catch (Throwable t) { @@ -233,16 +239,20 @@ public class SecondaryIndexManager implements IndexRegistry, INotificationConsum // otherwise run the initialization task asynchronously with a callback to mark it built or failed final Promise initialization = new AsyncPromise<>(); - asyncExecutor.submit(initialBuildTask) - .addCallback( - success -> { - markIndexBuilt(index, true); - initialization.trySuccess(null); - }, - failure -> { - logAndMarkIndexesFailed(Collections.singleton(index), failure, true); - initialization.tryFailure(failure); - }); + // we want to ensure we invoke this task asynchronously, so we want to add our callback before submission + // to ensure the work is not completed before we register the callback and so it gets performed by us. + // This is because Keyspace.open("system") can transitively attempt to open Keyspace.open("system") + initialBuildTask.addCallback( + success -> { + markIndexBuilt(index, true); + initialization.trySuccess(null); + }, + failure -> { + logAndMarkIndexesFailed(Collections.singleton(index), failure, true); + initialization.tryFailure(failure); + } + ); + asyncExecutor.execute(initialBuildTask); return initialization; } @@ -667,7 +677,7 @@ public class SecondaryIndexManager implements IndexRegistry, INotificationConsum if (counter.decrementAndGet() == 0) { inProgressBuilds.remove(indexName); - if (!needsFullRebuild.contains(indexName) && DatabaseDescriptor.isDaemonInitialized()) + if (!needsFullRebuild.contains(indexName) && DatabaseDescriptor.isDaemonInitialized() && Keyspace.isInitialized()) SystemKeyspace.setIndexBuilt(baseCfs.keyspace.getName(), indexName); } } @@ -798,17 +808,6 @@ public class SecondaryIndexManager implements IndexRegistry, INotificationConsum flushIndexesBlocking(indexes, null); } - /** - * Performs a blocking flush of all custom indexes - */ - public void flushAllNonCFSBackedIndexesBlocking() - { - executeAllBlocking(indexes.values() - .stream() - .filter(index -> !index.getBackingTable().isPresent()), - Index::getBlockingFlushTask, null); - } - /** * Performs a blocking execution of pre-join tasks of all indexes */ @@ -843,6 +842,18 @@ public class SecondaryIndexManager implements IndexRegistry, INotificationConsum FBUtilities.waitOnFutures(wait); } + /** + * Performs a blocking flush of all custom indexes + */ + public void flushAllNonCFSBackedIndexesBlocking(Memtable baseCfsMemtable) + { + executeAllBlocking(indexes.values() + .stream() + .filter(index -> !index.getBackingTable().isPresent()), + index -> index.getBlockingFlushTask(baseCfsMemtable), + null); + } + /** * @return all indexes which are marked as built and ready to use */ diff --git a/src/java/org/apache/cassandra/io/FSDiskFullWriteError.java b/src/java/org/apache/cassandra/io/FSDiskFullWriteError.java index ebb07e2c69..abf0b6ca69 100644 --- a/src/java/org/apache/cassandra/io/FSDiskFullWriteError.java +++ b/src/java/org/apache/cassandra/io/FSDiskFullWriteError.java @@ -28,10 +28,4 @@ public class FSDiskFullWriteError extends FSWriteError mutationSize, keyspace))); } - - @Override - public String toString() - { - return "FSDiskFullWriteError"; - } } diff --git a/src/java/org/apache/cassandra/io/FSError.java b/src/java/org/apache/cassandra/io/FSError.java index 65313947a1..4c06d9c61f 100644 --- a/src/java/org/apache/cassandra/io/FSError.java +++ b/src/java/org/apache/cassandra/io/FSError.java @@ -24,17 +24,30 @@ import org.apache.cassandra.io.util.File; public abstract class FSError extends IOError { + final String message; public final String path; public FSError(Throwable cause, File path) { - super(cause); - this.path = path.toString(); + this(null, cause, path); } public FSError(Throwable cause, Path path) + { + this(null, cause, path); + } + + public FSError(String message, Throwable cause, File path) { super(cause); + this.message = message; + this.path = path.toString(); + } + + public FSError(String message, Throwable cause, Path path) + { + super(cause); + this.message = message; this.path = path.toString(); } @@ -53,4 +66,10 @@ public abstract class FSError extends IOError return null; } + + @Override + public String toString() + { + return getClass().getSimpleName() + (message != null ? ' ' + message : "") + (path != null ? " in " + path : ""); + } } diff --git a/src/java/org/apache/cassandra/io/FSNoDiskAvailableForWriteError.java b/src/java/org/apache/cassandra/io/FSNoDiskAvailableForWriteError.java index 14dcd38f2a..415f204b11 100644 --- a/src/java/org/apache/cassandra/io/FSNoDiskAvailableForWriteError.java +++ b/src/java/org/apache/cassandra/io/FSNoDiskAvailableForWriteError.java @@ -30,10 +30,4 @@ public class FSNoDiskAvailableForWriteError extends FSWriteError super(new IOException(String.format("The data directories for the %s keyspace have been marked as unwritable", keyspace))); } - - @Override - public String toString() - { - return "FSNoDiskAvailableForWriteError"; - } } diff --git a/src/java/org/apache/cassandra/io/FSReadError.java b/src/java/org/apache/cassandra/io/FSReadError.java index 688182a324..ac15534770 100644 --- a/src/java/org/apache/cassandra/io/FSReadError.java +++ b/src/java/org/apache/cassandra/io/FSReadError.java @@ -39,9 +39,23 @@ public class FSReadError extends FSError this(cause, new File(path)); } - @Override - public String toString() + public FSReadError(String message, Throwable cause, Path path) { - return "FSReadError in " + path; + super(message, cause, path); + } + + public FSReadError(String message, Throwable cause, File path) + { + super(message, cause, path); + } + + public FSReadError(String message, Throwable cause, String path) + { + this(message, cause, new File(path)); + } + + public FSReadError(String message, Throwable cause) + { + this(message, cause, new File("")); } } diff --git a/src/java/org/apache/cassandra/io/FSWriteError.java b/src/java/org/apache/cassandra/io/FSWriteError.java index 481795db0e..81f00bd3d8 100644 --- a/src/java/org/apache/cassandra/io/FSWriteError.java +++ b/src/java/org/apache/cassandra/io/FSWriteError.java @@ -44,9 +44,23 @@ public class FSWriteError extends FSError this(cause, new File("")); } - @Override - public String toString() + public FSWriteError(String message, Throwable cause, Path path) { - return "FSWriteError in " + path; + super(message, cause, path); + } + + public FSWriteError(String message, Throwable cause, File path) + { + super(message, cause, path); + } + + public FSWriteError(String message, Throwable cause, String path) + { + this(message, cause, new File(path)); + } + + public FSWriteError(String message, Throwable cause) + { + this(message, cause, new File("")); } } diff --git a/src/java/org/apache/cassandra/io/sstable/CQLSSTableWriter.java b/src/java/org/apache/cassandra/io/sstable/CQLSSTableWriter.java index 6224cb7bfb..eab47c4289 100644 --- a/src/java/org/apache/cassandra/io/sstable/CQLSSTableWriter.java +++ b/src/java/org/apache/cassandra/io/sstable/CQLSSTableWriter.java @@ -251,7 +251,7 @@ public class CQLSSTableWriter implements Closeable ClientState.forInternalCalls(), options, insert.getTimestamp(TimeUnit.MILLISECONDS.toMicros(now), options), - (int) TimeUnit.MILLISECONDS.toSeconds(now), + options.getNowInSec((int) TimeUnit.MILLISECONDS.toSeconds(now)), insert.getTimeToLive(options), Collections.emptyMap()); diff --git a/src/java/org/apache/cassandra/io/sstable/IndexSummaryManager.java b/src/java/org/apache/cassandra/io/sstable/IndexSummaryManager.java index 4e3ae30d35..a95cbffeac 100644 --- a/src/java/org/apache/cassandra/io/sstable/IndexSummaryManager.java +++ b/src/java/org/apache/cassandra/io/sstable/IndexSummaryManager.java @@ -281,6 +281,11 @@ public class IndexSummaryManager implements IndexSummaryManagerMBean @VisibleForTesting public void shutdownAndWait(long timeout, TimeUnit unit) throws InterruptedException, TimeoutException { - ExecutorUtils.shutdownNowAndWait(timeout, unit, executor); + if (future != null) + { + future.cancel(false); + future = null; + } + ExecutorUtils.shutdownAndWait(timeout, unit, executor); } } diff --git a/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java b/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java index 8fb5ad5f0f..c5f07bc39e 100644 --- a/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java +++ b/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java @@ -34,7 +34,6 @@ import com.google.common.primitives.Longs; import com.google.common.util.concurrent.RateLimiter; import org.apache.cassandra.config.CassandraRelevantProperties; -import org.apache.cassandra.io.util.File; import org.apache.cassandra.concurrent.ExecutorPlus; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java b/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java index e25b0a9740..cff1f1c2d5 100644 --- a/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java +++ b/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java @@ -18,7 +18,6 @@ package org.apache.cassandra.io.sstable.format.big; import java.util.Collection; -import java.util.UUID; import org.apache.cassandra.io.sstable.SSTable; import org.apache.cassandra.schema.TableMetadata; diff --git a/src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java b/src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java index 0e20f285eb..e8dff32fbc 100644 --- a/src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java +++ b/src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java @@ -638,7 +638,7 @@ public class BigTableWriter extends SSTableWriter protected Throwable doAbort(Throwable accumulate) { - return indexFile.abort(accumulate); + return summary.close(indexFile.abort(accumulate)); } @Override diff --git a/src/java/org/apache/cassandra/io/sstable/metadata/StatsMetadata.java b/src/java/org/apache/cassandra/io/sstable/metadata/StatsMetadata.java index 9efcfa6ab2..ee5505f6fb 100644 --- a/src/java/org/apache/cassandra/io/sstable/metadata/StatsMetadata.java +++ b/src/java/org/apache/cassandra/io/sstable/metadata/StatsMetadata.java @@ -36,7 +36,6 @@ import org.apache.cassandra.io.ISerializer; import org.apache.cassandra.io.sstable.format.Version; import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputPlus; -import org.apache.cassandra.net.MessagingService; import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.EstimatedHistogram; import org.apache.cassandra.utils.TimeUUID; diff --git a/src/java/org/apache/cassandra/io/util/ChecksummedSequentialWriter.java b/src/java/org/apache/cassandra/io/util/ChecksummedSequentialWriter.java index 1477c5c08d..fa8fad7d57 100644 --- a/src/java/org/apache/cassandra/io/util/ChecksummedSequentialWriter.java +++ b/src/java/org/apache/cassandra/io/util/ChecksummedSequentialWriter.java @@ -20,8 +20,6 @@ package org.apache.cassandra.io.util; import java.nio.ByteBuffer; import java.util.Optional; -import org.apache.cassandra.io.util.File; - public class ChecksummedSequentialWriter extends SequentialWriter { private static final SequentialWriterOption CRC_WRITER_OPTION = SequentialWriterOption.newBuilder() diff --git a/src/java/org/apache/cassandra/io/util/File.java b/src/java/org/apache/cassandra/io/util/File.java index 8d4a0307a3..23113b164b 100644 --- a/src/java/org/apache/cassandra/io/util/File.java +++ b/src/java/org/apache/cassandra/io/util/File.java @@ -18,7 +18,9 @@ package org.apache.cassandra.io.util; +import java.io.FileNotFoundException; import java.io.IOException; +import java.io.UncheckedIOException; import java.net.URI; import java.nio.channels.FileChannel; import java.nio.file.*; // checkstyle: permit this import @@ -34,6 +36,7 @@ import javax.annotation.Nullable; import com.google.common.util.concurrent.RateLimiter; +import net.openhft.chronicle.core.util.ThrowingFunction; import org.apache.cassandra.io.FSWriteError; import static org.apache.cassandra.io.util.PathUtils.filename; @@ -164,6 +167,16 @@ public class File implements Comparable maybeFail(delete(null, null)); } + /** + * This file will be deleted, with any failures being reported with an FSError + * @throws FSWriteError if cannot be deleted + */ + public void deleteIfExists() + { + if (path != null) + PathUtils.deleteIfExists(path); + } + /** * This file will be deleted, obeying the provided rate limiter. * @throws FSWriteError if cannot be deleted @@ -336,6 +349,11 @@ public class File implements Comparable return PathUtils.createFileIfNotExists(toPathForWrite()); } + public boolean createDirectoriesIfNotExists() + { + return PathUtils.createDirectoriesIfNotExists(toPathForWrite()); + } + /** * Try to create a directory at this path. * Return true if a new directory was created at this path, and false otherwise. @@ -436,12 +454,29 @@ public class File implements Comparable PathUtils.forEachRecursive(path, path -> forEach.accept(new File(path))); } + private static ThrowingFunction nulls() { return ignore -> null; } + private static ThrowingFunction rethrow() + { + return fail -> { + if (fail == null) throw new FileNotFoundException(); + throw fail; + }; + } + private static ThrowingFunction unchecked() + { + return fail -> { + if (fail == null) fail = new FileNotFoundException(); + throw new UncheckedIOException(fail); + }; + } + + /** * @return if a directory, the names of the files within; null otherwise */ public String[] tryListNames() { - return tryListNames(path, Function.identity()); + return tryListNames(nulls()); } /** @@ -449,7 +484,7 @@ public class File implements Comparable */ public String[] tryListNames(BiPredicate filter) { - return tryList(path, stream -> stream.map(PathUtils::filename).filter(filename -> filter.test(this, filename)), String[]::new); + return tryListNames(filter, nulls()); } /** @@ -457,7 +492,7 @@ public class File implements Comparable */ public File[] tryList() { - return tryList(path, Function.identity()); + return tryList(nulls()); } /** @@ -465,7 +500,7 @@ public class File implements Comparable */ public File[] tryList(Predicate filter) { - return tryList(path, stream -> stream.filter(filter)); + return tryList(filter, nulls()); } /** @@ -473,28 +508,148 @@ public class File implements Comparable */ public File[] tryList(BiPredicate filter) { - return tryList(path, stream -> stream.filter(file -> filter.test(this, file.name()))); + return tryList(filter, nulls()); } - private static String[] tryListNames(Path path, Function, Stream> toFiles) + /** + * @return if a directory, the names of the files within; null otherwise + */ + public String[] listNames() throws IOException { - if (path == null) - return null; - return PathUtils.tryList(path, stream -> toFiles.apply(stream.map(File::new)).map(File::name), String[]::new); + return tryListNames(rethrow()); } - private static T[] tryList(Path path, Function, Stream> transformation, IntFunction constructor) + /** + * @return if a directory, the names of the files within, filtered by the provided predicate; null otherwise + */ + public String[] listNames(BiPredicate filter) throws IOException { - if (path == null) - return null; - return PathUtils.tryList(path, transformation, constructor); + return tryListNames(filter, rethrow()); } - private static File[] tryList(Path path, Function, Stream> toFiles) + /** + * @return if a directory, the files within; null otherwise + */ + public File[] list() throws IOException + { + return tryList(rethrow()); + } + + /** + * @return if a directory, the files within, filtered by the provided predicate; null otherwise + */ + public File[] list(Predicate filter) throws IOException + { + return tryList(filter, rethrow()); + } + + /** + * @return if a directory, the files within, filtered by the provided predicate; null otherwise + */ + public File[] list(BiPredicate filter) throws IOException + { + return tryList(filter, rethrow()); + } + + /** + * @return if a directory, the names of the files within; null otherwise + */ + public String[] listNamesUnchecked() throws UncheckedIOException + { + return tryListNames(unchecked()); + } + + /** + * @return if a directory, the names of the files within, filtered by the provided predicate; null otherwise + */ + public String[] listNamesUnchecked(BiPredicate filter) throws UncheckedIOException + { + return tryListNames(filter, unchecked()); + } + + /** + * @return if a directory, the files within; null otherwise + */ + public File[] listUnchecked() throws UncheckedIOException + { + return tryList(unchecked()); + } + + /** + * @return if a directory, the files within, filtered by the provided predicate; null otherwise + */ + public File[] listUnchecked(Predicate filter) throws UncheckedIOException + { + return tryList(filter, unchecked()); + } + + /** + * @return if a directory, the files within, filtered by the provided predicate; throw an UncheckedIO exception otherwise + */ + public File[] listUnchecked(BiPredicate filter) throws UncheckedIOException + { + return tryList(filter, unchecked()); + } + + /** + * @return if a directory, the names of the files within; null otherwise + */ + public String[] tryListNames(ThrowingFunction orElse) throws T + { + return tryListNames(path, Function.identity(), orElse); + } + + /** + * @return if a directory, the names of the files within, filtered by the provided predicate; null otherwise + */ + public String[] tryListNames(BiPredicate filter, ThrowingFunction orElse) throws T + { + return tryList(path, stream -> stream.map(PathUtils::filename).filter(filename -> filter.test(this, filename)), String[]::new, orElse); + } + + /** + * @return if a directory, the files within; null otherwise + */ + private File[] tryList(ThrowingFunction orElse) throws T + { + return tryList(path, Function.identity(), orElse); + } + + /** + * @return if a directory, the files within, filtered by the provided predicate; null otherwise + */ + private File[] tryList(Predicate filter, ThrowingFunction orElse) throws T + { + return tryList(path, stream -> stream.filter(filter), orElse); + } + + /** + * @return if a directory, the files within, filtered by the provided predicate; null otherwise + */ + private File[] tryList(BiPredicate filter, ThrowingFunction orElse) throws T + { + return tryList(path, stream -> stream.filter(file -> filter.test(this, file.name())), orElse); + } + + private static String[] tryListNames(Path path, Function, Stream> toFiles, ThrowingFunction orElse) throws T { if (path == null) - return null; - return PathUtils.tryList(path, stream -> toFiles.apply(stream.map(File::new)), File[]::new); + return orElse.apply(null); + return PathUtils.tryList(path, stream -> toFiles.apply(stream.map(File::new)).map(File::name), String[]::new, orElse); + } + + private static V[] tryList(Path path, Function, Stream> transformation, IntFunction constructor, ThrowingFunction orElse) throws T + { + if (path == null) + return orElse.apply(null); + return PathUtils.tryList(path, transformation, constructor, orElse); + } + + private static File[] tryList(Path path, Function, Stream> toFiles, ThrowingFunction orElse) throws T + { + if (path == null) + return orElse.apply(null); + return PathUtils.tryList(path, stream -> toFiles.apply(stream.map(File::new)), File[]::new, orElse); } /** diff --git a/src/java/org/apache/cassandra/io/util/PathUtils.java b/src/java/org/apache/cassandra/io/util/PathUtils.java index c5a1e96bdc..2e321c7e31 100644 --- a/src/java/org/apache/cassandra/io/util/PathUtils.java +++ b/src/java/org/apache/cassandra/io/util/PathUtils.java @@ -34,6 +34,7 @@ import com.google.common.util.concurrent.RateLimiter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import net.openhft.chronicle.core.util.ThrowingFunction; import org.apache.cassandra.config.CassandraRelevantProperties; import org.apache.cassandra.io.FSError; import org.apache.cassandra.io.FSReadError; @@ -132,7 +133,7 @@ public final class PathUtils } } - public static T[] tryList(Path path, Function, Stream> transform, IntFunction arrayFactory) + public static V[] tryList(Path path, Function, Stream> transform, IntFunction arrayFactory, ThrowingFunction orElse) throws T { try (Stream stream = Files.list(path)) { @@ -141,7 +142,7 @@ public final class PathUtils } catch (IOException e) { - return null; + return orElse.apply(e); } } @@ -255,6 +256,22 @@ public final class PathUtils } } + public static void deleteIfExists(Path file) + { + try + { + Files.delete(file); + onDeletion.accept(file); + } + catch (IOException e) + { + if (e instanceof FileNotFoundException | e instanceof NoSuchFileException) + return; + + throw propagateUnchecked(e, file, true); + } + } + public static boolean tryDelete(Path file) { try @@ -375,9 +392,8 @@ public final class PathUtils { logger.trace("Could not move file {} to {}", from, to, e); - // TODO: this should be an FSError (either read or write)? - // (but for now this is maintaining legacy semantics) - throw new RuntimeException(String.format("Failed to rename %s to %s", from, to), e); + // TODO: try to decide if is read or write? for now, have assumed write + throw propagateUnchecked(String.format("Failed to rename %s to %s", from, to), e, to, true); } } @@ -733,6 +749,14 @@ public final class PathUtils * propagate an IOException as an FSWriteError, FSReadError or UncheckedIOException */ public static RuntimeException propagateUnchecked(IOException ioe, Path path, boolean write) + { + return propagateUnchecked(null, ioe, path, write); + } + + /** + * propagate an IOException as an FSWriteError, FSReadError or UncheckedIOException + */ + public static RuntimeException propagateUnchecked(String message, IOException ioe, Path path, boolean write) { if (ioe instanceof FileAlreadyExistsException || ioe instanceof NoSuchFileException @@ -741,10 +765,10 @@ public final class PathUtils || ioe instanceof java.nio.file.FileSystemLoopException || ioe instanceof java.nio.file.NotDirectoryException || ioe instanceof java.nio.file.NotLinkException) - throw new UncheckedIOException(ioe); + throw new UncheckedIOException(message, ioe); - if (write) throw new FSWriteError(ioe, path); - else throw new FSReadError(ioe, path); + if (write) throw new FSWriteError(message, ioe, path); + else throw new FSReadError(message, ioe, path); } /** diff --git a/src/java/org/apache/cassandra/io/util/SequentialWriterOption.java b/src/java/org/apache/cassandra/io/util/SequentialWriterOption.java index 599c654421..49093c1bd3 100644 --- a/src/java/org/apache/cassandra/io/util/SequentialWriterOption.java +++ b/src/java/org/apache/cassandra/io/util/SequentialWriterOption.java @@ -40,6 +40,7 @@ public class SequentialWriterOption * */ public static final SequentialWriterOption DEFAULT = SequentialWriterOption.newBuilder().build(); + public static final SequentialWriterOption FINISH_ON_CLOSE = SequentialWriterOption.newBuilder().finishOnClose(true).build(); private final int bufferSize; private final BufferType bufferType; diff --git a/src/java/org/apache/cassandra/locator/AbstractReplicaCollection.java b/src/java/org/apache/cassandra/locator/AbstractReplicaCollection.java index 16db4925ed..cc210622d1 100644 --- a/src/java/org/apache/cassandra/locator/AbstractReplicaCollection.java +++ b/src/java/org/apache/cassandra/locator/AbstractReplicaCollection.java @@ -32,9 +32,12 @@ import java.util.Comparator; import java.util.Iterator; import java.util.List; import java.util.Objects; +import java.util.RandomAccess; import java.util.Set; +import java.util.Spliterator; import java.util.function.BiConsumer; import java.util.function.BinaryOperator; +import java.util.function.Consumer; import java.util.function.Function; import java.util.function.Predicate; import java.util.function.Supplier; @@ -45,6 +48,9 @@ import java.util.stream.Stream; * A collection like class for Replica objects. Since the Replica class contains inetaddress, range, and * transient replication status, basic contains and remove methods can be ambiguous. Replicas forces you * to be explicit about what you're checking the container for, or removing from it. + * + * TODO: there's nothing about this collection that's unique to Replicas, and the implementation + * could make a useful general purpose immutable list<->set */ public abstract class AbstractReplicaCollection> implements ReplicaCollection { @@ -69,8 +75,10 @@ public abstract class AbstractReplicaCollection + protected final static class ReplicaList implements Iterable { private static final Replica[] EMPTY = new Replica[0]; Replica[] contents; @@ -125,7 +133,7 @@ public abstract class AbstractReplicaCollection comparator) + public ReplicaList sorted(Comparator comparator) { Replica[] copy = Arrays.copyOfRange(contents, begin, begin + size); Arrays.sort(copy, comparator); @@ -137,6 +145,37 @@ public abstract class AbstractReplicaCollection forEach) + { + for (int i = begin, end = begin + size ; i < end ; ++i) + forEach.accept(contents[i]); + } + + /** see {@link ReplicaCollection#count(Predicate)}*/ + public int count(Predicate test) + { + int count = 0; + for (int i = begin, end = i + size ; i < end ; ++i) + if (test.test(contents[i])) + ++count; + return count; + } + + public final boolean anyMatch(Predicate predicate) + { + for (int i = begin, end = i + size ; i < end ; ++i) + if (predicate.test(contents[i])) + return true; + return false; + } + + @Override + public Spliterator spliterator() + { + return Arrays.spliterator(contents, begin, begin + size); + } + // we implement our own iterator, because it is trivial to do so, and in monomorphic call sites // will compile down to almost optimal indexed for loop @Override @@ -163,7 +202,7 @@ public abstract class AbstractReplicaCollection Iterator transformIterator(Function function) + public Iterator transformIterator(Function function) { return new Iterator() { @@ -186,7 +225,7 @@ public abstract class AbstractReplicaCollection filterIterator(Predicate predicate, int limit) + private Iterator filterIterator(Predicate predicate, int limit) { return new Iterator() { @@ -218,6 +257,12 @@ public abstract class AbstractReplicaCollection void forEach(Function function, Consumer action) + { + for (int i = begin, end = begin + size ; i < end ; ++i) + action.accept(function.apply(contents[i])); + } + @VisibleForTesting public boolean equals(Object to) { @@ -263,6 +308,12 @@ public abstract class AbstractReplicaCollection iterator() { return list.transformIterator(toKey); } + + @Override + public void forEach(Consumer action) + { + list.forEach(toKey, action); + } } class EntrySet extends AbstractImmutableSet> @@ -318,7 +369,7 @@ public abstract class AbstractReplicaCollection extends AbstractList implements RandomAccess + { + final Function view; + final ReplicaList list; + + AsList(Function view, ReplicaList list) + { + this.view = view; + this.list = list; + } + + public final T get(int index) + { + return view.apply(list.get(index)); + } + + public final int size() + { + return list.size; + } + + @Override + public final void forEach(Consumer forEach) + { + list.forEach(view, forEach); + } + } + + protected final ReplicaList list; AbstractReplicaCollection(ReplicaList list) { @@ -398,40 +478,30 @@ public abstract class AbstractReplicaCollection List asList(Function viewTransform) + public final List asList(Function view) { - return new AbstractList() - { - public T get(int index) - { - return viewTransform.apply(list.get(index)); - } - - public int size() - { - return list.size; - } - }; + return new AsList<>(view, list); } /** see {@link ReplicaCollection#count(Predicate)}*/ - public int count(Predicate predicate) + public final int count(Predicate test) { - int count = 0; - for (int i = 0 ; i < list.size() ; ++i) - if (predicate.test(list.get(i))) - ++count; - return count; + return list.count(test); + } + + public final boolean anyMatch(Predicate test) + { + return list.anyMatch(test); } /** see {@link ReplicaCollection#filter(Predicate)}*/ - public final C filter(Predicate predicate) + public final C filter(Predicate predicate) { return filter(predicate, Integer.MAX_VALUE); } /** see {@link ReplicaCollection#filter(Predicate, int)}*/ - public final C filter(Predicate predicate, int limit) + public final C filter(Predicate predicate, int limit) { if (isEmpty()) return snapshot(); @@ -475,19 +545,19 @@ public abstract class AbstractReplicaCollection filterLazily(Predicate predicate) + public final Iterable filterLazily(Predicate predicate) { return filterLazily(predicate, Integer.MAX_VALUE); } /** see {@link ReplicaCollection#filterLazily(Predicate,int)}*/ - public final Iterable filterLazily(Predicate predicate, int limit) + public final Iterable filterLazily(Predicate predicate, int limit) { return () -> list.filterIterator(predicate, limit); } /** see {@link ReplicaCollection#sorted(Comparator)}*/ - public final C sorted(Comparator comparator) + public final C sorted(Comparator comparator) { return snapshot(list.sorted(comparator)); } @@ -512,6 +582,11 @@ public abstract class AbstractReplicaCollection forEach) + { + list.forEach(forEach); + } + public final Stream stream() { return list.stream(); } /** diff --git a/src/java/org/apache/cassandra/locator/AbstractReplicationStrategy.java b/src/java/org/apache/cassandra/locator/AbstractReplicationStrategy.java index 0b7c378232..7f33c578ae 100644 --- a/src/java/org/apache/cassandra/locator/AbstractReplicationStrategy.java +++ b/src/java/org/apache/cassandra/locator/AbstractReplicationStrategy.java @@ -24,6 +24,7 @@ import java.util.Collection; import java.util.Collections; import java.util.Map; import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Supplier; import com.google.common.base.Preconditions; @@ -32,6 +33,7 @@ import org.slf4j.LoggerFactory; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.ConsistencyLevel; +import org.apache.cassandra.db.Mutation; import org.apache.cassandra.db.WriteType; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.RingPosition; @@ -128,17 +130,20 @@ public abstract class AbstractReplicationStrategy */ public abstract EndpointsForRange calculateNaturalReplicas(Token searchToken, TokenMetadata tokenMetadata); - public AbstractWriteResponseHandler getWriteResponseHandler(ReplicaPlan.ForTokenWrite replicaPlan, + public AbstractWriteResponseHandler getWriteResponseHandler(ReplicaPlan.ForWrite replicaPlan, Runnable callback, WriteType writeType, + Supplier hintOnFailure, long queryStartNanoTime) { - return getWriteResponseHandler(replicaPlan, callback, writeType, queryStartNanoTime, DatabaseDescriptor.getIdealConsistencyLevel()); + return getWriteResponseHandler(replicaPlan, callback, writeType, hintOnFailure, + queryStartNanoTime, DatabaseDescriptor.getIdealConsistencyLevel()); } - public AbstractWriteResponseHandler getWriteResponseHandler(ReplicaPlan.ForTokenWrite replicaPlan, + public AbstractWriteResponseHandler getWriteResponseHandler(ReplicaPlan.ForWrite replicaPlan, Runnable callback, WriteType writeType, + Supplier hintOnFailure, long queryStartNanoTime, ConsistencyLevel idealConsistencyLevel) { @@ -146,15 +151,15 @@ public abstract class AbstractReplicationStrategy if (replicaPlan.consistencyLevel().isDatacenterLocal()) { // block for in this context will be localnodes block. - resultResponseHandler = new DatacenterWriteResponseHandler(replicaPlan, callback, writeType, queryStartNanoTime); + resultResponseHandler = new DatacenterWriteResponseHandler(replicaPlan, callback, writeType, hintOnFailure, queryStartNanoTime); } else if (replicaPlan.consistencyLevel() == ConsistencyLevel.EACH_QUORUM && (this instanceof NetworkTopologyStrategy)) { - resultResponseHandler = new DatacenterSyncWriteResponseHandler(replicaPlan, callback, writeType, queryStartNanoTime); + resultResponseHandler = new DatacenterSyncWriteResponseHandler(replicaPlan, callback, writeType, hintOnFailure, queryStartNanoTime); } else { - resultResponseHandler = new WriteResponseHandler(replicaPlan, callback, writeType, queryStartNanoTime); + resultResponseHandler = new WriteResponseHandler(replicaPlan, callback, writeType, hintOnFailure, queryStartNanoTime); } //Check if tracking the ideal consistency level is configured @@ -173,6 +178,7 @@ public abstract class AbstractReplicationStrategy AbstractWriteResponseHandler idealHandler = getWriteResponseHandler(replicaPlan.withConsistencyLevel(idealConsistencyLevel), callback, writeType, + hintOnFailure, queryStartNanoTime, idealConsistencyLevel); resultResponseHandler.setIdealCLResponseHandler(idealHandler); diff --git a/src/java/org/apache/cassandra/locator/Endpoints.java b/src/java/org/apache/cassandra/locator/Endpoints.java index 32e972a445..1561db2bce 100644 --- a/src/java/org/apache/cassandra/locator/Endpoints.java +++ b/src/java/org/apache/cassandra/locator/Endpoints.java @@ -21,7 +21,6 @@ package org.apache.cassandra.locator; import org.apache.cassandra.locator.ReplicaCollection.Builder.Conflict; import org.apache.cassandra.utils.FBUtilities; -import java.util.AbstractList; import java.util.Collection; import java.util.List; import java.util.Map; @@ -54,20 +53,14 @@ public abstract class Endpoints> extends AbstractReplicaC return byEndpoint().keySet(); } + public InetAddressAndPort endpoint(int i) + { + return get(i).endpoint(); + } + public List endpointList() { - return new AbstractList() - { - public InetAddressAndPort get(int index) - { - return list.get(index).endpoint(); - } - - public int size() - { - return list.size; - } - }; + return asList(Replica::endpoint); } public Map byEndpoint() @@ -87,6 +80,11 @@ public abstract class Endpoints> extends AbstractReplicaC replica); } + public boolean contains(InetAddressAndPort endpoint) + { + return endpoint != null && byEndpoint().containsKey(endpoint); + } + public E withoutSelf() { InetAddressAndPort self = FBUtilities.getBroadcastAddressAndPort(); diff --git a/src/java/org/apache/cassandra/locator/EndpointsForToken.java b/src/java/org/apache/cassandra/locator/EndpointsForToken.java index c709988762..70cd76325c 100644 --- a/src/java/org/apache/cassandra/locator/EndpointsForToken.java +++ b/src/java/org/apache/cassandra/locator/EndpointsForToken.java @@ -19,7 +19,11 @@ package org.apache.cassandra.locator; import com.google.common.base.Preconditions; + +import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.dht.Token; +import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.service.StorageService; import java.util.Arrays; import java.util.Collection; @@ -67,6 +71,11 @@ public class EndpointsForToken extends Endpoints return new EndpointsForToken(token, newList, byEndpoint); } + public Replica lookup(InetAddressAndPort endpoint) + { + return byEndpoint().get(endpoint); + } + public static class Builder extends EndpointsForToken implements ReplicaCollection.Builder { boolean built; @@ -146,4 +155,34 @@ public class EndpointsForToken extends Endpoints if (replicas.isEmpty()) return empty(token); return builder(token, replicas.size()).addAll(replicas).build(); } + + public static EndpointsForToken natural(Keyspace keyspace, Token token) + { + return keyspace.getReplicationStrategy().getNaturalReplicasForToken(token); + } + + public static EndpointsForToken natural(AbstractReplicationStrategy replicationStrategy, Token token) + { + return replicationStrategy.getNaturalReplicasForToken(token); + } + + public static EndpointsForToken natural(TableMetadata table, Token token) + { + return natural(Keyspace.open(table.keyspace), token); + } + + public static EndpointsForToken pending(TableMetadata table, Token token) + { + return pending(table.keyspace, token); + } + + public static EndpointsForToken pending(Keyspace keyspace, Token token) + { + return pending(keyspace.getName(), token); + } + + public static EndpointsForToken pending(String keyspace, Token token) + { + return StorageService.instance.getTokenMetadata().pendingEndpointsForToken(token, keyspace); + } } diff --git a/src/java/org/apache/cassandra/locator/InOurDcTester.java b/src/java/org/apache/cassandra/locator/InOurDc.java similarity index 71% rename from src/java/org/apache/cassandra/locator/InOurDcTester.java rename to src/java/org/apache/cassandra/locator/InOurDc.java index 181caceea8..34e8ef89c3 100644 --- a/src/java/org/apache/cassandra/locator/InOurDcTester.java +++ b/src/java/org/apache/cassandra/locator/InOurDc.java @@ -18,11 +18,12 @@ package org.apache.cassandra.locator; -import org.apache.cassandra.config.DatabaseDescriptor; - import java.util.function.Predicate; -public class InOurDcTester +import static org.apache.cassandra.config.DatabaseDescriptor.getEndpointSnitch; +import static org.apache.cassandra.config.DatabaseDescriptor.getLocalDataCenter; + +public class InOurDc { private static ReplicaTester replicas; private static EndpointTester endpoints; @@ -30,7 +31,7 @@ public class InOurDcTester final String dc; final IEndpointSnitch snitch; - private InOurDcTester(String dc, IEndpointSnitch snitch) + private InOurDc(String dc, IEndpointSnitch snitch) { this.dc = dc; this.snitch = snitch; @@ -38,15 +39,15 @@ public class InOurDcTester boolean stale() { - return dc != DatabaseDescriptor.getLocalDataCenter() - || snitch != DatabaseDescriptor.getEndpointSnitch() + return dc != getLocalDataCenter() + || snitch != getEndpointSnitch() // this final clause checks if somehow the snitch/localDc have got out of whack; // presently, this is possible but very unlikely, but this check will also help // resolve races on these global fields as well || !dc.equals(snitch.getLocalDatacenter()); } - private static final class ReplicaTester extends InOurDcTester implements Predicate + private static final class ReplicaTester extends InOurDc implements Predicate { private ReplicaTester(String dc, IEndpointSnitch snitch) { @@ -60,7 +61,7 @@ public class InOurDcTester } } - private static final class EndpointTester extends InOurDcTester implements Predicate + private static final class EndpointTester extends InOurDc implements Predicate { private EndpointTester(String dc, IEndpointSnitch snitch) { @@ -78,7 +79,7 @@ public class InOurDcTester { ReplicaTester cur = replicas; if (cur == null || cur.stale()) - replicas = cur = new ReplicaTester(DatabaseDescriptor.getLocalDataCenter(), DatabaseDescriptor.getEndpointSnitch()); + replicas = cur = new ReplicaTester(getLocalDataCenter(), getEndpointSnitch()); return cur; } @@ -86,8 +87,18 @@ public class InOurDcTester { EndpointTester cur = endpoints; if (cur == null || cur.stale()) - endpoints = cur = new EndpointTester(DatabaseDescriptor.getLocalDataCenter(), DatabaseDescriptor.getEndpointSnitch()); + endpoints = cur = new EndpointTester(getLocalDataCenter(), getEndpointSnitch()); return cur; } + public static boolean isInOurDc(Replica replica) + { + return replicas().test(replica); + } + + public static boolean isInOurDc(InetAddressAndPort endpoint) + { + return endpoints().test(endpoint); + } + } diff --git a/src/java/org/apache/cassandra/locator/ReplicaCollection.java b/src/java/org/apache/cassandra/locator/ReplicaCollection.java index ec671d5aa5..b679b506b0 100644 --- a/src/java/org/apache/cassandra/locator/ReplicaCollection.java +++ b/src/java/org/apache/cassandra/locator/ReplicaCollection.java @@ -62,14 +62,14 @@ public interface ReplicaCollection> extends Itera /** * @return the number of replicas that match the predicate */ - public abstract int count(Predicate predicate); + public abstract int count(Predicate predicate); /** * @return a *eagerly constructed* copy of this collection containing the Replica that match the provided predicate. * An effort will be made to either return ourself, or a subList, where possible. * It is guaranteed that no changes to any upstream Builder will affect the state of the result. */ - public abstract C filter(Predicate predicate); + public abstract C filter(Predicate predicate); /** * @return a *eagerly constructed* copy of this collection containing the Replica that match the provided predicate. @@ -77,18 +77,18 @@ public interface ReplicaCollection> extends Itera * It is guaranteed that no changes to any upstream Builder will affect the state of the result. * Only the first maxSize items will be returned. */ - public abstract C filter(Predicate predicate, int maxSize); + public abstract C filter(Predicate predicate, int maxSize); /** * @return a *lazily constructed* Iterable over this collection, containing the Replica that match the provided predicate. */ - public abstract Iterable filterLazily(Predicate predicate); + public abstract Iterable filterLazily(Predicate predicate); /** * @return a *lazily constructed* Iterable over this collection, containing the Replica that match the provided predicate. * Only the first maxSize matching items will be returned. */ - public abstract Iterable filterLazily(Predicate predicate, int maxSize); + public abstract Iterable filterLazily(Predicate predicate, int maxSize); /** * @return an *eagerly constructed* copy of this collection containing the Replica at positions [start..end); @@ -101,7 +101,7 @@ public interface ReplicaCollection> extends Itera * @return an *eagerly constructed* copy of this collection containing the Replica re-ordered according to this comparator * It is guaranteed that no changes to any upstream Builder will affect the state of the result. */ - public abstract C sorted(Comparator comparator); + public abstract C sorted(Comparator comparator); public abstract Iterator iterator(); public abstract Stream stream(); diff --git a/src/java/org/apache/cassandra/locator/ReplicaLayout.java b/src/java/org/apache/cassandra/locator/ReplicaLayout.java index ff817321be..351e837a86 100644 --- a/src/java/org/apache/cassandra/locator/ReplicaLayout.java +++ b/src/java/org/apache/cassandra/locator/ReplicaLayout.java @@ -25,7 +25,6 @@ import org.apache.cassandra.db.PartitionPosition; import org.apache.cassandra.dht.AbstractBounds; import org.apache.cassandra.dht.Token; import org.apache.cassandra.gms.FailureDetector; -import org.apache.cassandra.service.StorageService; import org.apache.cassandra.utils.FBUtilities; import java.util.Set; @@ -169,13 +168,14 @@ public abstract class ReplicaLayout> @Override public Token token() { return natural().token(); } - public ReplicaLayout.ForTokenWrite filter(Predicate filter) + public ForTokenWrite filter(Predicate filter) { EndpointsForToken filtered = all().filter(filter); // AbstractReplicaCollection.filter returns itself if all elements match the filter if (filtered == all()) return this; + if (pending().isEmpty()) return new ForTokenWrite(replicationStrategy(), filtered, pending(), filtered); // unique by endpoint, so can for efficiency filter only on endpoint - return new ReplicaLayout.ForTokenWrite( + return new ForTokenWrite( replicationStrategy(), natural().keep(filtered.endpoints()), pending().keep(filtered.endpoints()), @@ -206,8 +206,8 @@ public abstract class ReplicaLayout> // TODO: these should be cached, not the natural replicas // TODO: race condition to fetch these. implications?? AbstractReplicationStrategy replicationStrategy = keyspace.getReplicationStrategy(); - EndpointsForToken natural = replicationStrategy.getNaturalReplicasForToken(token); - EndpointsForToken pending = StorageService.instance.getTokenMetadata().pendingEndpointsForToken(token, keyspace.getName()); + EndpointsForToken natural = EndpointsForToken.natural(replicationStrategy, token); + EndpointsForToken pending = EndpointsForToken.pending(keyspace, token); return forTokenWrite(replicationStrategy, natural, pending); } @@ -325,7 +325,7 @@ public abstract class ReplicaLayout> * @return the read layout for a token - this includes only live natural replicas, i.e. those that are not pending * and not marked down by the failure detector. these are reverse sorted by the badness score of the configured snitch */ - static ReplicaLayout.ForTokenRead forTokenReadLiveSorted(AbstractReplicationStrategy replicationStrategy, Token token) + public static ReplicaLayout.ForTokenRead forTokenReadLiveSorted(AbstractReplicationStrategy replicationStrategy, Token token) { EndpointsForToken replicas = replicationStrategy.getNaturalReplicasForToken(token); replicas = DatabaseDescriptor.getEndpointSnitch().sortedByProximity(FBUtilities.getBroadcastAddressAndPort(), replicas); diff --git a/src/java/org/apache/cassandra/locator/ReplicaPlan.java b/src/java/org/apache/cassandra/locator/ReplicaPlan.java index 51cab134b8..3bb3ec0222 100644 --- a/src/java/org/apache/cassandra/locator/ReplicaPlan.java +++ b/src/java/org/apache/cassandra/locator/ReplicaPlan.java @@ -25,69 +25,89 @@ import org.apache.cassandra.db.PartitionPosition; import org.apache.cassandra.dht.AbstractBounds; import java.util.function.Predicate; +import java.util.function.Supplier; -public abstract class ReplicaPlan> +public interface ReplicaPlan, P extends ReplicaPlan> { - protected final Keyspace keyspace; - protected final ConsistencyLevel consistencyLevel; - // The snapshot of the replication strategy when instantiating. - // It could be different than the one fetched from Keyspace later, e.g. RS altered during the query. - // Use the snapshot to calculate {@code blockFor} in order to have a consistent view of RS for the query. - protected final AbstractReplicationStrategy replicationStrategy; + Keyspace keyspace(); + AbstractReplicationStrategy replicationStrategy(); + ConsistencyLevel consistencyLevel(); - // all nodes we will contact via any mechanism, including hints - // i.e., for: - // - reads, only live natural replicas - // ==> live.natural().subList(0, blockFor + initial speculate) - // - writes, includes all full, and any pending replicas, (and only any necessary transient ones to make up the difference) - // ==> liveAndDown.natural().filter(isFull) ++ liveAndDown.pending() ++ live.natural.filter(isTransient, req) - // - paxos, includes all live replicas (natural+pending), for this DC if SERIAL_LOCAL - // ==> live.all() (if consistencyLevel.isDCLocal(), then .filter(consistencyLevel.isLocal)) - private final E contacts; + E contacts(); - ReplicaPlan(Keyspace keyspace, AbstractReplicationStrategy replicationStrategy, ConsistencyLevel consistencyLevel, E contacts) + Replica lookup(InetAddressAndPort endpoint); + P withContacts(E contacts); + + interface ForRead, P extends ReplicaPlan.ForRead> extends ReplicaPlan { - assert contacts != null; - this.keyspace = keyspace; - this.replicationStrategy = replicationStrategy; - this.consistencyLevel = consistencyLevel; - this.contacts = contacts; + int readQuorum(); + E readCandidates(); + + default Replica firstUncontactedCandidate(Predicate extraPredicate) + { + return Iterables.tryFind(readCandidates(), r -> extraPredicate.test(r) && !contacts().contains(r)).orNull(); + } } - public abstract int blockFor(); + abstract class AbstractReplicaPlan, P extends ReplicaPlan> implements ReplicaPlan + { + protected final Keyspace keyspace; + protected final ConsistencyLevel consistencyLevel; + // The snapshot of the replication strategy when instantiating. + // It could be different than the one fetched from Keyspace later, e.g. RS altered during the query. + // Use the snapshot to calculate {@code blockFor} in order to have a consistent view of RS for the query. + protected final AbstractReplicationStrategy replicationStrategy; - public E contacts() { return contacts; } + // all nodes we will contact via any mechanism, including hints + // i.e., for: + // - reads, only live natural replicas + // ==> live.natural().subList(0, blockFor + initial speculate) + // - writes, includes all full, and any pending replicas, (and only any necessary transient ones to make up the difference) + // ==> liveAndDown.natural().filter(isFull) ++ liveAndDown.pending() ++ live.natural.filter(isTransient, req) + // - paxos, includes all live replicas (natural+pending), for this DC if SERIAL_LOCAL + // ==> live.all() (if consistencyLevel.isDCLocal(), then .filter(consistencyLevel.isLocal)) + private final E contacts; - // TODO: should this semantically return true if we contain the endpoint, not the exact replica? - public boolean contacts(Replica replica) { return contacts.contains(replica); } - public Keyspace keyspace() { return keyspace; } - public AbstractReplicationStrategy replicationStrategy() { return replicationStrategy; } - public ConsistencyLevel consistencyLevel() { return consistencyLevel; } + AbstractReplicaPlan(Keyspace keyspace, AbstractReplicationStrategy replicationStrategy, ConsistencyLevel consistencyLevel, E contacts) + { + assert contacts != null; + this.keyspace = keyspace; + this.replicationStrategy = replicationStrategy; + this.consistencyLevel = consistencyLevel; + this.contacts = contacts; + } - public static abstract class ForRead> extends ReplicaPlan + public E contacts() { return contacts; } + + public Keyspace keyspace() { return keyspace; } + public AbstractReplicationStrategy replicationStrategy() { return replicationStrategy; } + public ConsistencyLevel consistencyLevel() { return consistencyLevel; } + } + + public static abstract class AbstractForRead, P extends ForRead> extends AbstractReplicaPlan implements ForRead { // all nodes we *could* contacts; typically all natural replicas that are believed to be alive // we will consult this collection to find uncontacted nodes we might contact if we doubt we will meet consistency level - private final E candidates; + final E candidates; - ForRead(Keyspace keyspace, AbstractReplicationStrategy replicationStrategy, ConsistencyLevel consistencyLevel, E candidates, E contacts) + AbstractForRead(Keyspace keyspace, AbstractReplicationStrategy replicationStrategy, ConsistencyLevel consistencyLevel, E candidates, E contacts) { super(keyspace, replicationStrategy, consistencyLevel, contacts); this.candidates = candidates; } - public int blockFor() { return consistencyLevel.blockFor(replicationStrategy); } + public int readQuorum() { return consistencyLevel.blockFor(replicationStrategy); } - public E candidates() { return candidates; } + public E readCandidates() { return candidates; } public Replica firstUncontactedCandidate(Predicate extraPredicate) { - return Iterables.tryFind(candidates(), r -> extraPredicate.test(r) && !contacts(r)).orNull(); + return Iterables.tryFind(readCandidates(), r -> extraPredicate.test(r) && !contacts().contains(r)).orNull(); } public Replica lookup(InetAddressAndPort endpoint) { - return candidates().byEndpoint().get(endpoint); + return readCandidates().byEndpoint().get(endpoint); } public String toString() @@ -96,7 +116,7 @@ public abstract class ReplicaPlan> } } - public static class ForTokenRead extends ForRead + public static class ForTokenRead extends AbstractForRead { public ForTokenRead(Keyspace keyspace, AbstractReplicationStrategy replicationStrategy, @@ -107,13 +127,13 @@ public abstract class ReplicaPlan> super(keyspace, replicationStrategy, consistencyLevel, candidates, contacts); } - ForTokenRead withContact(EndpointsForToken newContact) + public ForTokenRead withContacts(EndpointsForToken newContact) { - return new ForTokenRead(keyspace, replicationStrategy, consistencyLevel, candidates(), newContact); + return new ForTokenRead(keyspace, replicationStrategy, consistencyLevel, candidates, newContact); } } - public static class ForRangeRead extends ForRead + public static class ForRangeRead extends AbstractForRead { final AbstractBounds range; final int vnodeCount; @@ -138,20 +158,20 @@ public abstract class ReplicaPlan> */ public int vnodeCount() { return vnodeCount; } - ForRangeRead withContact(EndpointsForRange newContact) + public ForRangeRead withContacts(EndpointsForRange newContact) { - return new ForRangeRead(keyspace, replicationStrategy, consistencyLevel, range, candidates(), newContact, vnodeCount); + return new ForRangeRead(keyspace, replicationStrategy, consistencyLevel, range, readCandidates(), newContact, vnodeCount); } } - public static abstract class ForWrite> extends ReplicaPlan + public static class ForWrite extends AbstractReplicaPlan { // TODO: this is only needed because of poor isolation of concerns elsewhere - we can remove it soon, and will do so in a follow-up patch - final E pending; - final E liveAndDown; - final E live; + final EndpointsForToken pending; + final EndpointsForToken liveAndDown; + final EndpointsForToken live; - ForWrite(Keyspace keyspace, AbstractReplicationStrategy replicationStrategy, ConsistencyLevel consistencyLevel, E pending, E liveAndDown, E live, E contact) + public ForWrite(Keyspace keyspace, AbstractReplicationStrategy replicationStrategy, ConsistencyLevel consistencyLevel, EndpointsForToken pending, EndpointsForToken liveAndDown, EndpointsForToken live, EndpointsForToken contact) { super(keyspace, replicationStrategy, consistencyLevel, contact); this.pending = pending; @@ -159,46 +179,43 @@ public abstract class ReplicaPlan> this.live = live; } - public int blockFor() { return consistencyLevel.blockForWrite(replicationStrategy, pending()); } + public int writeQuorum() { return consistencyLevel.blockForWrite(replicationStrategy, pending()); } /** Replicas that a region of the ring is moving to; not yet ready to serve reads, but should receive writes */ - public E pending() { return pending; } + public EndpointsForToken pending() { return pending; } + /** Replicas that can participate in the write - this always includes all nodes (pending and natural) in all DCs, except for paxos LOCAL_QUORUM (which is local DC only) */ - public E liveAndDown() { return liveAndDown; } + public EndpointsForToken liveAndDown() { return liveAndDown; } + /** The live replicas present in liveAndDown, usually derived from FailureDetector.isReplicaAlive */ - public E live() { return live; } + public EndpointsForToken live() { return live; } + /** Calculate which live endpoints we could have contacted, but chose not to */ - public E liveUncontacted() { return live().filter(r -> !contacts(r)); } + public EndpointsForToken liveUncontacted() { return live().filter(r -> !contacts().contains(r)); } + /** Test liveness, consistent with the upfront analysis done for this operation (i.e. test membership of live()) */ public boolean isAlive(Replica replica) { return live.endpoints().contains(replica.endpoint()); } + public Replica lookup(InetAddressAndPort endpoint) { return liveAndDown().byEndpoint().get(endpoint); } + private ForWrite copy(ConsistencyLevel newConsistencyLevel, EndpointsForToken newContact) + { + return new ForWrite(keyspace, replicationStrategy, newConsistencyLevel, pending(), liveAndDown(), live(), newContact); + } + + ForWrite withConsistencyLevel(ConsistencyLevel newConsistencylevel) { return copy(newConsistencylevel, contacts()); } + public ForWrite withContacts(EndpointsForToken newContact) { return copy(consistencyLevel, newContact); } + public String toString() { return "ReplicaPlan.ForWrite [ CL: " + consistencyLevel + " keyspace: " + keyspace + " liveAndDown: " + liveAndDown + " live: " + live + " contacts: " + contacts() + " ]"; } } - public static class ForTokenWrite extends ForWrite - { - public ForTokenWrite(Keyspace keyspace, AbstractReplicationStrategy replicationStrategy, ConsistencyLevel consistencyLevel, EndpointsForToken pending, EndpointsForToken liveAndDown, EndpointsForToken live, EndpointsForToken contact) - { - super(keyspace, replicationStrategy, consistencyLevel, pending, liveAndDown, live, contact); - } - - private ReplicaPlan.ForTokenWrite copy(ConsistencyLevel newConsistencyLevel, EndpointsForToken newContact) - { - return new ReplicaPlan.ForTokenWrite(keyspace, replicationStrategy, newConsistencyLevel, pending(), liveAndDown(), live(), newContact); - } - - ForTokenWrite withConsistencyLevel(ConsistencyLevel newConsistencylevel) { return copy(newConsistencylevel, contacts()); } - public ForTokenWrite withContact(EndpointsForToken newContact) { return copy(consistencyLevel, newContact); } - } - - public static class ForPaxosWrite extends ForWrite + public static class ForPaxosWrite extends ForWrite { final int requiredParticipants; @@ -219,8 +236,11 @@ public abstract class ReplicaPlan> * the constructor should be visible by the normal process of sharing data between threads (i.e. executors, etc) * and any updates will either be seen or not seen, perhaps not promptly, but certainly not incompletely. * The contained ReplicaPlan has only final member properties, so it cannot be seen partially initialised. + * + * TODO: there's no reason this couldn't be achieved instead by a ReplicaPlan with mutable contacts, + * simplifying the hierarchy */ - public interface Shared, P extends ReplicaPlan> + public interface Shared, P extends ReplicaPlan> extends Supplier

{ /** * add the provided replica to this shared plan, by updating the internal reference @@ -230,29 +250,22 @@ public abstract class ReplicaPlan> * get the shared replica plan, non-volatile (so maybe stale) but no risk of partially initialised */ public P get(); - /** - * get the shared replica plan, non-volatile (so maybe stale) but no risk of partially initialised, - * but replace its 'contacts' with those provided - */ - public abstract P getWithContacts(E endpoints); } public static class SharedForTokenRead implements Shared { private ForTokenRead replicaPlan; SharedForTokenRead(ForTokenRead replicaPlan) { this.replicaPlan = replicaPlan; } - public void addToContacts(Replica replica) { replicaPlan = replicaPlan.withContact(Endpoints.append(replicaPlan.contacts(), replica)); } + public void addToContacts(Replica replica) { replicaPlan = replicaPlan.withContacts(Endpoints.append(replicaPlan.contacts(), replica)); } public ForTokenRead get() { return replicaPlan; } - public ForTokenRead getWithContacts(EndpointsForToken newContact) { return replicaPlan.withContact(newContact); } } public static class SharedForRangeRead implements Shared { private ForRangeRead replicaPlan; SharedForRangeRead(ForRangeRead replicaPlan) { this.replicaPlan = replicaPlan; } - public void addToContacts(Replica replica) { replicaPlan = replicaPlan.withContact(Endpoints.append(replicaPlan.contacts(), replica)); } + public void addToContacts(Replica replica) { replicaPlan = replicaPlan.withContacts(Endpoints.append(replicaPlan.contacts(), replica)); } public ForRangeRead get() { return replicaPlan; } - public ForRangeRead getWithContacts(EndpointsForRange newContact) { return replicaPlan.withContact(newContact); } } public static SharedForTokenRead shared(ForTokenRead replicaPlan) { return new SharedForTokenRead(replicaPlan); } diff --git a/src/java/org/apache/cassandra/locator/ReplicaPlans.java b/src/java/org/apache/cassandra/locator/ReplicaPlans.java index 85979d372c..c39862adff 100644 --- a/src/java/org/apache/cassandra/locator/ReplicaPlans.java +++ b/src/java/org/apache/cassandra/locator/ReplicaPlans.java @@ -133,7 +133,7 @@ public class ReplicaPlans if (logger.isTraceEnabled()) { logger.trace(String.format("Local replicas %s are insufficient to satisfy LOCAL_QUORUM requirement of %d live replicas and %d full replicas in '%s'", - allLive.filter(InOurDcTester.replicas()), blockFor, blockForFullReplicas, DatabaseDescriptor.getLocalDataCenter())); + allLive.filter(InOurDc.replicas()), blockFor, blockForFullReplicas, DatabaseDescriptor.getLocalDataCenter())); } throw UnavailableException.create(consistencyLevel, blockFor, blockForFullReplicas, localLive.allReplicas(), localLive.fullReplicas()); } @@ -175,23 +175,23 @@ public class ReplicaPlans /** * Construct a ReplicaPlan for writing to exactly one node, with CL.ONE. This node is *assumed* to be alive. */ - public static ReplicaPlan.ForTokenWrite forSingleReplicaWrite(Keyspace keyspace, Token token, Replica replica) + public static ReplicaPlan.ForWrite forSingleReplicaWrite(Keyspace keyspace, Token token, Replica replica) { EndpointsForToken one = EndpointsForToken.of(token, replica); EndpointsForToken empty = EndpointsForToken.empty(token); - return new ReplicaPlan.ForTokenWrite(keyspace, keyspace.getReplicationStrategy(), ConsistencyLevel.ONE, empty, one, one, one); + return new ReplicaPlan.ForWrite(keyspace, keyspace.getReplicationStrategy(), ConsistencyLevel.ONE, empty, one, one, one); } /** * A forwarding counter write is always sent to a single owning coordinator for the range, by the original coordinator * (if it is not itself an owner) */ - public static ReplicaPlan.ForTokenWrite forForwardingCounterWrite(Keyspace keyspace, Token token, Replica replica) + public static ReplicaPlan.ForWrite forForwardingCounterWrite(Keyspace keyspace, Token token, Replica replica) { return forSingleReplicaWrite(keyspace, token, replica); } - public static ReplicaPlan.ForTokenWrite forLocalBatchlogWrite() + public static ReplicaPlan.ForWrite forLocalBatchlogWrite() { Token token = DatabaseDescriptor.getPartitioner().getMinimumToken(); Keyspace systemKeypsace = Keyspace.open(SchemaConstants.SYSTEM_KEYSPACE_NAME); @@ -211,7 +211,7 @@ public class ReplicaPlans * * @param isAny if batch consistency level is ANY, in which case a local node will be picked */ - public static ReplicaPlan.ForTokenWrite forBatchlogWrite(boolean isAny) throws UnavailableException + public static ReplicaPlan.ForWrite forBatchlogWrite(boolean isAny) throws UnavailableException { // A single case we write not for range or token, but multiple mutations to many tokens Token token = DatabaseDescriptor.getPartitioner().getMinimumToken(); @@ -317,41 +317,41 @@ public class ReplicaPlans return result; } - public static ReplicaPlan.ForTokenWrite forReadRepair(Token token, ReplicaPlan.ForRead readPlan) throws UnavailableException + public static ReplicaPlan.ForWrite forReadRepair(Token token, ReplicaPlan readPlan) throws UnavailableException { - return forWrite(readPlan.keyspace, readPlan.consistencyLevel, token, writeReadRepair(readPlan)); + return forWrite(readPlan.keyspace(), readPlan.consistencyLevel(), token, writeReadRepair(readPlan)); } - public static ReplicaPlan.ForTokenWrite forWrite(Keyspace keyspace, ConsistencyLevel consistencyLevel, Token token, Selector selector) throws UnavailableException + public static ReplicaPlan.ForWrite forWrite(Keyspace keyspace, ConsistencyLevel consistencyLevel, Token token, Selector selector) throws UnavailableException { return forWrite(keyspace, consistencyLevel, ReplicaLayout.forTokenWriteLiveAndDown(keyspace, token), selector); } @VisibleForTesting - public static ReplicaPlan.ForTokenWrite forWrite(Keyspace keyspace, ConsistencyLevel consistencyLevel, EndpointsForToken natural, EndpointsForToken pending, Predicate isAlive, Selector selector) throws UnavailableException + public static ReplicaPlan.ForWrite forWrite(Keyspace keyspace, ConsistencyLevel consistencyLevel, EndpointsForToken natural, EndpointsForToken pending, Predicate isAlive, Selector selector) throws UnavailableException { return forWrite(keyspace, consistencyLevel, ReplicaLayout.forTokenWrite(keyspace.getReplicationStrategy(), natural, pending), isAlive, selector); } - public static ReplicaPlan.ForTokenWrite forWrite(Keyspace keyspace, ConsistencyLevel consistencyLevel, ReplicaLayout.ForTokenWrite liveAndDown, Selector selector) throws UnavailableException + public static ReplicaPlan.ForWrite forWrite(Keyspace keyspace, ConsistencyLevel consistencyLevel, ReplicaLayout.ForTokenWrite liveAndDown, Selector selector) throws UnavailableException { return forWrite(keyspace, consistencyLevel, liveAndDown, FailureDetector.isReplicaAlive, selector); } - private static ReplicaPlan.ForTokenWrite forWrite(Keyspace keyspace, ConsistencyLevel consistencyLevel, ReplicaLayout.ForTokenWrite liveAndDown, Predicate isAlive, Selector selector) throws UnavailableException + private static ReplicaPlan.ForWrite forWrite(Keyspace keyspace, ConsistencyLevel consistencyLevel, ReplicaLayout.ForTokenWrite liveAndDown, Predicate isAlive, Selector selector) throws UnavailableException { ReplicaLayout.ForTokenWrite live = liveAndDown.filter(isAlive); return forWrite(keyspace, consistencyLevel, liveAndDown, live, selector); } - public static ReplicaPlan.ForTokenWrite forWrite(Keyspace keyspace, ConsistencyLevel consistencyLevel, ReplicaLayout.ForTokenWrite liveAndDown, ReplicaLayout.ForTokenWrite live, Selector selector) throws UnavailableException + public static ReplicaPlan.ForWrite forWrite(Keyspace keyspace, ConsistencyLevel consistencyLevel, ReplicaLayout.ForTokenWrite liveAndDown, ReplicaLayout.ForTokenWrite live, Selector selector) throws UnavailableException { assert liveAndDown.replicationStrategy() == live.replicationStrategy() : "ReplicaLayout liveAndDown and live should be derived from the same replication strategy."; AbstractReplicationStrategy replicationStrategy = liveAndDown.replicationStrategy(); EndpointsForToken contacts = selector.select(consistencyLevel, liveAndDown, live); assureSufficientLiveReplicasForWrite(replicationStrategy, consistencyLevel, live.all(), liveAndDown.pending()); - return new ReplicaPlan.ForTokenWrite(keyspace, replicationStrategy, consistencyLevel, liveAndDown.pending(), liveAndDown.all(), live.all(), contacts); + return new ReplicaPlan.ForWrite(keyspace, replicationStrategy, consistencyLevel, liveAndDown.pending(), liveAndDown.all(), live.all(), contacts); } public interface Selector @@ -434,7 +434,7 @@ public class ReplicaPlans * the minimal number of nodes to meet the consistency level, and prefer nodes we contacted on read to minimise * data transfer. */ - public static Selector writeReadRepair(ReplicaPlan.ForRead readPlan) + public static Selector writeReadRepair(ReplicaPlan readPlan) { return new Selector() { @@ -497,7 +497,7 @@ public class ReplicaPlans { // TODO: we should cleanup our semantics here, as we're filtering ALL nodes to localDC which is unexpected for ReplicaPlan // Restrict natural and pending to node in the local DC only - liveAndDown = liveAndDown.filter(InOurDcTester.replicas()); + liveAndDown = liveAndDown.filter(InOurDc.replicas()); } ReplicaLayout.ForTokenWrite live = liveAndDown.filter(FailureDetector.isReplicaAlive); @@ -526,7 +526,7 @@ public class ReplicaPlans private static > E candidatesForRead(ConsistencyLevel consistencyLevel, E liveNaturalReplicas) { return consistencyLevel.isDatacenterLocal() - ? liveNaturalReplicas.filter(InOurDcTester.replicas()) + ? liveNaturalReplicas.filter(InOurDc.replicas()) : liveNaturalReplicas; } @@ -621,7 +621,7 @@ public class ReplicaPlans { // TODO: should we be asserting that the ranges are adjacent? AbstractBounds newRange = left.range().withNewRight(right.range().right); - EndpointsForRange mergedCandidates = left.candidates().keep(right.candidates().endpoints()); + EndpointsForRange mergedCandidates = left.readCandidates().keep(right.readCandidates().endpoints()); AbstractReplicationStrategy replicationStrategy = keyspace.getReplicationStrategy(); // Check if there are enough shared endpoints for the merge to be possible. diff --git a/src/java/org/apache/cassandra/locator/Replicas.java b/src/java/org/apache/cassandra/locator/Replicas.java index 1b299cf589..c53815c563 100644 --- a/src/java/org/apache/cassandra/locator/Replicas.java +++ b/src/java/org/apache/cassandra/locator/Replicas.java @@ -78,7 +78,7 @@ public class Replicas public static ReplicaCount countInOurDc(ReplicaCollection replicas) { ReplicaCount count = new ReplicaCount(); - Predicate inOurDc = InOurDcTester.replicas(); + Predicate inOurDc = InOurDc.replicas(); for (Replica replica : replicas) if (inOurDc.test(replica)) count.increment(replica); diff --git a/src/java/org/apache/cassandra/locator/TokenMetadata.java b/src/java/org/apache/cassandra/locator/TokenMetadata.java index 0e60a1f2a9..51dd12063f 100644 --- a/src/java/org/apache/cassandra/locator/TokenMetadata.java +++ b/src/java/org/apache/cassandra/locator/TokenMetadata.java @@ -1391,7 +1391,7 @@ public class TokenMetadata */ public Topology getTopology() { - assert this != StorageService.instance.getTokenMetadata(); + assert !DatabaseDescriptor.isDaemonInitialized() || this != StorageService.instance.getTokenMetadata(); return topology; } diff --git a/src/java/org/apache/cassandra/metrics/ClientRequestsMetricsHolder.java b/src/java/org/apache/cassandra/metrics/ClientRequestsMetricsHolder.java index 05ab338dab..26f2913263 100644 --- a/src/java/org/apache/cassandra/metrics/ClientRequestsMetricsHolder.java +++ b/src/java/org/apache/cassandra/metrics/ClientRequestsMetricsHolder.java @@ -30,8 +30,8 @@ public final class ClientRequestsMetricsHolder public static final CASClientRequestMetrics casReadMetrics = new CASClientRequestMetrics("CASRead"); public static final ViewWriteMetrics viewWriteMetrics = new ViewWriteMetrics("ViewWrite"); - private static final Map readMetricsMap = new EnumMap<>(ConsistencyLevel.class); - private static final Map writeMetricsMap = new EnumMap<>(ConsistencyLevel.class); + public static final Map readMetricsMap = new EnumMap<>(ConsistencyLevel.class); + public static final Map writeMetricsMap = new EnumMap<>(ConsistencyLevel.class); static { diff --git a/src/java/org/apache/cassandra/metrics/DecayingEstimatedHistogramReservoir.java b/src/java/org/apache/cassandra/metrics/DecayingEstimatedHistogramReservoir.java index 6af2389bf3..93b385bf6d 100644 --- a/src/java/org/apache/cassandra/metrics/DecayingEstimatedHistogramReservoir.java +++ b/src/java/org/apache/cassandra/metrics/DecayingEstimatedHistogramReservoir.java @@ -22,17 +22,23 @@ import java.io.OutputStream; import java.io.OutputStreamWriter; import java.io.PrintWriter; import java.nio.charset.StandardCharsets; +import java.util.Arrays; import java.util.Objects; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLongArray; import com.google.common.annotations.VisibleForTesting; import com.google.common.primitives.Ints; -import com.codahale.metrics.Clock; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import com.codahale.metrics.Reservoir; import com.codahale.metrics.Snapshot; import org.apache.cassandra.utils.EstimatedHistogram; +import org.apache.cassandra.utils.MonotonicClock; +import org.apache.cassandra.utils.NoSpamLogger; import static java.lang.Math.max; import static java.lang.Math.min; @@ -78,7 +84,8 @@ import static java.lang.Math.min; */ public class DecayingEstimatedHistogramReservoir implements Reservoir { - + private static Logger logger = LoggerFactory.getLogger(DecayingEstimatedHistogramReservoir.class); + private static NoSpamLogger noSpamLogger = NoSpamLogger.getLogger(logger, 5L, TimeUnit.MINUTES); /** * The default number of decayingBuckets. Use this bucket count to reduce memory allocation for bucket offsets. */ @@ -147,21 +154,20 @@ public class DecayingEstimatedHistogramReservoir implements Reservoir public static final long HALF_TIME_IN_S = 60L; public static final double MEAN_LIFETIME_IN_S = HALF_TIME_IN_S / Math.log(2.0); - public static final long LANDMARK_RESET_INTERVAL_IN_MS = 30L * 60L * 1000L; + public static final long LANDMARK_RESET_INTERVAL_IN_NS = TimeUnit.MINUTES.toNanos(30L); private final AtomicBoolean rescaling = new AtomicBoolean(false); private volatile long decayLandmark; // Wrapper around System.nanoTime() to simplify unit testing. - private final Clock clock; - + private final MonotonicClock clock; /** * Construct a decaying histogram with default number of buckets and without considering zeroes. */ public DecayingEstimatedHistogramReservoir() { - this(DEFAULT_ZERO_CONSIDERATION, DEFAULT_BUCKET_COUNT, DEFAULT_STRIPE_COUNT, Clock.defaultClock()); + this(DEFAULT_ZERO_CONSIDERATION, DEFAULT_BUCKET_COUNT, DEFAULT_STRIPE_COUNT, MonotonicClock.Global.approxTime); } /** @@ -172,7 +178,7 @@ public class DecayingEstimatedHistogramReservoir implements Reservoir */ public DecayingEstimatedHistogramReservoir(boolean considerZeroes) { - this(considerZeroes, DEFAULT_BUCKET_COUNT, DEFAULT_STRIPE_COUNT, Clock.defaultClock()); + this(considerZeroes, DEFAULT_BUCKET_COUNT, DEFAULT_STRIPE_COUNT, MonotonicClock.Global.approxTime); } /** @@ -184,17 +190,17 @@ public class DecayingEstimatedHistogramReservoir implements Reservoir */ public DecayingEstimatedHistogramReservoir(boolean considerZeroes, int bucketCount, int stripes) { - this(considerZeroes, bucketCount, stripes, Clock.defaultClock()); + this(considerZeroes, bucketCount, stripes, MonotonicClock.Global.approxTime); } @VisibleForTesting - public DecayingEstimatedHistogramReservoir(Clock clock) + public DecayingEstimatedHistogramReservoir(MonotonicClock clock) { this(DEFAULT_ZERO_CONSIDERATION, DEFAULT_BUCKET_COUNT, DEFAULT_STRIPE_COUNT, clock); } @VisibleForTesting - DecayingEstimatedHistogramReservoir(boolean considerZeroes, int bucketCount, int stripes, Clock clock) + DecayingEstimatedHistogramReservoir(boolean considerZeroes, int bucketCount, int stripes, MonotonicClock clock) { assert bucketCount <= MAX_BUCKET_COUNT : "bucket count cannot exceed: " + MAX_BUCKET_COUNT; @@ -218,7 +224,7 @@ public class DecayingEstimatedHistogramReservoir implements Reservoir decayingBuckets = new AtomicLongArray((bucketOffsets.length + 1) * nStripes); buckets = new AtomicLongArray((bucketOffsets.length + 1) * nStripes); this.clock = clock; - decayLandmark = clock.getTime(); + decayLandmark = clock.now(); int distributionPrime = 1; for (int prime : DISTRIBUTION_PRIMES) { @@ -238,7 +244,7 @@ public class DecayingEstimatedHistogramReservoir implements Reservoir */ public void update(long value) { - long now = clock.getTime(); + long now = clock.now(); rescaleIfNeeded(now); int index = findIndex(bucketOffsets, value); @@ -283,7 +289,7 @@ public class DecayingEstimatedHistogramReservoir implements Reservoir private double forwardDecayWeight(long now) { - return Math.exp(((now - decayLandmark) / 1000.0) / MEAN_LIFETIME_IN_S); + return Math.exp(TimeUnit.NANOSECONDS.toSeconds(now - decayLandmark) / MEAN_LIFETIME_IN_S); } /** @@ -345,7 +351,7 @@ public class DecayingEstimatedHistogramReservoir implements Reservoir private void rescaleIfNeeded() { - rescaleIfNeeded(clock.getTime()); + rescaleIfNeeded(clock.now()); } private void rescaleIfNeeded(long now) @@ -380,7 +386,7 @@ public class DecayingEstimatedHistogramReservoir implements Reservoir private boolean needRescale(long now) { - return (now - decayLandmark) > LANDMARK_RESET_INTERVAL_IN_MS; + return (now - decayLandmark) > LANDMARK_RESET_INTERVAL_IN_NS; } @VisibleForTesting @@ -452,7 +458,7 @@ public class DecayingEstimatedHistogramReservoir implements Reservoir public EstimatedHistogramReservoirSnapshot(DecayingEstimatedHistogramReservoir reservoir) { final int length = reservoir.size(); - final double rescaleFactor = reservoir.forwardDecayWeight(reservoir.clock.getTime()); + final double rescaleFactor = reservoir.forwardDecayWeight(reservoir.clock.now()); this.decayingBuckets = new long[length]; this.values = new long[length]; @@ -482,7 +488,10 @@ public class DecayingEstimatedHistogramReservoir implements Reservoir final int lastBucket = decayingBuckets.length - 1; if (decayingBuckets[lastBucket] > 0) - throw new IllegalStateException("Unable to compute when histogram overflowed"); + { + try { throw new IllegalStateException("EstimatedHistogram overflow: " + Arrays.toString(decayingBuckets)); } + catch (IllegalStateException e) { noSpamLogger.warn("", e); } + } final long qcount = (long) Math.ceil(count() * quantile); if (qcount == 0) @@ -723,7 +732,7 @@ public class DecayingEstimatedHistogramReservoir implements Reservoir } } - public void rebaseReservoir() + public void rebaseReservoir() { this.reservoir.rebase(this); } @@ -753,5 +762,11 @@ public class DecayingEstimatedHistogramReservoir implements Reservoir { return Objects.hash(min, max); } + + @Override + public String toString() + { + return "[" + min + ',' + max + ']'; + } } } diff --git a/src/java/org/apache/cassandra/metrics/HintedHandoffMetrics.java b/src/java/org/apache/cassandra/metrics/HintedHandoffMetrics.java index c611bd0599..f580d1c049 100644 --- a/src/java/org/apache/cassandra/metrics/HintedHandoffMetrics.java +++ b/src/java/org/apache/cassandra/metrics/HintedHandoffMetrics.java @@ -28,7 +28,7 @@ import com.github.benmanes.caffeine.cache.LoadingCache; import org.apache.cassandra.concurrent.ImmediateExecutor; import org.apache.cassandra.db.SystemKeyspace; import org.apache.cassandra.locator.InetAddressAndPort; -import org.apache.cassandra.utils.UUIDGen; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/src/java/org/apache/cassandra/metrics/PaxosMetrics.java b/src/java/org/apache/cassandra/metrics/PaxosMetrics.java new file mode 100644 index 0000000000..62c62d806e --- /dev/null +++ b/src/java/org/apache/cassandra/metrics/PaxosMetrics.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.metrics; + +import com.codahale.metrics.Counter; + +import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics; + +public class PaxosMetrics +{ + private static final MetricNameFactory factory = new DefaultNameFactory("Paxos"); + public static final Counter linearizabilityViolations = Metrics.counter(factory.createMetricName("LinearizabilityViolations")); + public static void initialize() {} +} diff --git a/src/java/org/apache/cassandra/net/Message.java b/src/java/org/apache/cassandra/net/Message.java index cbe258d17d..fc0e5c60cd 100644 --- a/src/java/org/apache/cassandra/net/Message.java +++ b/src/java/org/apache/cassandra/net/Message.java @@ -43,7 +43,6 @@ import org.apache.cassandra.io.util.DataOutputPlus; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.tracing.Tracing; import org.apache.cassandra.tracing.Tracing.TraceType; -import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.MonotonicClockTranslation; import org.apache.cassandra.utils.NoSpamLogger; import org.apache.cassandra.utils.TimeUUID; @@ -57,6 +56,7 @@ import static org.apache.cassandra.net.MessagingService.VERSION_3014; import static org.apache.cassandra.net.MessagingService.VERSION_30; import static org.apache.cassandra.net.MessagingService.VERSION_40; import static org.apache.cassandra.net.MessagingService.instance; +import static org.apache.cassandra.utils.FBUtilities.getBroadcastAddressAndPort; import static org.apache.cassandra.utils.MonotonicClock.Global.approxTime; import static org.apache.cassandra.utils.vint.VIntCoding.computeUnsignedVIntSize; import static org.apache.cassandra.utils.vint.VIntCoding.getUnsignedVInt; @@ -76,7 +76,7 @@ public class Message public final Header header; public final T payload; - private Message(Header header, T payload) + Message(Header header, T payload) { this.header = header; this.payload = payload; @@ -91,7 +91,7 @@ public class Message /** Whether the message has crossed the node boundary, that is whether it originated from another node. */ public boolean isCrossNode() { - return !from().equals(FBUtilities.getBroadcastAddressAndPort()); + return !from().equals(getBroadcastAddressAndPort()); } /** @@ -201,6 +201,11 @@ public class Message return outWithParam(nextId(), verb, payload, null, null); } + public static Message synthetic(InetAddressAndPort from, Verb verb, T payload) + { + return new Message<>(new Header(-1, verb, from, -1, -1, 0, NO_PARAMS), payload); + } + public static Message out(Verb verb, T payload, long expiresAtNanos) { return outWithParam(nextId(), verb, expiresAtNanos, payload, 0, null, null); @@ -218,6 +223,7 @@ public class Message return outWithParam(nextId(), verb, 0, payload, flag2.addTo(flag1.addTo(0)), null, null); } + @VisibleForTesting static Message outWithParam(long id, Verb verb, T payload, ParamType paramType, Object paramValue) { return outWithParam(id, verb, 0, payload, paramType, paramValue); @@ -229,11 +235,15 @@ public class Message } private static Message outWithParam(long id, Verb verb, long expiresAtNanos, T payload, int flags, ParamType paramType, Object paramValue) + { + return withParam(getBroadcastAddressAndPort(), id, verb, expiresAtNanos, payload, flags, paramType, paramValue); + } + + private static Message withParam(InetAddressAndPort from, long id, Verb verb, long expiresAtNanos, T payload, int flags, ParamType paramType, Object paramValue) { if (payload == null) throw new IllegalArgumentException(); - InetAddressAndPort from = FBUtilities.getBroadcastAddressAndPort(); long createdAtNanos = approxTime.now(); if (expiresAtNanos == 0) expiresAtNanos = verb.expiresAtNanos(createdAtNanos); @@ -270,6 +280,11 @@ public class Message return outWithParam(id, Verb.FAILURE_RSP, expiresAtNanos, reason, null, null); } + public Message withPayload(V newPayload) + { + return new Message<>(header, newPayload); + } + Message withCallBackOnFailure() { return new Message<>(header.withFlag(MessageFlag.CALL_BACK_ON_FAILURE), payload); @@ -452,7 +467,9 @@ public class Message @Nullable InetAddressAndPort respondTo() { - return (InetAddressAndPort) params.get(ParamType.RESPOND_TO); + InetAddressAndPort respondTo = (InetAddressAndPort) params.get(ParamType.RESPOND_TO); + if (respondTo == null) respondTo = from; + return respondTo; } @Nullable @@ -551,7 +568,7 @@ public class Message if (expiresAtNanos == 0 && verb != null && createdAtNanos != 0) expiresAtNanos = verb.expiresAtNanos(createdAtNanos); if (!this.verb.isResponse() && from == null) // default to sending from self if we're a request verb - from = FBUtilities.getBroadcastAddressAndPort(); + from = getBroadcastAddressAndPort(); return this; } @@ -812,7 +829,7 @@ public class Message { serializeHeaderPost40(message.header, out, version); out.writeUnsignedVInt(message.payloadSize(version)); - message.getPayloadSerializer().serialize(message.payload, out, version); + message.verb().serializer().serialize(message.payload, out, version); } private Message deserializePost40(DataInputPlus in, InetAddressAndPort peer, int version) throws IOException diff --git a/src/java/org/apache/cassandra/net/MessagingService.java b/src/java/org/apache/cassandra/net/MessagingService.java index e2e3104e9c..ce2fdecfc8 100644 --- a/src/java/org/apache/cassandra/net/MessagingService.java +++ b/src/java/org/apache/cassandra/net/MessagingService.java @@ -206,6 +206,7 @@ public final class MessagingService extends MessagingServiceMBeanImpl public static final int VERSION_30 = 10; public static final int VERSION_3014 = 11; public static final int VERSION_40 = 12; + public static final int VERSION_41 = 13; public static final int minimum_version = VERSION_30; public static final int current_version = VERSION_40; static AcceptVersions accept_messaging = new AcceptVersions(minimum_version, current_version); @@ -367,6 +368,23 @@ public final class MessagingService extends MessagingServiceMBeanImpl send(message, to, null); } + /** + * Send a message to a given endpoint. This method adheres to the fire and forget + * style messaging. + * + * @param message messages to be sent. + * @param response + */ + public void respond(V response, Message message) + { + send(message.responseWith(response), message.respondTo()); + } + + public void respondWithFailure(RequestFailureReason reason, Message message) + { + send(Message.failureResponse(message.id(), message.expiresAtNanos(), reason), message.respondTo()); + } + public void send(Message message, InetAddressAndPort to, ConnectionType specifyConnection) { if (logger.isTraceEnabled()) diff --git a/src/java/org/apache/cassandra/net/OutboundMessageQueue.java b/src/java/org/apache/cassandra/net/OutboundMessageQueue.java index 860890a7ae..8280055e68 100644 --- a/src/java/org/apache/cassandra/net/OutboundMessageQueue.java +++ b/src/java/org/apache/cassandra/net/OutboundMessageQueue.java @@ -519,7 +519,6 @@ class OutboundMessageQueue } } - //noinspection UnstableApiUsage runner.done.awaitUninterruptibly(); return runner.removed.contains(remove); } diff --git a/src/java/org/apache/cassandra/net/RequestCallback.java b/src/java/org/apache/cassandra/net/RequestCallback.java index 5bbe011fe5..bd14cae1d0 100644 --- a/src/java/org/apache/cassandra/net/RequestCallback.java +++ b/src/java/org/apache/cassandra/net/RequestCallback.java @@ -41,6 +41,12 @@ public interface RequestCallback } /** + * Returns true if the callback handles failure reporting - in which case the remove host will be asked to + * report failures to us in the event of a problem processing the request. + * + * TODO: this is an error prone method, and we should be handling failures everywhere + * so we should probably just start doing that, and remove this method + * * @return true if the callback should be invoked on failure */ default boolean invokeOnFailure() @@ -56,4 +62,5 @@ public interface RequestCallback { return false; } + } diff --git a/src/java/org/apache/cassandra/serializers/TimeUUIDSerializer.java b/src/java/org/apache/cassandra/net/RequestCallbackWithFailure.java similarity index 55% rename from src/java/org/apache/cassandra/serializers/TimeUUIDSerializer.java rename to src/java/org/apache/cassandra/net/RequestCallbackWithFailure.java index 9cde1dd487..685797abeb 100644 --- a/src/java/org/apache/cassandra/serializers/TimeUUIDSerializer.java +++ b/src/java/org/apache/cassandra/net/RequestCallbackWithFailure.java @@ -16,23 +16,23 @@ * limitations under the License. */ -package org.apache.cassandra.serializers; +package org.apache.cassandra.net; -import org.apache.cassandra.db.marshal.ValueAccessor; +import org.apache.cassandra.exceptions.RequestFailureReason; +import org.apache.cassandra.locator.InetAddressAndPort; -public class TimeUUIDSerializer extends UUIDSerializer +public interface RequestCallbackWithFailure extends RequestCallback { - public static final TimeUUIDSerializer instance = new TimeUUIDSerializer(); + /** + * Called when there is an exception on the remote node or timeout happens + */ + void onFailure(InetAddressAndPort from, RequestFailureReason failureReason); - public void validate(V value, ValueAccessor accessor) throws MarshalException + /** + * @return true if the callback should be invoked on failure + */ + default boolean invokeOnFailure() { - super.validate(value, accessor); - // Super class only validates the Time UUID - // version is bits 4-7 of byte 6. - if (!accessor.isEmpty(value)) - { - if ((accessor.getByte(value, 6) & 0xf0) != 0x10) - throw new MarshalException("Invalid version for TimeUUID type."); - } + return true; } } diff --git a/src/java/org/apache/cassandra/net/RequestCallbacks.java b/src/java/org/apache/cassandra/net/RequestCallbacks.java index fa1a03e994..4409a704e3 100644 --- a/src/java/org/apache/cassandra/net/RequestCallbacks.java +++ b/src/java/org/apache/cassandra/net/RequestCallbacks.java @@ -26,6 +26,7 @@ import javax.annotation.Nullable; import com.google.common.annotations.VisibleForTesting; +import org.apache.cassandra.concurrent.ExecutorFactory; import org.apache.cassandra.concurrent.ScheduledExecutorPlus; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -47,6 +48,7 @@ import static java.lang.String.format; import static java.util.concurrent.TimeUnit.MILLISECONDS; import static java.util.concurrent.TimeUnit.NANOSECONDS; import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory; +import static org.apache.cassandra.concurrent.ExecutorFactory.SimulatorSemantics.DISCARD; import static org.apache.cassandra.concurrent.Stage.INTERNAL_RESPONSE; import static org.apache.cassandra.utils.Clock.Global.nanoTime; import static org.apache.cassandra.utils.MonotonicClock.Global.preciseTime; @@ -65,14 +67,14 @@ public class RequestCallbacks implements OutboundMessageCallbacks private static final Logger logger = LoggerFactory.getLogger(RequestCallbacks.class); private final MessagingService messagingService; - private final ScheduledExecutorPlus executor = executorFactory().scheduled("Callback-Map-Reaper"); + private final ScheduledExecutorPlus executor = executorFactory().scheduled("Callback-Map-Reaper", DISCARD); private final ConcurrentMap callbacks = new ConcurrentHashMap<>(); RequestCallbacks(MessagingService messagingService) { this.messagingService = messagingService; - long expirationInterval = DatabaseDescriptor.getMinRpcTimeout(NANOSECONDS) / 2; + long expirationInterval = defaultExpirationInterval(); executor.scheduleWithFixedDelay(this::expire, expirationInterval, expirationInterval, NANOSECONDS); } @@ -101,12 +103,11 @@ public class RequestCallbacks implements OutboundMessageCallbacks void addWithExpiration(RequestCallback cb, Message message, InetAddressAndPort to) { // mutations need to call the overload with a ConsistencyLevel - assert message.verb() != Verb.MUTATION_REQ && message.verb() != Verb.COUNTER_MUTATION_REQ && message.verb() != Verb.PAXOS_COMMIT_REQ; + assert message.verb() != Verb.MUTATION_REQ && message.verb() != Verb.COUNTER_MUTATION_REQ; CallbackInfo previous = callbacks.put(key(message.id(), to), new CallbackInfo(message, to, cb)); assert previous == null : format("Callback already exists for id %d/%s! (%s)", message.id(), to, previous); } - // FIXME: shouldn't need a special overload for writes; hinting should be part of AbstractWriteResponseHandler public void addWithExpiration(AbstractWriteResponseHandler cb, Message message, Replica to, @@ -171,14 +172,6 @@ public class RequestCallbacks implements OutboundMessageCallbacks if (info.invokeOnFailure()) INTERNAL_RESPONSE.submit(() -> info.callback.onFailure(info.peer, RequestFailureReason.TIMEOUT)); - - // FIXME: this has never belonged here, should be part of onFailure() in AbstractWriteResponseHandler - if (info.shouldHint()) - { - WriteCallbackInfo writeCallbackInfo = ((WriteCallbackInfo) info); - Mutation mutation = writeCallbackInfo.mutation(); - StorageProxy.submitHint(mutation, writeCallbackInfo.getReplica(), null); - } } void shutdownNow(boolean expireCallbacks) @@ -378,4 +371,9 @@ public class RequestCallbacks implements OutboundMessageCallbacks if (null != forwardTo) forwardTo.forEach(this::removeAndExpire); } + + public static long defaultExpirationInterval() + { + return DatabaseDescriptor.getMinRpcTimeout(NANOSECONDS) / 2; + } } diff --git a/src/java/org/apache/cassandra/net/Verb.java b/src/java/org/apache/cassandra/net/Verb.java index bcd3070f0d..a43372e22d 100644 --- a/src/java/org/apache/cassandra/net/Verb.java +++ b/src/java/org/apache/cassandra/net/Verb.java @@ -74,14 +74,26 @@ import org.apache.cassandra.repair.messages.ValidationRequest; import org.apache.cassandra.schema.SchemaPullVerbHandler; import org.apache.cassandra.schema.SchemaPushVerbHandler; import org.apache.cassandra.schema.SchemaVersionVerbHandler; +import org.apache.cassandra.service.paxos.PaxosCommit; +import org.apache.cassandra.service.paxos.PaxosCommitAndPrepare; +import org.apache.cassandra.service.paxos.PaxosPrepare; +import org.apache.cassandra.service.paxos.PaxosPrepareRefresh; +import org.apache.cassandra.service.paxos.PaxosPropose; +import org.apache.cassandra.service.paxos.PaxosRepair; +import org.apache.cassandra.service.paxos.cleanup.PaxosCleanupHistory; +import org.apache.cassandra.service.paxos.cleanup.PaxosCleanupRequest; +import org.apache.cassandra.service.paxos.cleanup.PaxosCleanupResponse; +import org.apache.cassandra.service.paxos.cleanup.PaxosCleanupComplete; +import org.apache.cassandra.service.paxos.cleanup.PaxosStartPrepareCleanup; +import org.apache.cassandra.service.paxos.cleanup.PaxosFinishPrepareCleanup; import org.apache.cassandra.utils.BooleanSerializer; import org.apache.cassandra.service.EchoVerbHandler; import org.apache.cassandra.service.SnapshotVerbHandler; import org.apache.cassandra.service.paxos.Commit; -import org.apache.cassandra.service.paxos.CommitVerbHandler; +import org.apache.cassandra.service.paxos.Commit.Agreed; import org.apache.cassandra.service.paxos.PrepareResponse; -import org.apache.cassandra.service.paxos.PrepareVerbHandler; -import org.apache.cassandra.service.paxos.ProposeVerbHandler; +import org.apache.cassandra.service.paxos.v1.PrepareVerbHandler; +import org.apache.cassandra.service.paxos.v1.ProposeVerbHandler; import org.apache.cassandra.streaming.ReplicationDoneVerbHandler; import org.apache.cassandra.utils.UUIDSerializer; @@ -113,7 +125,7 @@ public enum Verb PAXOS_PROPOSE_RSP (94, P2, writeTimeout, REQUEST_RESPONSE, () -> BooleanSerializer.serializer, () -> ResponseVerbHandler.instance ), PAXOS_PROPOSE_REQ (34, P2, writeTimeout, MUTATION, () -> Commit.serializer, () -> ProposeVerbHandler.instance, PAXOS_PROPOSE_RSP ), PAXOS_COMMIT_RSP (95, P2, writeTimeout, REQUEST_RESPONSE, () -> NoPayload.serializer, () -> ResponseVerbHandler.instance ), - PAXOS_COMMIT_REQ (35, P2, writeTimeout, MUTATION, () -> Commit.serializer, () -> CommitVerbHandler.instance, PAXOS_COMMIT_RSP ), + PAXOS_COMMIT_REQ (35, P2, writeTimeout, MUTATION, () -> Agreed.serializer, () -> PaxosCommit.requestHandler, PAXOS_COMMIT_RSP ), TRUNCATE_RSP (79, P0, truncateTimeout, REQUEST_RESPONSE, () -> TruncateResponse.serializer, () -> ResponseVerbHandler.instance ), TRUNCATE_REQ (19, P0, truncateTimeout, MUTATION, () -> TruncateRequest.serializer, () -> TruncateVerbHandler.instance, TRUNCATE_RSP ), @@ -167,12 +179,34 @@ public enum Verb SNAPSHOT_RSP (87, P0, rpcTimeout, MISC, () -> NoPayload.serializer, () -> ResponseVerbHandler.instance ), SNAPSHOT_REQ (27, P0, rpcTimeout, MISC, () -> SnapshotCommand.serializer, () -> SnapshotVerbHandler.instance, SNAPSHOT_RSP ), + PAXOS2_COMMIT_REMOTE_REQ (38, P2, writeTimeout, MUTATION, () -> Mutation.serializer, () -> MutationVerbHandler.instance, MUTATION_RSP ), + PAXOS2_COMMIT_REMOTE_RSP (39, P2, writeTimeout, REQUEST_RESPONSE, () -> NoPayload.serializer, () -> ResponseVerbHandler.instance ), + PAXOS2_PREPARE_RSP (50, P2, writeTimeout, REQUEST_RESPONSE, () -> PaxosPrepare.responseSerializer, () -> ResponseVerbHandler.instance ), + PAXOS2_PREPARE_REQ (40, P2, writeTimeout, MUTATION, () -> PaxosPrepare.requestSerializer, () -> PaxosPrepare.requestHandler, PAXOS2_PREPARE_RSP ), + PAXOS2_PREPARE_REFRESH_RSP (51, P2, writeTimeout, REQUEST_RESPONSE, () -> PaxosPrepareRefresh.responseSerializer, () -> ResponseVerbHandler.instance ), + PAXOS2_PREPARE_REFRESH_REQ (41, P2, writeTimeout, MUTATION, () -> PaxosPrepareRefresh.requestSerializer, () -> PaxosPrepareRefresh.requestHandler, PAXOS2_PREPARE_REFRESH_RSP ), + PAXOS2_PROPOSE_RSP (52, P2, writeTimeout, REQUEST_RESPONSE, () -> PaxosPropose.responseSerializer, () -> ResponseVerbHandler.instance ), + PAXOS2_PROPOSE_REQ (42, P2, writeTimeout, MUTATION, () -> PaxosPropose.requestSerializer, () -> PaxosPropose.requestHandler, PAXOS2_PROPOSE_RSP ), + PAXOS2_COMMIT_AND_PREPARE_RSP (53, P2, writeTimeout, REQUEST_RESPONSE, () -> PaxosPrepare.responseSerializer, () -> ResponseVerbHandler.instance ), + PAXOS2_COMMIT_AND_PREPARE_REQ (43, P2, writeTimeout, MUTATION, () -> PaxosCommitAndPrepare.requestSerializer, () -> PaxosCommitAndPrepare.requestHandler, PAXOS2_COMMIT_AND_PREPARE_RSP ), + PAXOS2_REPAIR_RSP (54, P2, writeTimeout, PAXOS_REPAIR, () -> PaxosRepair.responseSerializer, () -> ResponseVerbHandler.instance ), + PAXOS2_REPAIR_REQ (44, P2, writeTimeout, PAXOS_REPAIR, () -> PaxosRepair.requestSerializer, () -> PaxosRepair.requestHandler, PAXOS2_REPAIR_RSP ), + PAXOS2_CLEANUP_START_PREPARE_RSP (55, P2, rpcTimeout, PAXOS_REPAIR, () -> PaxosCleanupHistory.serializer, () -> ResponseVerbHandler.instance ), + PAXOS2_CLEANUP_START_PREPARE_REQ (45, P2, rpcTimeout, PAXOS_REPAIR, () -> PaxosStartPrepareCleanup.serializer, () -> PaxosStartPrepareCleanup.verbHandler, PAXOS2_CLEANUP_START_PREPARE_RSP ), + PAXOS2_CLEANUP_RSP (56, P2, rpcTimeout, PAXOS_REPAIR, () -> NoPayload.serializer, () -> ResponseVerbHandler.instance ), + PAXOS2_CLEANUP_REQ (46, P2, rpcTimeout, PAXOS_REPAIR, () -> PaxosCleanupRequest.serializer, () -> PaxosCleanupRequest.verbHandler, PAXOS2_CLEANUP_RSP ), + PAXOS2_CLEANUP_RSP2 (57, P2, rpcTimeout, PAXOS_REPAIR, () -> PaxosCleanupResponse.serializer, () -> PaxosCleanupResponse.verbHandler ), + PAXOS2_CLEANUP_FINISH_PREPARE_RSP(58, P2, rpcTimeout, PAXOS_REPAIR, () -> NoPayload.serializer, () -> ResponseVerbHandler.instance ), + PAXOS2_CLEANUP_FINISH_PREPARE_REQ(47, P2, rpcTimeout, IMMEDIATE, () -> PaxosCleanupHistory.serializer, () -> PaxosFinishPrepareCleanup.verbHandler, PAXOS2_CLEANUP_FINISH_PREPARE_RSP), + PAXOS2_CLEANUP_COMPLETE_RSP (59, P2, rpcTimeout, PAXOS_REPAIR, () -> NoPayload.serializer, () -> ResponseVerbHandler.instance ), + PAXOS2_CLEANUP_COMPLETE_REQ (48, P2, rpcTimeout, PAXOS_REPAIR, () -> PaxosCleanupComplete.serializer, () -> PaxosCleanupComplete.verbHandler, PAXOS2_CLEANUP_COMPLETE_RSP ), + // generic failure response FAILURE_RSP (99, P0, noTimeout, REQUEST_RESPONSE, () -> RequestFailureReason.serializer, () -> ResponseVerbHandler.instance ), // dummy verbs _TRACE (30, P1, rpcTimeout, TRACING, () -> NoPayload.serializer, () -> null ), - _SAMPLE (42, P1, rpcTimeout, INTERNAL_RESPONSE, () -> NoPayload.serializer, () -> null ), + _SAMPLE (49, P1, rpcTimeout, INTERNAL_RESPONSE, () -> NoPayload.serializer, () -> null ), _TEST_1 (10, P0, writeTimeout, IMMEDIATE, () -> NoPayload.serializer, () -> null ), _TEST_2 (11, P1, rpcTimeout, IMMEDIATE, () -> NoPayload.serializer, () -> null ), @@ -186,7 +220,6 @@ public enum Verb // CUSTOM VERBS UNUSED_CUSTOM_VERB (CUSTOM, 0, P1, rpcTimeout, INTERNAL_RESPONSE, () -> null, () -> null ), - ; public static final List VERBS = ImmutableList.copyOf(Verb.values()); @@ -447,5 +480,5 @@ class VerbTimeouts static final ToLongFunction truncateTimeout = DatabaseDescriptor::getTruncateRpcTimeout; static final ToLongFunction pingTimeout = DatabaseDescriptor::getPingTimeout; static final ToLongFunction longTimeout = units -> Math.max(DatabaseDescriptor.getRpcTimeout(units), units.convert(5L, TimeUnit.MINUTES)); - static final ToLongFunction noTimeout = units -> { throw new IllegalStateException(); }; + static final ToLongFunction noTimeout = units -> { throw new IllegalStateException(); }; } diff --git a/src/java/org/apache/cassandra/repair/AbstractRepairTask.java b/src/java/org/apache/cassandra/repair/AbstractRepairTask.java index 2446c043fe..d2a6f1a786 100644 --- a/src/java/org/apache/cassandra/repair/AbstractRepairTask.java +++ b/src/java/org/apache/cassandra/repair/AbstractRepairTask.java @@ -71,6 +71,8 @@ public abstract class AbstractRepairTask implements RepairTask options.isPullRepair(), options.getPreviewKind(), options.optimiseStreams(), + options.repairPaxos(), + options.paxosOnly(), executor, cfnames); if (session == null) diff --git a/src/java/org/apache/cassandra/repair/RepairJob.java b/src/java/org/apache/cassandra/repair/RepairJob.java index c78cc73c0c..33234e2227 100644 --- a/src/java/org/apache/cassandra/repair/RepairJob.java +++ b/src/java/org/apache/cassandra/repair/RepairJob.java @@ -18,8 +18,8 @@ package org.apache.cassandra.repair; import java.util.*; -import java.util.function.Predicate; import java.util.function.Function; +import java.util.function.Predicate; import java.util.stream.Collectors; import com.google.common.annotations.VisibleForTesting; @@ -27,6 +27,8 @@ import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; import com.google.common.util.concurrent.*; import org.apache.cassandra.concurrent.ExecutorPlus; +import org.apache.cassandra.schema.Schema; +import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.utils.concurrent.AsyncFuture; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -44,6 +46,7 @@ import org.apache.cassandra.service.ActiveRepairService; import org.apache.cassandra.streaming.PreviewKind; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; +import org.apache.cassandra.service.paxos.cleanup.PaxosCleanup; import org.apache.cassandra.tracing.Tracing; import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.MerkleTrees; @@ -52,6 +55,8 @@ import org.apache.cassandra.utils.concurrent.Future; import org.apache.cassandra.utils.concurrent.FutureCombiner; import org.apache.cassandra.utils.concurrent.ImmediateFuture; +import static org.apache.cassandra.config.DatabaseDescriptor.paxosRepairEnabled; +import static org.apache.cassandra.service.paxos.Paxos.useV2; import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis; /** @@ -71,8 +76,7 @@ public class RepairJob extends AsyncFuture implements Runnable /** * Create repair job to run on specific columnfamily - * - * @param session RepairSession that this RepairJob belongs + * @param session RepairSession that this RepairJob belongs * @param columnFamily name of the ColumnFamily to repair */ public RepairJob(RepairSession session, String columnFamily) @@ -112,40 +116,77 @@ public class RepairJob extends AsyncFuture implements Runnable allEndpoints.add(FBUtilities.getBroadcastAddressAndPort()); Future> treeResponses; + Future paxosRepair; + if (paxosRepairEnabled() && ((useV2() && session.repairPaxos) || session.paxosOnly)) + { + logger.info("{} {}.{} starting paxos repair", session.previewKind.logPrefix(session.getId()), desc.keyspace, desc.columnFamily); + TableMetadata metadata = Schema.instance.getTableMetadata(desc.keyspace, desc.columnFamily); + paxosRepair = PaxosCleanup.cleanup(allEndpoints, metadata, desc.ranges, session.commonRange.hasSkippedReplicas, taskExecutor); + } + else + { + logger.info("{} {}.{} not running paxos repair", session.previewKind.logPrefix(session.getId()), desc.keyspace, desc.columnFamily); + paxosRepair = ImmediateFuture.success(null); + } + + if (session.paxosOnly) + { + paxosRepair.addCallback(new FutureCallback() + { + public void onSuccess(Void v) + { + logger.info("{} {}.{} paxos repair completed", session.previewKind.logPrefix(session.getId()), desc.keyspace, desc.columnFamily); + trySuccess(new RepairResult(desc, Collections.emptyList())); + } + + /** + * Snapshot, validation and sync failures are all handled here + */ + public void onFailure(Throwable t) + { + logger.warn("{} {}.{} paxos repair failed", session.previewKind.logPrefix(session.getId()), desc.keyspace, desc.columnFamily); + tryFailure(t); + } + }, taskExecutor); + return; + } + // Create a snapshot at all nodes unless we're using pure parallel repairs if (parallelismDegree != RepairParallelism.PARALLEL) { - Future> allSnapshotTasks; + Future allSnapshotTasks; if (session.isIncremental) { // consistent repair does it's own "snapshotting" - allSnapshotTasks = ImmediateFuture.success(allEndpoints); + allSnapshotTasks = paxosRepair.map(input -> allEndpoints); } else { // Request snapshot to all replica - List> snapshotTasks = new ArrayList<>(allEndpoints.size()); - for (InetAddressAndPort endpoint : allEndpoints) - { - SnapshotTask snapshotTask = new SnapshotTask(desc, endpoint); - snapshotTasks.add(snapshotTask); - taskExecutor.execute(snapshotTask); - } - allSnapshotTasks = FutureCombiner.allOf(snapshotTasks); + allSnapshotTasks = paxosRepair.flatMap(input -> { + List> snapshotTasks = new ArrayList<>(allEndpoints.size()); + for (InetAddressAndPort endpoint : allEndpoints) + { + SnapshotTask snapshotTask = new SnapshotTask(desc, endpoint); + snapshotTasks.add(snapshotTask); + taskExecutor.execute(snapshotTask); + } + return FutureCombiner.allOf(snapshotTasks); + }); } // When all snapshot complete, send validation requests treeResponses = allSnapshotTasks.flatMap(endpoints -> { if (parallelismDegree == RepairParallelism.SEQUENTIAL) - return sendSequentialValidationRequest(endpoints); + return sendSequentialValidationRequest(allEndpoints); else - return sendDCAwareValidationRequest(endpoints); - }, taskExecutor); + return sendDCAwareValidationRequest(allEndpoints); + }, taskExecutor); } else { // If not sequential, just send validation request to all replica - treeResponses = sendValidationRequest(allEndpoints); + treeResponses = paxosRepair.flatMap(input -> sendValidationRequest(allEndpoints)); } // When all validations complete, submit sync tasks diff --git a/src/java/org/apache/cassandra/repair/RepairRunnable.java b/src/java/org/apache/cassandra/repair/RepairRunnable.java index b58d4f67c3..7ad2c499c4 100644 --- a/src/java/org/apache/cassandra/repair/RepairRunnable.java +++ b/src/java/org/apache/cassandra/repair/RepairRunnable.java @@ -73,11 +73,9 @@ import org.apache.cassandra.tracing.TraceKeyspace; import org.apache.cassandra.tracing.TraceState; import org.apache.cassandra.tracing.Tracing; import org.apache.cassandra.transport.messages.ResultMessage; -import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.Throwables; import org.apache.cassandra.utils.WrappedRunnable; -import org.apache.cassandra.utils.concurrent.Future; import org.apache.cassandra.utils.progress.ProgressEvent; import org.apache.cassandra.utils.progress.ProgressEventNotifier; import org.apache.cassandra.utils.progress.ProgressEventType; @@ -225,7 +223,7 @@ public class RepairRunnable implements Runnable, ProgressEventNotifier, RepairNo long durationMillis = currentTimeMillis() - creationTimeMillis; if (msg == null) { - String duration = DurationFormatUtils.formatDurationWords(durationMillis, true, true); + String duration = DurationFormatUtils.formatDurationWords(Math.max(0, durationMillis), true, true); msg = String.format("Repair command #%d finished in %s", cmd, duration); } diff --git a/src/java/org/apache/cassandra/repair/RepairSession.java b/src/java/org/apache/cassandra/repair/RepairSession.java index 4bce9ea74d..bfd6a7e049 100644 --- a/src/java/org/apache/cassandra/repair/RepairSession.java +++ b/src/java/org/apache/cassandra/repair/RepairSession.java @@ -69,8 +69,11 @@ import org.apache.cassandra.utils.concurrent.Future; * of column families. For each of the column family to repair, RepairSession * creates a {@link RepairJob} that handles the repair of that CF. * - * A given RepairJob has the 2 main phases: + * A given RepairJob has the 3 main phases: *

    + *
  1. + * Paxos repair: unfinished paxos operations in the range/keyspace/table are first completed + *
    2. *
  2. Validation phase: the job requests merkle trees from each of the replica involves * ({@link org.apache.cassandra.repair.ValidationTask}) and waits until all trees are received (in * validationComplete()). @@ -115,6 +118,8 @@ public class RepairSession extends AsyncFuture implements I public final CommonRange commonRange; public final boolean isIncremental; public final PreviewKind previewKind; + public final boolean repairPaxos; + public final boolean paxosOnly; private final AtomicBoolean isFailed = new AtomicBoolean(false); @@ -137,6 +142,8 @@ public class RepairSession extends AsyncFuture implements I * @param keyspace name of keyspace * @param parallelismDegree specifies the degree of parallelism when calculating the merkle trees * @param pullRepair true if the repair should be one way (from remote host to this host and only applicable between two hosts--see RepairOption) + * @param repairPaxos true if incomplete paxos operations should be completed as part of repair + * @param paxosOnly true if we should only complete paxos operations, not run a normal repair * @param cfnames names of columnfamilies */ public RepairSession(TimeUUID parentRepairSession, @@ -148,8 +155,12 @@ public class RepairSession extends AsyncFuture implements I boolean pullRepair, PreviewKind previewKind, boolean optimiseStreams, + boolean repairPaxos, + boolean paxosOnly, String... cfnames) { + this.repairPaxos = repairPaxos; + this.paxosOnly = paxosOnly; assert cfnames.length > 0 : "Repairing no column families seems pointless, doesn't it"; this.parentRepairSession = parentRepairSession; @@ -275,7 +286,7 @@ public class RepairSession extends AsyncFuture implements I logger.info("{} parentSessionId = {}: new session: will sync {} on range {} for {}.{}", previewKind.logPrefix(getId()), parentRepairSession, repairedNodes(), commonRange, keyspace, Arrays.toString(cfnames)); Tracing.traceRepair("Syncing range {}", commonRange); - if (!previewKind.isPreview()) + if (!previewKind.isPreview() && !paxosOnly) { SystemDistributedKeyspace.startRepairs(getId(), parentRepairSession, keyspace, cfnames, commonRange); } diff --git a/src/java/org/apache/cassandra/repair/consistent/LocalSessions.java b/src/java/org/apache/cassandra/repair/consistent/LocalSessions.java index bdd8e95220..c9c751ad9d 100644 --- a/src/java/org/apache/cassandra/repair/consistent/LocalSessions.java +++ b/src/java/org/apache/cassandra/repair/consistent/LocalSessions.java @@ -866,7 +866,6 @@ public class LocalSessions }); } -<<<<<<< HEAD /** * Checks for the session state, and sets it to prepared unless it is on a failed state. * Making the checks inside a synchronized block to prevent the session state from diff --git a/src/java/org/apache/cassandra/repair/consistent/admin/CleanupSummary.java b/src/java/org/apache/cassandra/repair/consistent/admin/CleanupSummary.java index 1605fae5c7..2d21debe76 100644 --- a/src/java/org/apache/cassandra/repair/consistent/admin/CleanupSummary.java +++ b/src/java/org/apache/cassandra/repair/consistent/admin/CleanupSummary.java @@ -22,7 +22,6 @@ import java.util.HashMap; import java.util.HashSet; import java.util.Map; import java.util.Set; -import java.util.UUID; import javax.management.openmbean.ArrayType; import javax.management.openmbean.CompositeData; diff --git a/src/java/org/apache/cassandra/repair/messages/PrepareMessage.java b/src/java/org/apache/cassandra/repair/messages/PrepareMessage.java index 49380d73c6..ac8bc7dbe0 100644 --- a/src/java/org/apache/cassandra/repair/messages/PrepareMessage.java +++ b/src/java/org/apache/cassandra/repair/messages/PrepareMessage.java @@ -22,7 +22,6 @@ import java.util.ArrayList; import java.util.Collection; import java.util.List; import java.util.Objects; -import java.util.UUID; import com.google.common.base.Preconditions; diff --git a/src/java/org/apache/cassandra/repair/messages/RepairOption.java b/src/java/org/apache/cassandra/repair/messages/RepairOption.java index 6a0568e743..6bb7fdb61f 100644 --- a/src/java/org/apache/cassandra/repair/messages/RepairOption.java +++ b/src/java/org/apache/cassandra/repair/messages/RepairOption.java @@ -20,6 +20,7 @@ package org.apache.cassandra.repair.messages; import java.util.*; import com.google.common.base.Joiner; +import com.google.common.base.Preconditions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -50,6 +51,8 @@ public class RepairOption public static final String PREVIEW = "previewKind"; public static final String OPTIMISE_STREAMS_KEY = "optimiseStreams"; public static final String IGNORE_UNREPLICATED_KS = "ignoreUnreplicatedKeyspaces"; + public static final String REPAIR_PAXOS_KEY = "repairPaxos"; + public static final String PAXOS_ONLY_KEY = "paxosOnly"; // we don't want to push nodes too much for repair public static final int MAX_JOB_THREADS = 4; @@ -179,6 +182,14 @@ public class RepairOption boolean force = Boolean.parseBoolean(options.get(FORCE_REPAIR_KEY)); boolean pullRepair = Boolean.parseBoolean(options.get(PULL_REPAIR_KEY)); boolean ignoreUnreplicatedKeyspaces = Boolean.parseBoolean(options.get(IGNORE_UNREPLICATED_KS)); + boolean repairPaxos = Boolean.parseBoolean(options.get(REPAIR_PAXOS_KEY)); + boolean paxosOnly = Boolean.parseBoolean(options.get(PAXOS_ONLY_KEY)); + + if (previewKind != PreviewKind.NONE) + { + Preconditions.checkArgument(!repairPaxos, "repairPaxos must be set to false for preview repairs"); + Preconditions.checkArgument(!paxosOnly, "paxosOnly must be set to false for preview repairs"); + } int jobThreads = 1; if (options.containsKey(JOB_THREADS_KEY)) @@ -195,7 +206,7 @@ public class RepairOption boolean asymmetricSyncing = Boolean.parseBoolean(options.get(OPTIMISE_STREAMS_KEY)); - RepairOption option = new RepairOption(parallelism, primaryRange, incremental, trace, jobThreads, ranges, !ranges.isEmpty(), pullRepair, force, previewKind, asymmetricSyncing, ignoreUnreplicatedKeyspaces); + RepairOption option = new RepairOption(parallelism, primaryRange, incremental, trace, jobThreads, ranges, !ranges.isEmpty(), pullRepair, force, previewKind, asymmetricSyncing, ignoreUnreplicatedKeyspaces, repairPaxos, paxosOnly); // data centers String dataCentersStr = options.get(DATACENTERS_KEY); @@ -275,13 +286,15 @@ public class RepairOption private final PreviewKind previewKind; private final boolean optimiseStreams; private final boolean ignoreUnreplicatedKeyspaces; + private final boolean repairPaxos; + private final boolean paxosOnly; private final Collection columnFamilies = new HashSet<>(); private final Collection dataCenters = new HashSet<>(); private final Collection hosts = new HashSet<>(); private final Collection> ranges = new HashSet<>(); - public RepairOption(RepairParallelism parallelism, boolean primaryRange, boolean incremental, boolean trace, int jobThreads, Collection> ranges, boolean isSubrangeRepair, boolean pullRepair, boolean forceRepair, PreviewKind previewKind, boolean optimiseStreams, boolean ignoreUnreplicatedKeyspaces) + public RepairOption(RepairParallelism parallelism, boolean primaryRange, boolean incremental, boolean trace, int jobThreads, Collection> ranges, boolean isSubrangeRepair, boolean pullRepair, boolean forceRepair, PreviewKind previewKind, boolean optimiseStreams, boolean ignoreUnreplicatedKeyspaces, boolean repairPaxos, boolean paxosOnly) { this.parallelism = parallelism; @@ -296,6 +309,8 @@ public class RepairOption this.previewKind = previewKind; this.optimiseStreams = optimiseStreams; this.ignoreUnreplicatedKeyspaces = ignoreUnreplicatedKeyspaces; + this.repairPaxos = repairPaxos; + this.paxosOnly = paxosOnly; } public RepairParallelism getParallelism() @@ -403,6 +418,16 @@ public class RepairOption return ignoreUnreplicatedKeyspaces; } + public boolean repairPaxos() + { + return repairPaxos; + } + + public boolean paxosOnly() + { + return paxosOnly; + } + @Override public String toString() { @@ -420,6 +445,8 @@ public class RepairOption ", force repair: " + forceRepair + ", optimise streams: "+ optimiseStreams() + ", ignore unreplicated keyspaces: "+ ignoreUnreplicatedKeyspaces + + ", repairPaxos: " + repairPaxos + + ", paxosOnly: " + paxosOnly + ')'; } @@ -440,6 +467,8 @@ public class RepairOption options.put(FORCE_REPAIR_KEY, Boolean.toString(forceRepair)); options.put(PREVIEW, previewKind.toString()); options.put(OPTIMISE_STREAMS_KEY, Boolean.toString(optimiseStreams)); + options.put(REPAIR_PAXOS_KEY, Boolean.toString(repairPaxos)); + options.put(PAXOS_ONLY_KEY, Boolean.toString(paxosOnly)); return options; } } diff --git a/src/java/org/apache/cassandra/schema/MigrationCoordinator.java b/src/java/org/apache/cassandra/schema/MigrationCoordinator.java index 3050886417..399f917cb7 100644 --- a/src/java/org/apache/cassandra/schema/MigrationCoordinator.java +++ b/src/java/org/apache/cassandra/schema/MigrationCoordinator.java @@ -40,6 +40,7 @@ import com.google.common.collect.ImmutableSet; import com.google.common.collect.Sets; import org.apache.cassandra.concurrent.FutureTask; import org.apache.cassandra.config.CassandraRelevantProperties; +import org.apache.cassandra.utils.Simulate; import org.apache.cassandra.utils.concurrent.Future; import org.apache.cassandra.utils.concurrent.ImmediateFuture; import org.slf4j.Logger; @@ -63,8 +64,10 @@ import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.concurrent.WaitQueue; import static org.apache.cassandra.utils.Clock.Global.nanoTime; +import static org.apache.cassandra.utils.Simulate.With.MONITORS; import static org.apache.cassandra.utils.concurrent.WaitQueue.newWaitQueue; +@Simulate(with = MONITORS) public class MigrationCoordinator { private static final Logger logger = LoggerFactory.getLogger(MigrationCoordinator.class); diff --git a/src/java/org/apache/cassandra/security/DisableSslContextFactory.java b/src/java/org/apache/cassandra/security/DisableSslContextFactory.java new file mode 100644 index 0000000000..9dab062f0b --- /dev/null +++ b/src/java/org/apache/cassandra/security/DisableSslContextFactory.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.security; + +import javax.net.ssl.KeyManagerFactory; +import javax.net.ssl.SSLException; +import javax.net.ssl.TrustManagerFactory; + +public class DisableSslContextFactory extends AbstractSslContextFactory +{ + @Override + protected KeyManagerFactory buildKeyManagerFactory() throws SSLException + { + throw new UnsupportedOperationException(); + } + + @Override + protected TrustManagerFactory buildTrustManagerFactory() throws SSLException + { + throw new UnsupportedOperationException(); + } + + @Override + public boolean hasKeystore() + { + return false; + } + + @Override + public void initHotReloading() throws SSLException + { + } + + @Override + public boolean shouldReload() + { + return false; + } +} diff --git a/src/java/org/apache/cassandra/service/AbstractWriteResponseHandler.java b/src/java/org/apache/cassandra/service/AbstractWriteResponseHandler.java index b249ca69c6..4d75f19bca 100644 --- a/src/java/org/apache/cassandra/service/AbstractWriteResponseHandler.java +++ b/src/java/org/apache/cassandra/service/AbstractWriteResponseHandler.java @@ -22,10 +22,16 @@ import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; +import java.util.function.Supplier; + +import javax.annotation.Nullable; import org.apache.cassandra.db.ConsistencyLevel; +import org.apache.cassandra.db.Mutation; import org.apache.cassandra.locator.EndpointsForToken; +import org.apache.cassandra.locator.ReplicaPlan; +import org.apache.cassandra.locator.ReplicaPlan.ForWrite; import org.apache.cassandra.utils.concurrent.Condition; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -50,7 +56,6 @@ import static java.util.stream.Collectors.toList; import static org.apache.cassandra.config.DatabaseDescriptor.getCounterWriteRpcTimeout; import static org.apache.cassandra.config.DatabaseDescriptor.getWriteRpcTimeout; import static org.apache.cassandra.db.WriteType.COUNTER; -import static org.apache.cassandra.locator.ReplicaPlan.ForTokenWrite; import static org.apache.cassandra.schema.Schema.instance; import static org.apache.cassandra.service.StorageProxy.WritePerformer; import static org.apache.cassandra.utils.concurrent.Condition.newOneTimeCondition; @@ -64,7 +69,7 @@ public abstract class AbstractWriteResponseHandler implements RequestCallback //Count down until all responses and expirations have occured before deciding whether the ideal CL was reached. private AtomicInteger responsesAndExpirations; private final Condition condition = newOneTimeCondition(); - protected final ForTokenWrite replicaPlan; + protected final ReplicaPlan.ForWrite replicaPlan; protected final Runnable callback; protected final WriteType writeType; @@ -73,6 +78,7 @@ public abstract class AbstractWriteResponseHandler implements RequestCallback private volatile int failures = 0; private final Map failureReasonByEndpoint; private final long queryStartNanoTime; + private @Nullable final Supplier hintOnFailure; /** * Delegate to another WriteResponseHandler or possibly this one to track if the ideal consistency level was reached. @@ -89,16 +95,16 @@ public abstract class AbstractWriteResponseHandler implements RequestCallback /** * @param callback A callback to be called when the write is successful. + * @param hintOnFailure * @param queryStartNanoTime */ - protected AbstractWriteResponseHandler(ForTokenWrite replicaPlan, - Runnable callback, - WriteType writeType, - long queryStartNanoTime) + protected AbstractWriteResponseHandler(ForWrite replicaPlan, Runnable callback, WriteType writeType, + Supplier hintOnFailure, long queryStartNanoTime) { this.replicaPlan = replicaPlan; this.callback = callback; this.writeType = writeType; + this.hintOnFailure = hintOnFailure; this.failureReasonByEndpoint = new ConcurrentHashMap<>(); this.queryStartNanoTime = queryStartNanoTime; } @@ -208,7 +214,7 @@ public abstract class AbstractWriteResponseHandler implements RequestCallback { // During bootstrap, we have to include the pending endpoints or we may fail the consistency level // guarantees (see #833) - return replicaPlan.blockFor(); + return replicaPlan.writeQuorum(); } /** @@ -274,6 +280,9 @@ public abstract class AbstractWriteResponseHandler implements RequestCallback if (blockFor() + n > candidateReplicaCount()) signal(); + + if (hintOnFailure != null) + StorageProxy.submitHint(hintOnFailure.get(), replicaPlan.lookup(from), null); } @Override @@ -332,7 +341,7 @@ public abstract class AbstractWriteResponseHandler implements RequestCallback for (ColumnFamilyStore cf : cfs) cf.metric.additionalWrites.inc(); - writePerformer.apply(mutation, replicaPlan.withContact(uncontacted), + writePerformer.apply(mutation, replicaPlan.withContacts(uncontacted), (AbstractWriteResponseHandler) this, localDC); } diff --git a/src/java/org/apache/cassandra/service/ActiveRepairService.java b/src/java/org/apache/cassandra/service/ActiveRepairService.java index da8652943f..9269e0fc19 100644 --- a/src/java/org/apache/cassandra/service/ActiveRepairService.java +++ b/src/java/org/apache/cassandra/service/ActiveRepairService.java @@ -24,6 +24,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicBoolean; import javax.management.openmbean.CompositeData; import java.util.concurrent.atomic.AtomicInteger; @@ -36,15 +37,23 @@ import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; import com.google.common.collect.Multimap; +import com.google.common.util.concurrent.MoreExecutors; import org.apache.cassandra.concurrent.ExecutorPlus; import org.apache.cassandra.config.Config; import org.apache.cassandra.db.compaction.CompactionManager; +import org.apache.cassandra.locator.AbstractReplicationStrategy; import org.apache.cassandra.locator.EndpointsByRange; import org.apache.cassandra.locator.EndpointsForRange; +import org.apache.cassandra.utils.ExecutorUtils; import org.apache.cassandra.utils.Simulate; +import org.apache.cassandra.locator.EndpointsForToken; +import org.apache.cassandra.schema.Schema; +import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.streaming.PreviewKind; +import org.apache.cassandra.utils.Simulate; import org.apache.cassandra.utils.TimeUUID; import org.apache.cassandra.utils.concurrent.CountDownLatch; import org.slf4j.Logger; @@ -74,6 +83,8 @@ import org.apache.cassandra.net.Message; import org.apache.cassandra.net.MessagingService; import org.apache.cassandra.repair.CommonRange; import org.apache.cassandra.repair.NoSuchRepairSessionException; +import org.apache.cassandra.service.paxos.PaxosRepair; +import org.apache.cassandra.service.paxos.cleanup.PaxosCleanup; import org.apache.cassandra.repair.RepairJobDesc; import org.apache.cassandra.repair.RepairParallelism; import org.apache.cassandra.repair.RepairSession; @@ -91,12 +102,13 @@ import org.apache.cassandra.repair.messages.RepairOption; import org.apache.cassandra.repair.messages.SyncResponse; import org.apache.cassandra.repair.messages.ValidationResponse; import org.apache.cassandra.schema.TableId; -import org.apache.cassandra.streaming.PreviewKind; import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.MBeanWrapper; import org.apache.cassandra.utils.MerkleTrees; import org.apache.cassandra.utils.Pair; import org.apache.cassandra.utils.concurrent.Future; +import org.apache.cassandra.utils.concurrent.FutureCombiner; +import org.apache.cassandra.utils.concurrent.ImmediateFuture; import static com.google.common.collect.Iterables.concat; import static com.google.common.collect.Iterables.transform; @@ -342,9 +354,14 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai boolean pullRepair, PreviewKind previewKind, boolean optimiseStreams, + boolean repairPaxos, + boolean paxosOnly, ExecutorPlus executor, String... cfnames) { + if (repairPaxos && previewKind != PreviewKind.NONE) + throw new IllegalArgumentException("cannot repair paxos in a preview repair"); + if (range.endpoints.isEmpty()) return null; @@ -353,7 +370,7 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai final RepairSession session = new RepairSession(parentRepairSession, nextTimeUUID(), range, keyspace, parallelismDegree, isIncremental, pullRepair, - previewKind, optimiseStreams, cfnames); + previewKind, optimiseStreams, repairPaxos, paxosOnly, cfnames); sessions.put(session.getId(), session); // register listeners @@ -418,7 +435,8 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai } - Pair> getRepairStatus(Integer cmd) + @VisibleForTesting + public Pair> getRepairStatus(Integer cmd) { return repairStatusByCmd.getIfPresent(cmd); } @@ -924,4 +942,74 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai { return parentRepairSessions.size(); } + + public Future repairPaxosForTopologyChange(String ksName, Collection> ranges, String reason) + { + if (!paxosRepairEnabled()) + { + logger.warn("Not running paxos repair for topology change because paxos repair has been disabled"); + return ImmediateFuture.success(null); + } + + if (ranges.isEmpty()) + { + logger.warn("Not running paxos repair for topology change because there are no ranges to repair"); + return ImmediateFuture.success(null); + } + List tables = Lists.newArrayList(Schema.instance.getKeyspaceMetadata(ksName).tables); + List> futures = new ArrayList<>(ranges.size() * tables.size()); + Keyspace keyspace = Keyspace.open(ksName); + AbstractReplicationStrategy replication = keyspace.getReplicationStrategy(); + for (Range range: ranges) + { + for (TableMetadata table : tables) + { + Set endpoints = replication.getNaturalReplicas(range.right).filter(FailureDetector.isReplicaAlive).endpoints(); + if (!PaxosRepair.hasSufficientLiveNodesForTopologyChange(keyspace, range, endpoints)) + { + Set downEndpoints = replication.getNaturalReplicas(range.right).filter(e -> !endpoints.contains(e)).endpoints(); + downEndpoints.removeAll(endpoints); + + throw new RuntimeException(String.format("Insufficient live nodes to repair paxos for %s in %s for %s.\n" + + "There must be enough live nodes to satisfy EACH_QUORUM, but the following nodes are down: %s\n" + + "This check can be skipped by setting either the yaml property skip_paxos_repair_on_topology_change or " + + "the system property cassandra.skip_paxos_repair_on_topology_change to false. The jmx property " + + "StorageService.SkipPaxosRepairOnTopologyChange can also be set to false to temporarily disable without " + + "restarting the node\n" + + "Individual keyspaces can be skipped with the yaml property skip_paxos_repair_on_topology_change_keyspaces, the" + + "system property cassandra.skip_paxos_repair_on_topology_change_keyspaces, or temporarily with the jmx" + + "property StorageService.SkipPaxosRepairOnTopologyChangeKeyspaces\n" + + "Skipping this check can lead to paxos correctness issues", + range, ksName, reason, downEndpoints)); + } + EndpointsForToken pending = StorageService.instance.getTokenMetadata().pendingEndpointsForToken(range.right, ksName); + if (pending.size() > 1 && !Boolean.getBoolean("cassandra.paxos_repair_allow_multiple_pending_unsafe")) + { + throw new RuntimeException(String.format("Cannot begin paxos auto repair for %s in %s.%s, multiple pending endpoints exist for range (%s). " + + "Set -Dcassandra.paxos_repair_allow_multiple_pending_unsafe=true to skip this check", + range, table.keyspace, table.name, pending)); + + } + Future future = PaxosCleanup.cleanup(endpoints, table, Collections.singleton(range), false, repairCommandExecutor()); + futures.add(future); + } + } + + return FutureCombiner.allOf(futures); + } + + public int getPaxosRepairParallelism() + { + return DatabaseDescriptor.getPaxosRepairParallelism(); + } + + public void setPaxosRepairParallelism(int v) + { + DatabaseDescriptor.setPaxosRepairParallelism(v); + } + + public void shutdownNowAndWait(long timeout, TimeUnit unit) throws InterruptedException, TimeoutException + { + ExecutorUtils.shutdownNowAndWait(timeout, unit, clearSnapshotExecutor); + } } diff --git a/src/java/org/apache/cassandra/service/ActiveRepairServiceMBean.java b/src/java/org/apache/cassandra/service/ActiveRepairServiceMBean.java index 3f03602039..009ad562af 100644 --- a/src/java/org/apache/cassandra/service/ActiveRepairServiceMBean.java +++ b/src/java/org/apache/cassandra/service/ActiveRepairServiceMBean.java @@ -55,4 +55,6 @@ public interface ActiveRepairServiceMBean * @return current size of the internal cache holding {@link ActiveRepairService.ParentRepairSession} instances */ int parentRepairSessionsCount(); + public int getPaxosRepairParallelism(); + public void setPaxosRepairParallelism(int v); } diff --git a/src/java/org/apache/cassandra/service/BatchlogResponseHandler.java b/src/java/org/apache/cassandra/service/BatchlogResponseHandler.java index 155f42ddaf..c1c828f75b 100644 --- a/src/java/org/apache/cassandra/service/BatchlogResponseHandler.java +++ b/src/java/org/apache/cassandra/service/BatchlogResponseHandler.java @@ -36,7 +36,7 @@ public class BatchlogResponseHandler extends AbstractWriteResponseHandler public BatchlogResponseHandler(AbstractWriteResponseHandler wrapped, int requiredBeforeFinish, BatchlogCleanup cleanup, long queryStartNanoTime) { - super(wrapped.replicaPlan, wrapped.callback, wrapped.writeType, queryStartNanoTime); + super(wrapped.replicaPlan, wrapped.callback, wrapped.writeType, null, queryStartNanoTime); this.wrapped = wrapped; this.requiredBeforeFinish = requiredBeforeFinish; this.cleanup = cleanup; diff --git a/src/java/org/apache/cassandra/service/CASRequest.java b/src/java/org/apache/cassandra/service/CASRequest.java index 9197ef00c0..19966c883c 100644 --- a/src/java/org/apache/cassandra/service/CASRequest.java +++ b/src/java/org/apache/cassandra/service/CASRequest.java @@ -17,11 +17,11 @@ */ package org.apache.cassandra.service; -import org.apache.cassandra.db.SinglePartitionReadQuery; +import org.apache.cassandra.db.SinglePartitionReadCommand; import org.apache.cassandra.db.partitions.FilteredPartition; import org.apache.cassandra.db.partitions.PartitionUpdate; import org.apache.cassandra.exceptions.InvalidRequestException; -import org.apache.cassandra.utils.TimeUUID; +import org.apache.cassandra.service.paxos.Ballot; /** * Abstract the conditions and updates for a CAS operation. @@ -31,7 +31,7 @@ public interface CASRequest /** * The command to use to fetch the value to compare for the CAS. */ - public SinglePartitionReadQuery readCommand(int nowInSec); + public SinglePartitionReadCommand readCommand(int nowInSec); /** * Returns whether the provided CF, that represents the values fetched using the @@ -43,5 +43,5 @@ public interface CASRequest * The updates to perform of a CAS success. The values fetched using the readFilter() * are passed as argument. */ - public PartitionUpdate makeUpdates(FilteredPartition current, ClientState state, TimeUUID ballot) throws InvalidRequestException; + public PartitionUpdate makeUpdates(FilteredPartition current, ClientState clientState, Ballot ballot) throws InvalidRequestException; } diff --git a/src/java/org/apache/cassandra/service/CassandraDaemon.java b/src/java/org/apache/cassandra/service/CassandraDaemon.java index 67195d7565..7e9c208d5e 100644 --- a/src/java/org/apache/cassandra/service/CassandraDaemon.java +++ b/src/java/org/apache/cassandra/service/CassandraDaemon.java @@ -78,6 +78,7 @@ import org.apache.cassandra.schema.SchemaConstants; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.security.ThreadAwareSecurityManager; import org.apache.cassandra.streaming.StreamManager; +import org.apache.cassandra.service.paxos.PaxosState; import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.JMXServerUtils; import org.apache.cassandra.utils.JVMStabilityInspector; @@ -347,6 +348,9 @@ public class CassandraDaemon } // Replay any CommitLogSegments found on disk + PaxosState.initializeTrackers(); + + // replay the log if necessary try { CommitLog.instance.recoverSegmentsOnDisk(); @@ -359,6 +363,15 @@ public class CassandraDaemon // Re-populate token metadata after commit log recover (new peers might be loaded onto system keyspace #10293) StorageService.instance.populateTokenMetadata(); + try + { + PaxosState.maybeRebuildUncommittedState(); + } + catch (IOException e) + { + throw new RuntimeException(e); + } + SystemKeyspace.finishStartup(); // Clean up system.size_estimates entries left lying around from missed keyspace drops (CASSANDRA-14905) diff --git a/src/java/org/apache/cassandra/service/ClientState.java b/src/java/org/apache/cassandra/service/ClientState.java index 65c562dc00..c7def848ec 100644 --- a/src/java/org/apache/cassandra/service/ClientState.java +++ b/src/java/org/apache/cassandra/service/ClientState.java @@ -26,8 +26,11 @@ import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; +import java.util.function.LongSupplier; +import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; @@ -136,6 +139,14 @@ public class ClientState // is unrealistic expectation, doing it node-wise is easy). private static final AtomicLong lastTimestampMicros = new AtomicLong(0); + @VisibleForTesting + public static void resetLastTimestamp(long nowMillis) + { + long nowMicros = TimeUnit.MILLISECONDS.toMicros(nowMillis); + if (lastTimestampMicros.get() > nowMicros) + lastTimestampMicros.set(nowMicros); + } + /** * Construct a new, empty ClientState for internal calls. */ @@ -252,7 +263,7 @@ public class ClientState * with a clock in the future compared to the local one), we use the last proposal timestamp plus 1, ensuring * progress. * - * @param minTimestampToUse the max timestamp of the last proposal accepted by replica having responded + * @param minUnixMicros the max timestamp of the last proposal accepted by replica having responded * to the prepare phase of the paxos round this is for. In practice, that's the minimum timestamp this method * may return. * @return a timestamp suitable for a Paxos proposal (using the reasoning described above). Note that @@ -261,18 +272,18 @@ public class ClientState * it may be returned multiple times). Note that we still ensure Paxos "ballot" are unique (for different * proposal) by (securely) randomizing the non-timestamp part of the UUID. */ - public static long getTimestampForPaxos(long minTimestampToUse) + public static long getTimestampForPaxos(long minUnixMicros) { while (true) { - long current = Math.max(currentTimeMillis() * 1000, minTimestampToUse); + long current = Math.max(currentTimeMillis() * 1000, minUnixMicros); long last = lastTimestampMicros.get(); long tstamp = last >= current ? last + 1 : current; // Note that if we ended up picking minTimestampMicrosToUse (it was "in the future"), we don't // want to change the local clock, otherwise a single node in the future could corrupt the clock // of all nodes and for all inserts (since non-paxos inserts also use lastTimestampMicros). // See CASSANDRA-11991 - if (tstamp == minTimestampToUse || lastTimestampMicros.compareAndSet(last, tstamp)) + if (tstamp == minUnixMicros || lastTimestampMicros.compareAndSet(last, tstamp)) return tstamp; } } diff --git a/src/java/org/apache/cassandra/service/DatacenterSyncWriteResponseHandler.java b/src/java/org/apache/cassandra/service/DatacenterSyncWriteResponseHandler.java index 65cf3cce66..2e26bb9a55 100644 --- a/src/java/org/apache/cassandra/service/DatacenterSyncWriteResponseHandler.java +++ b/src/java/org/apache/cassandra/service/DatacenterSyncWriteResponseHandler.java @@ -20,8 +20,10 @@ package org.apache.cassandra.service; import java.util.HashMap; import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Supplier; import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.db.Mutation; import org.apache.cassandra.locator.IEndpointSnitch; import org.apache.cassandra.locator.NetworkTopologyStrategy; import org.apache.cassandra.locator.Replica; @@ -40,13 +42,14 @@ public class DatacenterSyncWriteResponseHandler extends AbstractWriteResponse private final Map responses = new HashMap(); private final AtomicInteger acks = new AtomicInteger(0); - public DatacenterSyncWriteResponseHandler(ReplicaPlan.ForTokenWrite replicaPlan, + public DatacenterSyncWriteResponseHandler(ReplicaPlan.ForWrite replicaPlan, Runnable callback, WriteType writeType, + Supplier hintOnFailure, long queryStartNanoTime) { // Response is been managed by the map so make it 1 for the superclass. - super(replicaPlan, callback, writeType, queryStartNanoTime); + super(replicaPlan, callback, writeType, hintOnFailure, queryStartNanoTime); assert replicaPlan.consistencyLevel() == ConsistencyLevel.EACH_QUORUM; if (replicaPlan.replicationStrategy() instanceof NetworkTopologyStrategy) diff --git a/src/java/org/apache/cassandra/service/DatacenterWriteResponseHandler.java b/src/java/org/apache/cassandra/service/DatacenterWriteResponseHandler.java index a9583a3c35..4920a54516 100644 --- a/src/java/org/apache/cassandra/service/DatacenterWriteResponseHandler.java +++ b/src/java/org/apache/cassandra/service/DatacenterWriteResponseHandler.java @@ -17,27 +17,30 @@ */ package org.apache.cassandra.service; +import org.apache.cassandra.db.Mutation; import org.apache.cassandra.db.WriteType; -import org.apache.cassandra.locator.InOurDcTester; +import org.apache.cassandra.locator.InOurDc; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.locator.ReplicaPlan; import org.apache.cassandra.net.Message; import java.util.function.Predicate; +import java.util.function.Supplier; /** * This class blocks for a quorum of responses _in the local datacenter only_ (CL.LOCAL_QUORUM). */ public class DatacenterWriteResponseHandler extends WriteResponseHandler { - private final Predicate waitingFor = InOurDcTester.endpoints(); + private final Predicate waitingFor = InOurDc.endpoints(); - public DatacenterWriteResponseHandler(ReplicaPlan.ForTokenWrite replicaPlan, + public DatacenterWriteResponseHandler(ReplicaPlan.ForWrite replicaPlan, Runnable callback, WriteType writeType, + Supplier hintOnFailure, long queryStartNanoTime) { - super(replicaPlan, callback, writeType, queryStartNanoTime); + super(replicaPlan, callback, writeType, hintOnFailure, queryStartNanoTime); assert replicaPlan.consistencyLevel().isDatacenterLocal(); } diff --git a/src/java/org/apache/cassandra/service/FailureRecordingCallback.java b/src/java/org/apache/cassandra/service/FailureRecordingCallback.java new file mode 100644 index 0000000000..c4ca8e22f5 --- /dev/null +++ b/src/java/org/apache/cassandra/service/FailureRecordingCallback.java @@ -0,0 +1,153 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service; + +import java.util.AbstractMap; +import java.util.AbstractSet; +import java.util.Iterator; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; + +import org.apache.cassandra.exceptions.RequestFailureReason; +import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.net.RequestCallbackWithFailure; +import org.apache.cassandra.utils.concurrent.IntrusiveStack; + +import static org.apache.cassandra.exceptions.RequestFailureReason.TIMEOUT; + +public abstract class FailureRecordingCallback implements RequestCallbackWithFailure +{ + public static class FailureResponses extends IntrusiveStack implements Map.Entry + { + final InetAddressAndPort from; + final RequestFailureReason reason; + + public FailureResponses(InetAddressAndPort from, RequestFailureReason reason) + { + this.from = from; + this.reason = reason; + } + + @Override + public InetAddressAndPort getKey() + { + return from; + } + + @Override + public RequestFailureReason getValue() + { + return reason; + } + + @Override + public RequestFailureReason setValue(RequestFailureReason value) + { + throw new UnsupportedOperationException(); + } + + public static void push(AtomicReferenceFieldUpdater headUpdater, O owner, InetAddressAndPort from, RequestFailureReason reason) + { + push(headUpdater, owner, new FailureResponses(from, reason)); + } + + public static void pushExclusive(AtomicReferenceFieldUpdater headUpdater, O owner, InetAddressAndPort from, RequestFailureReason reason) + { + pushExclusive(headUpdater, owner, new FailureResponses(from, reason)); + } + + public static FailureResponses pushExclusive(FailureResponses head, InetAddressAndPort from, RequestFailureReason reason) + { + return IntrusiveStack.pushExclusive(head, new FailureResponses(from, reason)); + } + + public static int size(FailureResponses head) + { + return IntrusiveStack.size(head); + } + + public static Iterator iterator(FailureResponses head) + { + return IntrusiveStack.iterator(head); + } + + public static int failureCount(FailureResponses head) + { + return (int)IntrusiveStack.accumulate(head, (f, v) -> f.reason == TIMEOUT ? v : v + 1, 0); + } + } + + public static class AsMap extends AbstractMap + { + final FailureResponses head; + int size = -1; + + AsMap(FailureResponses head) + { + this.head = head; + } + + @Override + public Set> entrySet() + { + return new AbstractSet>() + { + @Override + public Iterator> iterator() + { + return (Iterator>) + (Iterator) FailureResponses.iterator(head); + } + + @Override + public int size() + { + if (size < 0) + size = FailureResponses.size(head); + return size; + } + }; + } + + public int failureCount() + { + return FailureResponses.failureCount(head); + } + } + + private volatile FailureResponses failureResponses; + private static final AtomicReferenceFieldUpdater responsesUpdater = AtomicReferenceFieldUpdater.newUpdater(FailureRecordingCallback.class, FailureResponses.class, "failureResponses"); + + @Override + public void onFailure(InetAddressAndPort from, RequestFailureReason failureReason) + { + FailureResponses.push(responsesUpdater, this, from, failureReason); + } + + protected void onFailureWithMutex(InetAddressAndPort from, RequestFailureReason failureReason) + { + FailureResponses.pushExclusive(responsesUpdater, this, from, failureReason); + } + + protected AsMap failureReasonsAsMap() + { + return new AsMap(failureResponses); + } +} diff --git a/src/java/org/apache/cassandra/service/PendingRangeCalculatorService.java b/src/java/org/apache/cassandra/service/PendingRangeCalculatorService.java index cb8dbc3ed2..ad0e0e0b0f 100644 --- a/src/java/org/apache/cassandra/service/PendingRangeCalculatorService.java +++ b/src/java/org/apache/cassandra/service/PendingRangeCalculatorService.java @@ -77,6 +77,12 @@ public class PendingRangeCalculatorService update.sync(); } + + public void executeWhenFinished(Runnable runnable) + { + update.runAfter(runnable); + } + // public & static for testing purposes public static void calculatePendingRanges(AbstractReplicationStrategy strategy, String keyspaceName) { diff --git a/src/java/org/apache/cassandra/service/StorageProxy.java b/src/java/org/apache/cassandra/service/StorageProxy.java index 267436ff33..99ed347be0 100644 --- a/src/java/org/apache/cassandra/service/StorageProxy.java +++ b/src/java/org/apache/cassandra/service/StorageProxy.java @@ -37,17 +37,16 @@ import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import java.util.function.Function; -import java.util.function.Supplier; import java.util.stream.Collectors; import com.google.common.base.Preconditions; import com.google.common.cache.CacheLoader; import com.google.common.collect.Iterables; -import com.google.common.primitives.Ints; import com.google.common.util.concurrent.Uninterruptibles; import org.apache.cassandra.config.Config; import org.apache.cassandra.service.paxos.*; +import org.apache.cassandra.service.paxos.Paxos; import org.apache.cassandra.utils.TimeUUID; import org.apache.cassandra.utils.concurrent.CountDownLatch; @@ -125,6 +124,8 @@ import org.apache.cassandra.schema.Schema; import org.apache.cassandra.schema.SchemaConstants; import org.apache.cassandra.schema.TableId; import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.service.paxos.v1.PrepareCallback; +import org.apache.cassandra.service.paxos.v1.ProposeCallback; import org.apache.cassandra.service.reads.AbstractReadExecutor; import org.apache.cassandra.service.reads.ReadCallback; import org.apache.cassandra.service.reads.range.RangeCommands; @@ -154,10 +155,11 @@ import static org.apache.cassandra.metrics.ClientRequestsMetricsHolder.writeMetr import static org.apache.cassandra.net.NoPayload.noPayload; import static org.apache.cassandra.net.Verb.*; import static org.apache.cassandra.service.BatchlogResponseHandler.BatchlogCleanup; -import static org.apache.cassandra.service.paxos.BallotGenerator.Global.nextBallotTimestampMicros; -import static org.apache.cassandra.service.paxos.BallotGenerator.Global.randomBallot; -import static org.apache.cassandra.service.paxos.PrepareVerbHandler.doPrepare; -import static org.apache.cassandra.service.paxos.ProposeVerbHandler.doPropose; +import static org.apache.cassandra.service.paxos.Ballot.Flag.GLOBAL; +import static org.apache.cassandra.service.paxos.Ballot.Flag.LOCAL; +import static org.apache.cassandra.service.paxos.BallotGenerator.Global.nextBallot; +import static org.apache.cassandra.service.paxos.v1.PrepareVerbHandler.doPrepare; +import static org.apache.cassandra.service.paxos.v1.ProposeVerbHandler.doPropose; import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis; import static org.apache.cassandra.utils.Clock.Global.nanoTime; import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID; @@ -217,7 +219,7 @@ public class StorageProxy implements StorageProxyMBean { EndpointsForToken selected = targets.contacts().withoutSelf(); Replicas.temporaryAssertFull(selected); // TODO CASSANDRA-14548 - counterWriteTask(mutation, targets.withContact(selected), responseHandler, localDataCenter).run(); + counterWriteTask(mutation, targets.withContacts(selected), responseHandler, localDataCenter).run(); }; counterWriteOnCoordinatorPerformer = (mutation, targets, responseHandler, localDataCenter) -> @@ -225,11 +227,21 @@ public class StorageProxy implements StorageProxyMBean EndpointsForToken selected = targets.contacts().withoutSelf(); Replicas.temporaryAssertFull(selected); // TODO CASSANDRA-14548 Stage.COUNTER_MUTATION.executor() - .execute(counterWriteTask(mutation, targets.withContact(selected), responseHandler, localDataCenter)); + .execute(counterWriteTask(mutation, targets.withContacts(selected), responseHandler, localDataCenter)); }; ReadRepairMetrics.init(); + + if (!Paxos.isLinearizable()) + { + logger.warn("This node was started with paxos variant {}. SERIAL (and LOCAL_SERIAL) reads coordinated by this node " + + "will not offer linearizability (see CASSANDRA-12126 for details on what this means) with " + + "respect to other SERIAL operations. Please note that with this variant, SERIAL reads will be " + + "slower than QUORUM reads, yet offer no additional guarantees. This flag should only be used in " + + "the restricted case of upgrading from a pre-CASSANDRA-12126 version, and only if you " + + "understand the tradeoff.", Paxos.getPaxosVariant()); + } } /** @@ -279,24 +291,40 @@ public class StorageProxy implements StorageProxyMBean CASRequest request, ConsistencyLevel consistencyForPaxos, ConsistencyLevel consistencyForCommit, - ClientState state, + ClientState clientState, int nowInSeconds, long queryStartNanoTime) throws UnavailableException, IsBootstrappingException, RequestFailureException, RequestTimeoutException, InvalidRequestException, CasWriteUnknownResultException + { + if (DatabaseDescriptor.getPartitionDenylistEnabled() && DatabaseDescriptor.getDenylistWritesEnabled() && !partitionDenylist.isKeyPermitted(keyspaceName, cfName, key.getKey())) + { + denylistMetrics.incrementWritesRejected(); + throw new InvalidRequestException(String.format("Unable to CAS write to denylisted partition [0x%s] in %s/%s", + key.toString(), keyspaceName, cfName)); + } + + return Paxos.useV2() + ? Paxos.cas(key, request, consistencyForPaxos, consistencyForCommit, clientState) + : legacyCas(keyspaceName, cfName, key, request, consistencyForPaxos, consistencyForCommit, clientState, nowInSeconds, queryStartNanoTime); + } + + public static RowIterator legacyCas(String keyspaceName, + String cfName, + DecoratedKey key, + CASRequest request, + ConsistencyLevel consistencyForPaxos, + ConsistencyLevel consistencyForCommit, + ClientState clientState, + int nowInSeconds, + long queryStartNanoTime) + throws UnavailableException, IsBootstrappingException, RequestFailureException, RequestTimeoutException, InvalidRequestException { final long startTimeForMetrics = nanoTime(); try { TableMetadata metadata = Schema.instance.validateTable(keyspaceName, cfName); - if (DatabaseDescriptor.getPartitionDenylistEnabled() && DatabaseDescriptor.getDenylistWritesEnabled() && !partitionDenylist.isKeyPermitted(keyspaceName, cfName, key.getKey())) - { - denylistMetrics.incrementWritesRejected(); - throw new InvalidRequestException(String.format("Unable to CAS write to denylisted partition [0x%s] in %s/%s", - key.toString(), keyspaceName, cfName)); - } - - Function> updateProposer = ballot -> + Function> updateProposer = ballot -> { // read the current values and check they validate the conditions Tracing.trace("Reading existing values for CAS precondition"); @@ -317,7 +345,7 @@ public class StorageProxy implements StorageProxyMBean } // Create the desired updates - PartitionUpdate updates = request.makeUpdates(current, state, ballot); + PartitionUpdate updates = request.makeUpdates(current, clientState, ballot); long size = updates.dataSize(); casWriteMetrics.mutationSize.update(size); @@ -435,7 +463,7 @@ public class StorageProxy implements StorageProxyMBean ConsistencyLevel consistencyForCommit, long queryStartNanoTime, CASClientRequestMetrics casMetrics, - Function> createUpdateProposal) + Function> createUpdateProposal) throws UnavailableException, IsBootstrappingException, RequestFailureException, RequestTimeoutException, InvalidRequestException { int contentions = 0; @@ -461,7 +489,7 @@ public class StorageProxy implements StorageProxyMBean consistencyForReplayCommits, casMetrics); - final TimeUUID ballot = pair.ballot; + final Ballot ballot = pair.ballot; contentions += pair.contentions; Pair proposalPair = createUpdateProposal.apply(ballot); @@ -489,6 +517,7 @@ public class StorageProxy implements StorageProxyMBean Tracing.trace("Paxos proposal not accepted (pre-empted by a higher ballot)"); contentions++; + Uninterruptibles.sleepUninterruptibly(ThreadLocalRandom.current().nextInt(100), TimeUnit.MILLISECONDS); // continue to retry } @@ -540,10 +569,9 @@ public class StorageProxy implements StorageProxyMBean // in progress (#5667). Lastly, we don't want to use a timestamp that is older than the last one assigned by ClientState or operations may appear // out-of-order (#7801). long minTimestampMicrosToUse = summary == null ? Long.MIN_VALUE : 1 + summary.mostRecentInProgressCommit.ballot.unixMicros(); - long ballotMicros = nextBallotTimestampMicros(minTimestampMicrosToUse); // Note that ballotMicros is not guaranteed to be unique if two proposal are being handled concurrently by the same coordinator. But we still // need ballots to be unique for each proposal so we have to use getRandomTimeUUIDFromMicros. - TimeUUID ballot = randomBallot(ballotMicros, consistencyForPaxos == SERIAL); + Ballot ballot = nextBallot(minTimestampMicrosToUse, consistencyForPaxos == SERIAL ? GLOBAL : LOCAL); // prepare try @@ -604,8 +632,7 @@ public class StorageProxy implements StorageProxyMBean // https://issues.apache.org/jira/browse/CASSANDRA-5062?focusedCommentId=13619810&page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel#comment-13619810) // Since we waited for quorum nodes, if some of them haven't seen the last commit (which may just be a timing issue, but may also // mean we lost messages), we pro-actively "repair" those nodes, and retry. - int nowInSec = Ints.checkedCast(TimeUnit.MICROSECONDS.toSeconds(ballotMicros)); - Iterable missingMRC = summary.replicasMissingMostRecentCommit(metadata, nowInSec); + Iterable missingMRC = summary.replicasMissingMostRecentCommit(); if (Iterables.size(missingMRC) > 0) { Tracing.trace("Repairing replicas that missed the most recent commit"); @@ -729,11 +756,11 @@ public class StorageProxy implements StorageProxyMBean AbstractWriteResponseHandler responseHandler = null; // NOTE: this ReplicaPlan is a lie, this usage of ReplicaPlan could do with being clarified - the selected() collection is essentially (I think) never used - ReplicaPlan.ForTokenWrite replicaPlan = ReplicaPlans.forWrite(keyspace, consistencyLevel, tk, ReplicaPlans.writeAll); + ReplicaPlan.ForWrite replicaPlan = ReplicaPlans.forWrite(keyspace, consistencyLevel, tk, ReplicaPlans.writeAll); if (shouldBlock) { AbstractReplicationStrategy rs = replicaPlan.replicationStrategy(); - responseHandler = rs.getWriteResponseHandler(replicaPlan, null, WriteType.SIMPLE, queryStartNanoTime); + responseHandler = rs.getWriteResponseHandler(replicaPlan, null, WriteType.SIMPLE, proposal::makeMutation, queryStartNanoTime); } Message message = Message.outWithFlag(PAXOS_COMMIT_REQ, proposal, MessageFlag.CALL_BACK_ON_FAILURE); @@ -786,7 +813,7 @@ public class StorageProxy implements StorageProxyMBean { try { - PaxosState.commit(message.payload); + PaxosState.commitDirect(message.payload); if (responseHandler != null) responseHandler.onResponse(null); } @@ -993,7 +1020,7 @@ public class StorageProxy implements StorageProxyMBean ConsistencyLevel consistencyLevel = ConsistencyLevel.ONE; //Since the base -> view replication is 1:1 we only need to store the BL locally - ReplicaPlan.ForTokenWrite replicaPlan = ReplicaPlans.forLocalBatchlogWrite(); + ReplicaPlan.ForWrite replicaPlan = ReplicaPlans.forLocalBatchlogWrite(); BatchlogCleanup cleanup = new BatchlogCleanup(mutations.size(), () -> asyncRemoveFromBatchlog(replicaPlan, batchUUID)); @@ -1162,7 +1189,7 @@ public class StorageProxy implements StorageProxyMBean batchConsistencyLevel = consistency_level; } - ReplicaPlan.ForTokenWrite replicaPlan = ReplicaPlans.forBatchlogWrite(batchConsistencyLevel == ConsistencyLevel.ANY); + ReplicaPlan.ForWrite replicaPlan = ReplicaPlans.forBatchlogWrite(batchConsistencyLevel == ConsistencyLevel.ANY); final TimeUUID batchUUID = nextTimeUUID(); BatchlogCleanup cleanup = new BatchlogCleanup(mutations.size(), @@ -1244,11 +1271,12 @@ public class StorageProxy implements StorageProxyMBean } } - private static void syncWriteToBatchlog(Collection mutations, ReplicaPlan.ForTokenWrite replicaPlan, TimeUUID uuid, long queryStartNanoTime) + private static void syncWriteToBatchlog(Collection mutations, ReplicaPlan.ForWrite replicaPlan, TimeUUID uuid, long queryStartNanoTime) throws WriteTimeoutException, WriteFailureException { WriteResponseHandler handler = new WriteResponseHandler(replicaPlan, WriteType.BATCH_LOG, + null, queryStartNanoTime); Batch batch = Batch.createLocal(uuid, FBUtilities.timestampMicros(), mutations); @@ -1265,7 +1293,7 @@ public class StorageProxy implements StorageProxyMBean handler.get(); } - private static void asyncRemoveFromBatchlog(ReplicaPlan.ForTokenWrite replicaPlan, TimeUUID uuid) + private static void asyncRemoveFromBatchlog(ReplicaPlan.ForWrite replicaPlan, TimeUUID uuid) { Message message = Message.out(Verb.BATCH_REMOVE_REQ, uuid); for (Replica target : replicaPlan.contacts()) @@ -1285,7 +1313,7 @@ public class StorageProxy implements StorageProxyMBean for (WriteResponseHandlerWrapper wrapper : wrappers) { Replicas.temporaryAssertFull(wrapper.handler.replicaPlan.liveAndDown()); // TODO: CASSANDRA-14549 - ReplicaPlan.ForTokenWrite replicas = wrapper.handler.replicaPlan.withContact(wrapper.handler.replicaPlan.liveAndDown()); + ReplicaPlan.ForWrite replicas = wrapper.handler.replicaPlan.withContacts(wrapper.handler.replicaPlan.liveAndDown()); try { @@ -1307,7 +1335,7 @@ public class StorageProxy implements StorageProxyMBean { EndpointsForToken sendTo = wrapper.handler.replicaPlan.liveAndDown(); Replicas.temporaryAssertFull(sendTo); // TODO: CASSANDRA-14549 - sendToHintedReplicas(wrapper.mutation, wrapper.handler.replicaPlan.withContact(sendTo), wrapper.handler, localDataCenter, stage); + sendToHintedReplicas(wrapper.mutation, wrapper.handler.replicaPlan.withContacts(sendTo), wrapper.handler, localDataCenter, stage); } for (WriteResponseHandlerWrapper wrapper : wrappers) @@ -1340,9 +1368,9 @@ public class StorageProxy implements StorageProxyMBean Keyspace keyspace = Keyspace.open(keyspaceName); Token tk = mutation.key().getToken(); - ReplicaPlan.ForTokenWrite replicaPlan = ReplicaPlans.forWrite(keyspace, consistencyLevel, tk, ReplicaPlans.writeNormal); + ReplicaPlan.ForWrite replicaPlan = ReplicaPlans.forWrite(keyspace, consistencyLevel, tk, ReplicaPlans.writeNormal); AbstractReplicationStrategy rs = replicaPlan.replicationStrategy(); - AbstractWriteResponseHandler responseHandler = rs.getWriteResponseHandler(replicaPlan, callback, writeType, queryStartNanoTime); + AbstractWriteResponseHandler responseHandler = rs.getWriteResponseHandler(replicaPlan, callback, writeType, mutation.hintOnFailure(), queryStartNanoTime); performer.apply(mutation, replicaPlan, responseHandler, localDataCenter); return responseHandler; @@ -1359,9 +1387,9 @@ public class StorageProxy implements StorageProxyMBean Keyspace keyspace = Keyspace.open(mutation.getKeyspaceName()); Token tk = mutation.key().getToken(); - ReplicaPlan.ForTokenWrite replicaPlan = ReplicaPlans.forWrite(keyspace, consistencyLevel, tk, ReplicaPlans.writeNormal); + ReplicaPlan.ForWrite replicaPlan = ReplicaPlans.forWrite(keyspace, consistencyLevel, tk, ReplicaPlans.writeNormal); AbstractReplicationStrategy rs = replicaPlan.replicationStrategy(); - AbstractWriteResponseHandler writeHandler = rs.getWriteResponseHandler(replicaPlan,null, writeType, queryStartNanoTime); + AbstractWriteResponseHandler writeHandler = rs.getWriteResponseHandler(replicaPlan, null, writeType, mutation, queryStartNanoTime); BatchlogResponseHandler batchHandler = new BatchlogResponseHandler<>(writeHandler, batchConsistencyLevel.blockFor(rs), cleanup, queryStartNanoTime); return new WriteResponseHandlerWrapper(batchHandler, mutation); } @@ -1380,12 +1408,12 @@ public class StorageProxy implements StorageProxyMBean long queryStartNanoTime) { Keyspace keyspace = Keyspace.open(mutation.getKeyspaceName()); - ReplicaPlan.ForTokenWrite replicaPlan = ReplicaPlans.forWrite(keyspace, consistencyLevel, liveAndDown, ReplicaPlans.writeAll); + ReplicaPlan.ForWrite replicaPlan = ReplicaPlans.forWrite(keyspace, consistencyLevel, liveAndDown, ReplicaPlans.writeAll); AbstractReplicationStrategy replicationStrategy = replicaPlan.replicationStrategy(); AbstractWriteResponseHandler writeHandler = replicationStrategy.getWriteResponseHandler(replicaPlan, () -> { long delay = Math.max(0, currentTimeMillis() - baseComplete.get()); viewWriteMetrics.viewWriteLatency.update(delay, MILLISECONDS); - }, writeType, queryStartNanoTime); + }, writeType, mutation, queryStartNanoTime); BatchlogResponseHandler batchHandler = new ViewWriteMetricsWrapped(writeHandler, batchConsistencyLevel.blockFor(replicationStrategy), cleanup, queryStartNanoTime); return new WriteResponseHandlerWrapper(batchHandler, mutation); } @@ -1421,7 +1449,7 @@ public class StorageProxy implements StorageProxyMBean * @throws OverloadedException if the hints cannot be written/enqueued */ public static void sendToHintedReplicas(final Mutation mutation, - ReplicaPlan.ForTokenWrite plan, + ReplicaPlan.ForWrite plan, AbstractWriteResponseHandler responseHandler, String localDataCenter, Stage stage) @@ -1658,7 +1686,7 @@ public class StorageProxy implements StorageProxyMBean // Forward the actual update to the chosen leader replica AbstractWriteResponseHandler responseHandler = new WriteResponseHandler<>(ReplicaPlans.forForwardingCounterWrite(keyspace, tk, replica), - WriteType.COUNTER, queryStartNanoTime); + WriteType.COUNTER, null, queryStartNanoTime); Tracing.trace("Enqueuing counter update to {}", replica); Message message = Message.outWithFlag(Verb.COUNTER_MUTATION_REQ, cm, MessageFlag.CALL_BACK_ON_FAILURE); @@ -1728,7 +1756,7 @@ public class StorageProxy implements StorageProxyMBean } private static Runnable counterWriteTask(final IMutation mutation, - final ReplicaPlan.ForTokenWrite replicaPlan, + final ReplicaPlan.ForWrite replicaPlan, final AbstractWriteResponseHandler responseHandler, final String localDataCenter) { @@ -1757,28 +1785,14 @@ public class StorageProxy implements StorageProxyMBean public static RowIterator readOne(SinglePartitionReadCommand command, ConsistencyLevel consistencyLevel, long queryStartNanoTime) throws UnavailableException, IsBootstrappingException, ReadFailureException, ReadTimeoutException, InvalidRequestException { - return readOne(command, consistencyLevel, null, queryStartNanoTime); - } - - public static RowIterator readOne(SinglePartitionReadCommand command, ConsistencyLevel consistencyLevel, ClientState state, long queryStartNanoTime) - throws UnavailableException, IsBootstrappingException, ReadFailureException, ReadTimeoutException, InvalidRequestException - { - return PartitionIterators.getOnlyElement(read(SinglePartitionReadCommand.Group.one(command), consistencyLevel, state, queryStartNanoTime), command); - } - - public static PartitionIterator read(SinglePartitionReadCommand.Group group, ConsistencyLevel consistencyLevel, long queryStartNanoTime) - throws UnavailableException, IsBootstrappingException, ReadFailureException, ReadTimeoutException, InvalidRequestException - { - // When using serial CL, the ClientState should be provided - assert !consistencyLevel.isSerialConsistency(); - return read(group, consistencyLevel, null, queryStartNanoTime); + return PartitionIterators.getOnlyElement(read(SinglePartitionReadCommand.Group.one(command), consistencyLevel, queryStartNanoTime), command); } /** * Performs the actual reading of a row out of the StorageService, fetching * a specific set of column names from a given column family. */ - public static PartitionIterator read(SinglePartitionReadCommand.Group group, ConsistencyLevel consistencyLevel, ClientState state, long queryStartNanoTime) + public static PartitionIterator read(SinglePartitionReadCommand.Group group, ConsistencyLevel consistencyLevel, long queryStartNanoTime) throws UnavailableException, IsBootstrappingException, ReadFailureException, ReadTimeoutException, InvalidRequestException { if (StorageService.instance.isBootstrapMode() && !systemKeyspaceQuery(group.queries)) @@ -1804,14 +1818,21 @@ public class StorageProxy implements StorageProxyMBean } return consistencyLevel.isSerialConsistency() - ? readWithPaxos(group, consistencyLevel, state, queryStartNanoTime) + ? readWithPaxos(group, consistencyLevel, queryStartNanoTime) : readRegular(group, consistencyLevel, queryStartNanoTime); } - private static PartitionIterator readWithPaxos(SinglePartitionReadCommand.Group group, ConsistencyLevel consistencyLevel, ClientState state, long queryStartNanoTime) + private static PartitionIterator readWithPaxos(SinglePartitionReadCommand.Group group, ConsistencyLevel consistencyLevel, long queryStartNanoTime) + throws InvalidRequestException, UnavailableException, ReadFailureException, ReadTimeoutException + { + return Paxos.useV2() + ? Paxos.read(group, consistencyLevel) + : legacyReadWithPaxos(group, consistencyLevel, queryStartNanoTime); + } + + private static PartitionIterator legacyReadWithPaxos(SinglePartitionReadCommand.Group group, ConsistencyLevel consistencyLevel, long queryStartNanoTime) throws InvalidRequestException, UnavailableException, ReadFailureException, ReadTimeoutException { - assert state != null; if (group.queries.size() > 1) throw new InvalidRequestException("SERIAL/LOCAL_SERIAL consistency may only be requested for one partition at a time"); @@ -1833,8 +1854,8 @@ public class StorageProxy implements StorageProxyMBean { // Commit an empty update to make sure all in-progress updates that should be finished first is, _and_ // that no other in-progress can get resurrected. - Function> updateProposer = - Paxos.getPaxosVariant() == Config.PaxosVariant.v1_without_linearizable_reads + Function> updateProposer = + !Paxos.isLinearizable() ? ballot -> null : ballot -> Pair.create(PartitionUpdate.emptyUpdate(metadata, key), null); // When replaying, we commit at quorum/local quorum, as we want to be sure the following read (done at @@ -2421,7 +2442,7 @@ public class StorageProxy implements StorageProxyMBean public interface WritePerformer { public void apply(IMutation mutation, - ReplicaPlan.ForTokenWrite targets, + ReplicaPlan.ForWrite targets, AbstractWriteResponseHandler responseHandler, String localDataCenter) throws OverloadedException; } @@ -2810,10 +2831,10 @@ public class StorageProxy implements StorageProxyMBean static class PaxosBallotAndContention { - final TimeUUID ballot; + final Ballot ballot; final int contentions; - PaxosBallotAndContention(TimeUUID ballot, int contentions) + PaxosBallotAndContention(Ballot ballot, int contentions) { this.ballot = ballot; this.contentions = contentions; @@ -3032,4 +3053,26 @@ public class StorageProxy implements StorageProxyMBean { DatabaseDescriptor.setUseStatementsEnabled(enabled); } + + public void setPaxosContentionStrategy(String spec) + { + ContentionStrategy.setStrategy(spec); + } + + public String getPaxosContentionStrategy() + { + return ContentionStrategy.getStrategySpec(); + } + + @Override + public void setPaxosCoordinatorLockingDisabled(boolean disabled) + { + PaxosState.setDisableCoordinatorLocking(disabled); + } + + @Override + public boolean getPaxosCoordinatorLockingDisabled() + { + return PaxosState.getDisableCoordinatorLocking(); + } } diff --git a/src/java/org/apache/cassandra/service/StorageProxyMBean.java b/src/java/org/apache/cassandra/service/StorageProxyMBean.java index 0f69032f4b..416a31284a 100644 --- a/src/java/org/apache/cassandra/service/StorageProxyMBean.java +++ b/src/java/org/apache/cassandra/service/StorageProxyMBean.java @@ -126,4 +126,10 @@ public interface StorageProxyMBean boolean getUseStatementsEnabled(); void setUseStatementsEnabled(boolean enabled); + + void setPaxosContentionStrategy(String variant); + String getPaxosContentionStrategy(); + + void setPaxosCoordinatorLockingDisabled(boolean disabled); + boolean getPaxosCoordinatorLockingDisabled(); } diff --git a/src/java/org/apache/cassandra/service/StorageService.java b/src/java/org/apache/cassandra/service/StorageService.java index 1a7b457f37..83fc2d8c0e 100644 --- a/src/java/org/apache/cassandra/service/StorageService.java +++ b/src/java/org/apache/cassandra/service/StorageService.java @@ -54,6 +54,7 @@ import javax.management.openmbean.TabularData; import javax.management.openmbean.TabularDataSupport; import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Joiner; import com.google.common.base.Preconditions; import com.google.common.base.Predicate; import com.google.common.base.Predicates; @@ -71,6 +72,9 @@ import org.apache.cassandra.fql.FullQueryLoggerOptionsCompositeData; import org.apache.cassandra.io.util.File; import org.apache.cassandra.locator.ReplicaCollection.Builder.Conflict; import org.apache.cassandra.utils.concurrent.Future; +import org.apache.cassandra.schema.TableId; +import org.apache.cassandra.utils.concurrent.Future; +import org.apache.cassandra.utils.concurrent.FutureCombiner; import org.apache.cassandra.utils.concurrent.ImmediateFuture; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; @@ -79,6 +83,10 @@ import org.slf4j.LoggerFactory; import org.apache.cassandra.audit.AuditLogManager; import org.apache.cassandra.audit.AuditLogOptions; import org.apache.cassandra.auth.AuthCacheService; +import org.apache.cassandra.config.Config.PaxosStatePurging; +import org.apache.cassandra.service.paxos.*; +import org.apache.cassandra.service.paxos.cleanup.*; +import org.apache.cassandra.utils.progress.ProgressListener; import org.apache.cassandra.auth.AuthKeyspace; import org.apache.cassandra.auth.AuthSchemaChangeListener; import org.apache.cassandra.batchlog.BatchlogManager; @@ -129,6 +137,8 @@ import org.apache.cassandra.schema.TableMetadataRef; import org.apache.cassandra.schema.ViewMetadata; import org.apache.cassandra.service.snapshot.SnapshotManager; import org.apache.cassandra.service.snapshot.TableSnapshot; +import org.apache.cassandra.net.AsyncOneResponse; +import org.apache.cassandra.net.MessagingService; import org.apache.cassandra.streaming.*; import org.apache.cassandra.tracing.TraceKeyspace; import org.apache.cassandra.transport.ClientResourceLimits; @@ -140,7 +150,6 @@ import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException; import org.apache.cassandra.utils.logging.LoggingSupportFactory; import org.apache.cassandra.utils.progress.ProgressEvent; import org.apache.cassandra.utils.progress.ProgressEventType; -import org.apache.cassandra.utils.progress.ProgressListener; import org.apache.cassandra.utils.progress.jmx.JMXBroadcastExecutor; import org.apache.cassandra.utils.progress.jmx.JMXProgressSupport; @@ -148,19 +157,23 @@ import static com.google.common.collect.Iterables.transform; import static com.google.common.collect.Iterables.tryFind; import static java.util.Arrays.asList; import static java.util.Arrays.stream; +import static java.util.concurrent.TimeUnit.MICROSECONDS; import static java.util.concurrent.TimeUnit.MINUTES; import static java.util.concurrent.TimeUnit.MILLISECONDS; import static java.util.concurrent.TimeUnit.NANOSECONDS; +import static java.util.concurrent.TimeUnit.SECONDS; import static java.util.stream.Collectors.toList; import static java.util.stream.Collectors.toMap; import static org.apache.cassandra.config.CassandraRelevantProperties.BOOTSTRAP_SCHEMA_DELAY_MS; import static org.apache.cassandra.config.CassandraRelevantProperties.BOOTSTRAP_SKIP_SCHEMA_CHECK; import static org.apache.cassandra.config.CassandraRelevantProperties.REPLACEMENT_ALLOW_EMPTY; +import static org.apache.cassandra.config.Config.PaxosStatePurging.repaired; import static org.apache.cassandra.index.SecondaryIndexManager.getIndexName; import static org.apache.cassandra.index.SecondaryIndexManager.isIndexColumnFamily; import static org.apache.cassandra.net.NoPayload.noPayload; import static org.apache.cassandra.net.Verb.REPLICATION_DONE_REQ; import static org.apache.cassandra.schema.MigrationManager.evolveSystemKeyspace; +import static org.apache.cassandra.service.ActiveRepairService.*; import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis; import static org.apache.cassandra.utils.Clock.Global.nanoTime; import static org.apache.cassandra.service.ActiveRepairService.*; @@ -252,6 +265,16 @@ public class StorageService extends NotificationBroadcasterSupport implements IE .getAddressReplicas(FBUtilities.getBroadcastAddressAndPort()); } + public List> getLocalRanges(String ks) + { + InetAddressAndPort broadcastAddress = FBUtilities.getBroadcastAddressAndPort(); + Keyspace keyspace = Keyspace.open(ks); + List> ranges = new ArrayList<>(); + for (Replica r : keyspace.getReplicationStrategy().getAddressReplicas(broadcastAddress)) + ranges.add(r.range()); + return ranges; + } + public List> getLocalAndPendingRanges(String ks) { InetAddressAndPort broadcastAddress = FBUtilities.getBroadcastAddressAndPort(); @@ -1280,18 +1303,20 @@ public class StorageService extends NotificationBroadcasterSupport implements IE throw new IllegalStateException("Node is still rebuilding. Check nodetool netstats."); } - // check the arguments - if (keyspace == null && tokens != null) - { - throw new IllegalArgumentException("Cannot specify tokens without keyspace."); - } - - logger.info("rebuild from dc: {}, {}, {}", sourceDc == null ? "(any dc)" : sourceDc, - keyspace == null ? "(All keyspaces)" : keyspace, - tokens == null ? "(All tokens)" : tokens); - try { + // check the arguments + if (keyspace == null && tokens != null) + { + throw new IllegalArgumentException("Cannot specify tokens without keyspace."); + } + + logger.info("rebuild from dc: {}, {}, {}", sourceDc == null ? "(any dc)" : sourceDc, + keyspace == null ? "(All keyspaces)" : keyspace, + tokens == null ? "(All tokens)" : tokens); + + repairPaxosForTopologyChange("rebuild"); + RangeStreamer streamer = new RangeStreamer(tokenMetadata, null, FBUtilities.getBroadcastAddressAndPort(), @@ -1862,6 +1887,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE // Force disk boundary invalidation now that local tokens are set invalidateDiskBoundaries(); + repairPaxosForTopologyChange("bootstrap"); Future bootstrapStream = startBootstrap(tokens); try @@ -4300,18 +4326,102 @@ public class StorageService extends NotificationBroadcasterSupport implements IE return new FutureTask<>(task); } + private void tryRepairPaxosForTopologyChange(String reason) + { + try + { + startRepairPaxosForTopologyChange(reason).get(); + } + catch (InterruptedException e) + { + logger.error("Error during paxos repair", e); + throw new AssertionError(e); + } + catch (ExecutionException e) + { + logger.error("Error during paxos repair", e); + throw new RuntimeException(e); + } + } + + private void repairPaxosForTopologyChange(String reason) + { + if (getSkipPaxosRepairOnTopologyChange() || !Paxos.useV2()) + { + logger.info("skipping paxos repair for {}. skip_paxos_repair_on_topology_change is set, or v2 paxos variant is not being used", reason); + return; + } + + logger.info("repairing paxos for {}", reason); + + int retries = 0; + int maxRetries = Integer.getInteger("cassandra.paxos_repair_on_topology_change_retries", 10); + int delaySec = Integer.getInteger("cassandra.paxos_repair_on_topology_change_retry_delay_seconds", 10); + + boolean completed = false; + while (!completed) + { + try + { + tryRepairPaxosForTopologyChange(reason); + completed = true; + } + catch (Exception e) + { + if (retries >= maxRetries) + throw e; + + retries++; + int sleep = delaySec * retries; + logger.info("Sleeping {} seconds before retrying paxos repair...", sleep); + Uninterruptibles.sleepUninterruptibly(sleep, TimeUnit.SECONDS); + logger.info("Retrying paxos repair for {}. Retry {}/{}", reason, retries, maxRetries); + } + } + + logger.info("paxos repair for {} complete", reason); + } + + @VisibleForTesting + public Future startRepairPaxosForTopologyChange(String reason) + { + logger.info("repairing paxos for {}", reason); + + List> futures = new ArrayList<>(); + + List keyspaces = Schema.instance.getNonLocalStrategyKeyspaces() ; + for (String ksName : keyspaces) + { + if (SchemaConstants.REPLICATED_SYSTEM_KEYSPACE_NAMES.contains(ksName)) + continue; + + if (DatabaseDescriptor.skipPaxosRepairOnTopologyChangeKeyspaces().contains(ksName)) + continue; + + List> ranges = getLocalAndPendingRanges(ksName); + futures.add(ActiveRepairService.instance.repairPaxosForTopologyChange(ksName, ranges, reason)); + } + + return FutureCombiner.allOf(futures); + } + + public Future autoRepairPaxos(TableId tableId) + { + TableMetadata table = Schema.instance.getTableMetadata(tableId); + if (table == null) + return ImmediateFuture.success(null); + + List> ranges = getLocalAndPendingRanges(table.keyspace); + PaxosCleanupLocalCoordinator coordinator = PaxosCleanupLocalCoordinator.createForAutoRepair(tableId, ranges); + ScheduledExecutors.optionalTasks.submit(coordinator::start); + return coordinator; + } + public void forceTerminateAllRepairSessions() { ActiveRepairService.instance.terminateSessions(); } - // TODO: remove this after forward-porting Paxos, before final commit - public Future startRepairPaxosForTopologyChange(String reason) - { - // PLACEHOLDER METHOD for simple porting of Simulator without breaking correctness - return ImmediateFuture.success(null); - } - @Nullable public List getParentRepairStatus(int cmd) { @@ -4719,6 +4829,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE setMode(Mode.LEAVING, "replaying batch log and streaming data to other nodes", true); + repairPaxosForTopologyChange("decommission"); // Start with BatchLog replay, which may create hints but no writes since this is no longer a valid endpoint. Future batchlogReplay = BatchlogManager.instance.startBatchlogReplay(); Future streamSuccess = startStreaming.get(); @@ -4834,6 +4945,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE RangeRelocator relocator = new RangeRelocator(Collections.singleton(newToken), keyspacesToProcess, tokenMetadata); relocator.calculateToFromStreams(); + repairPaxosForTopologyChange("move"); if (relocator.streamsNeeded()) { setMode(Mode.MOVING, "fetching new ranges and streaming old ranges", true); @@ -6401,4 +6513,188 @@ public class StorageService extends NotificationBroadcasterSupport implements IE { return DatabaseDescriptor.getMinimumKeyspaceRF(); } + + public boolean getSkipPaxosRepairOnTopologyChange() + { + return DatabaseDescriptor.skipPaxosRepairOnTopologyChange(); + } + + public void setSkipPaxosRepairOnTopologyChange(boolean v) + { + DatabaseDescriptor.setSkipPaxosRepairOnTopologyChange(v); + logger.info("paxos skip topology change repair {} via jmx", v ? "enabled" : "disabled"); + } + + public String getSkipPaxosRepairOnTopologyChangeKeyspaces() + { + return Joiner.on(',').join(DatabaseDescriptor.skipPaxosRepairOnTopologyChangeKeyspaces()); + } + + public void setSkipPaxosRepairOnTopologyChangeKeyspaces(String v) + { + DatabaseDescriptor.setSkipPaxosRepairOnTopologyChangeKeyspaces(v); + logger.info("paxos skip topology change repair keyspaces set to {} via jmx", v); + } + + public int getPaxosAutoRepairThresholdMb() + { + return DatabaseDescriptor.getPaxosAutoRepairThresholdMB(); + } + + public void setPaxosAutoRepairThresholdMb(int threshold) + { + if (threshold < 0) + throw new RuntimeException("Paxos auto repair threshold must not be negative"); + + int oldThreshold = getPaxosAutoRepairThresholdMb(); + if (oldThreshold == threshold) + { + logger.info("Supplied paxos auto repair threshold is the same as current value"); + } + else + { + logger.info("Changing paxos auto repair threshold (MiB) from {} to {}", getPaxosAutoRepairThresholdMb(), threshold); + DatabaseDescriptor.setPaxosAutoRepairThresholdMB(threshold); + } + } + + public boolean getPaxosAutoRepairsEnabled() + { + return PaxosState.uncommittedTracker().isAutoRepairsEnabled(); + } + + public void setPaxosAutoRepairsEnabled(boolean enabled) + { + PaxosState.uncommittedTracker().setAutoRepairsEnabled(enabled); + logger.info("paxos auto repairs {} via jmx", enabled ? "enabled" : "disabled"); + } + + public boolean getPaxosStateFlushEnabled() + { + return PaxosState.uncommittedTracker().isStateFlushEnabled(); + } + + public void setPaxosStateFlushEnabled(boolean enabled) + { + PaxosState.uncommittedTracker().setStateFlushEnabled(enabled); + logger.info("paxos state flush {} via jmx", enabled ? "enabled" : "disabled"); + } + + public List getPaxosAutoRepairTables() + { + Set tableIds = PaxosState.uncommittedTracker().tableIds(); + List tables = new ArrayList<>(tableIds.size()); + for (TableId tableId : tableIds) + { + TableMetadata table = Schema.instance.getTableMetadata(tableId); + if (table == null) + continue; + tables.add(table.keyspace + '.' + table.name); + } + return tables; + } + + public long getPaxosPurgeGraceSeconds() + { + return DatabaseDescriptor.getPaxosPurgeGrace(SECONDS); + } + + public void setPaxosPurgeGraceSeconds(long v) + { + DatabaseDescriptor.setPaxosPurgeGrace(v, SECONDS); + logger.info("paxos purging grace seconds set to {} via jmx", v); + } + + public String getPaxosOnLinearizabilityViolations() + { + return DatabaseDescriptor.paxosOnLinearizabilityViolations().toString(); + } + + public void setPaxosOnLinearizabilityViolations(String v) + { + DatabaseDescriptor.setPaxosOnLinearizabilityViolations(Config.PaxosOnLinearizabilityViolation.valueOf(v)); + logger.info("paxos on linearizability violations {} via jmx", v); + } + + public String getPaxosStatePurging() + { + return DatabaseDescriptor.paxosStatePurging().name(); + } + + public void setPaxosStatePurging(String v) + { + DatabaseDescriptor.setPaxosStatePurging(PaxosStatePurging.valueOf(v)); + logger.info("paxos state purging {} via jmx", v); + } + + public boolean getPaxosRepairEnabled() + { + return DatabaseDescriptor.paxosRepairEnabled(); + } + + public void setPaxosRepairEnabled(boolean enabled) + { + DatabaseDescriptor.setPaxosRepairEnabled(enabled); + logger.info("paxos repair {} via jmx", enabled ? "enabled" : "disabled"); + } + + public boolean getPaxosDcLocalCommitEnabled() + { + return PaxosCommit.getEnableDcLocalCommit(); + } + + public void setPaxosDcLocalCommitEnabled(boolean enabled) + { + PaxosCommit.setEnableDcLocalCommit(enabled); + logger.info("paxos dc local commit {} via jmx", enabled ? "enabled" : "disabled"); + } + + public String getPaxosBallotLowBound(String ksName, String tblName, String key) + { + Keyspace keyspace = Keyspace.open(ksName); + if (keyspace == null) + throw new IllegalArgumentException("Unknown keyspace '" + ksName + "'"); + + ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(tblName); + if (cfs == null) + throw new IllegalArgumentException("Unknown table '" + tblName + "' in keyspace '" + ksName + "'"); + + TableMetadata table = cfs.metadata.get(); + DecoratedKey dk = table.partitioner.decorateKey(table.partitionKeyType.fromString(key)); + return cfs.getPaxosRepairHistory().ballotForToken(dk.getToken()).toString(); + } + + public Long getRepairRpcTimeout() + { + return DatabaseDescriptor.getRepairRpcTimeout(); + } + + public void setRepairRpcTimeout(Long timeoutInMillis) + { + Preconditions.checkState(timeoutInMillis > 0); + DatabaseDescriptor.setRepairRpcTimeout(timeoutInMillis); + logger.info("RepairRpcTimeout set to {}ms via JMX", timeoutInMillis); + } + public void evictHungRepairs() + { + logger.info("StorageService#clearPaxosRateLimiters called via jmx"); + Paxos.evictHungRepairs(); + } + + public void clearPaxosRepairs() + { + logger.info("StorageService#clearPaxosRepairs called via jmx"); + PaxosTableRepairs.clearRepairs(); + } + + public void setSkipPaxosRepairCompatibilityCheck(boolean v) + { + PaxosRepair.setSkipPaxosRepairCompatibilityCheck(v); + logger.info("SkipPaxosRepairCompatibilityCheck set to {} via jmx", v); + } + + public boolean getSkipPaxosRepairCompatibilityCheck() + { + return PaxosRepair.getSkipPaxosRepairCompatibilityCheck(); + } } diff --git a/src/java/org/apache/cassandra/service/StorageServiceMBean.java b/src/java/org/apache/cassandra/service/StorageServiceMBean.java index 213c5a920d..d87f5b5b7d 100644 --- a/src/java/org/apache/cassandra/service/StorageServiceMBean.java +++ b/src/java/org/apache/cassandra/service/StorageServiceMBean.java @@ -939,4 +939,46 @@ public interface StorageServiceMBean extends NotificationEmitter public int getDefaultKeyspaceReplicationFactor(); public void setMinimumKeyspaceReplicationFactor(int value); public int getMinimumKeyspaceReplicationFactor(); + + boolean getSkipPaxosRepairOnTopologyChange(); + void setSkipPaxosRepairOnTopologyChange(boolean v); + + String getSkipPaxosRepairOnTopologyChangeKeyspaces(); + void setSkipPaxosRepairOnTopologyChangeKeyspaces(String v); + + int getPaxosAutoRepairThresholdMb(); + void setPaxosAutoRepairThresholdMb(int threshold); + + boolean getPaxosAutoRepairsEnabled(); + void setPaxosAutoRepairsEnabled(boolean enabled); + + boolean getPaxosStateFlushEnabled(); + void setPaxosStateFlushEnabled(boolean enabled); + + List getPaxosAutoRepairTables(); + + long getPaxosPurgeGraceSeconds(); + void setPaxosPurgeGraceSeconds(long v); + + String getPaxosOnLinearizabilityViolations(); + void setPaxosOnLinearizabilityViolations(String v); + + String getPaxosStatePurging(); + void setPaxosStatePurging(String v); + + boolean getPaxosRepairEnabled(); + void setPaxosRepairEnabled(boolean v); + + boolean getPaxosDcLocalCommitEnabled(); + void setPaxosDcLocalCommitEnabled(boolean v); + + String getPaxosBallotLowBound(String keyspace, String table, String key); + + public Long getRepairRpcTimeout(); + public void setRepairRpcTimeout(Long timeoutInMillis); + + public void evictHungRepairs(); + public void clearPaxosRepairs(); + public void setSkipPaxosRepairCompatibilityCheck(boolean v); + public boolean getSkipPaxosRepairCompatibilityCheck(); } diff --git a/src/java/org/apache/cassandra/service/WriteResponseHandler.java b/src/java/org/apache/cassandra/service/WriteResponseHandler.java index 94f5a80729..6fe9e527cd 100644 --- a/src/java/org/apache/cassandra/service/WriteResponseHandler.java +++ b/src/java/org/apache/cassandra/service/WriteResponseHandler.java @@ -18,7 +18,9 @@ package org.apache.cassandra.service; import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; +import java.util.function.Supplier; +import org.apache.cassandra.db.Mutation; import org.apache.cassandra.locator.ReplicaPlan; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -37,18 +39,19 @@ public class WriteResponseHandler extends AbstractWriteResponseHandler private static final AtomicIntegerFieldUpdater responsesUpdater = AtomicIntegerFieldUpdater.newUpdater(WriteResponseHandler.class, "responses"); - public WriteResponseHandler(ReplicaPlan.ForTokenWrite replicaPlan, + public WriteResponseHandler(ReplicaPlan.ForWrite replicaPlan, Runnable callback, WriteType writeType, + Supplier hintOnFailure, long queryStartNanoTime) { - super(replicaPlan, callback, writeType, queryStartNanoTime); + super(replicaPlan, callback, writeType, hintOnFailure, queryStartNanoTime); responses = blockFor(); } - public WriteResponseHandler(ReplicaPlan.ForTokenWrite replicaPlan, WriteType writeType, long queryStartNanoTime) + public WriteResponseHandler(ReplicaPlan.ForWrite replicaPlan, WriteType writeType, Supplier hintOnFailure, long queryStartNanoTime) { - this(replicaPlan, null, writeType, queryStartNanoTime); + this(replicaPlan, null, writeType, hintOnFailure, queryStartNanoTime); } public void onResponse(Message m) diff --git a/src/java/org/apache/cassandra/service/paxos/AbstractPaxosRepair.java b/src/java/org/apache/cassandra/service/paxos/AbstractPaxosRepair.java new file mode 100644 index 0000000000..05c67da3f1 --- /dev/null +++ b/src/java/org/apache/cassandra/service/paxos/AbstractPaxosRepair.java @@ -0,0 +1,290 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.paxos; + +import java.io.PrintWriter; +import java.io.StringWriter; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.function.Consumer; + +import com.google.common.base.Preconditions; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.utils.Throwables; + +import static org.apache.cassandra.service.paxos.AbstractPaxosRepair.Result.Outcome; +import static org.apache.cassandra.utils.Clock.Global.nanoTime; + +public abstract class AbstractPaxosRepair +{ + private static final Logger logger = LoggerFactory.getLogger(AbstractPaxosRepair.class); + + public static class State {} + + public interface StateUpdater + { + State apply(S in, I param) throws T; + } + + public interface Listener + { + void onComplete(AbstractPaxosRepair repair, Result result); + } + + abstract class ConsumerState extends State implements Consumer + { + public void accept(T input) + { + updateState(this, input, ConsumerState::execute); + } + + abstract State execute(T input) throws Throwable; + } + + public static class Result extends State + { + enum Outcome { DONE, CANCELLED, FAILURE } + + final Outcome outcome; + + public Result(Outcome outcome) + { + this.outcome = outcome; + } + + public String toString() + { + return outcome.toString(); + } + + public boolean wasSuccessful() + { + return outcome == Outcome.DONE; + } + } + + static boolean isResult(State state) + { + return state instanceof Result; + } + + public static final Result DONE = new Result(Outcome.DONE); + + public static final Result CANCELLED = new Result(Outcome.CANCELLED); + + public static final class Failure extends Result + { + public final Throwable failure; + + Failure(Throwable failure) + { + super(Outcome.FAILURE); + this.failure = failure; + } + + public String toString() + { + StringWriter sw = new StringWriter(); + PrintWriter pw = new PrintWriter(sw); + if (failure != null) + failure.printStackTrace(pw); + return outcome.toString() + ": " + sw; + } + + public boolean equals(Object o) + { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Failure failure1 = (Failure) o; + return Objects.equals(failure, failure1.failure); + } + + public int hashCode() + { + return Objects.hash(failure); + } + } + + private final DecoratedKey partitionKey; + private final Ballot incompleteBallot; + private List listeners = null; + private volatile State state; + private volatile long startedNanos = Long.MIN_VALUE; + + public AbstractPaxosRepair(DecoratedKey partitionKey, Ballot incompleteBallot) + { + this.partitionKey = partitionKey; + this.incompleteBallot = incompleteBallot; + } + + public State state() + { + return state; + } + + public long startedNanos() + { + return startedNanos; + } + + public boolean isStarted() + { + return startedNanos != Long.MIN_VALUE; + } + + public boolean isComplete() + { + return isResult(state); + } + + public Ballot incompleteBallot() + { + return incompleteBallot; + } + + /** + * add a listener to this repair, or if the repair has already completed, call the listener with the result + */ + public AbstractPaxosRepair addListener(Listener listener) + { + Result result = null; + synchronized (this) + { + if (isResult(state)) + { + result = (Result) state; + } + else + { + if (listeners == null) + listeners = new ArrayList<>(); + + listeners.add(listener); + } + } + + if (result != null) + listener.onComplete(this, result); + + return this; + } + + public AbstractPaxosRepair addListener(Consumer listener) + { + return addListener(((repair, result) -> listener.accept(result))); + } + + public final DecoratedKey partitionKey() + { + return partitionKey; + } + + public State restart(State state) { return restart(state, Long.MIN_VALUE); } + public abstract State restart(State state, long waitUntil); + + public final synchronized AbstractPaxosRepair start() + { + updateState(null, null, (state, i2) -> { + Preconditions.checkState(!isStarted()); + startedNanos = Math.max(Long.MIN_VALUE + 1, nanoTime()); + return restart(state); + }); + return this; + } + + public final void cancel() + { + set(CANCELLED); + } + + public final void cancelUnexceptionally() + { + try + { + cancel(); + } + catch (Throwable t) + { + logger.error("Exception cancelling paxos repair", t); + } + } + + public final synchronized Result await() throws InterruptedException + { + while (!isResult(state)) + wait(); + return (Result) state; + } + + protected void set(Result result) + { + updateState(state(), null, (i1, i2) -> result); + } + + protected void updateState(S expect, I param, StateUpdater transform) + { + Result result = null; + List listeners = null; + synchronized (this) + { + State next; + try + { + if (state != expect) + return; + + state = next = transform.apply(expect, param); + } + catch (Throwable t) + { + state = next = new Failure(t); + } + + if (isResult(next)) + { + notifyAll(); + result = (Result) next; + listeners = this.listeners; + this.listeners = null; + } + } + + if (result != null && listeners != null) + { + Throwable t = null; + for (int i=0, mi=listeners.size(); i implements IVersionedSerializer + { + public static final Serializer instance = new Serializer(); + + public Ballot deserialize(V value, ValueAccessor accessor) + { + return accessor.isEmpty(value) ? null : accessor.toBallot(value); + } + + public Class getType() + { + return Ballot.class; + } + + @Override + public void serialize(Ballot t, DataOutputPlus out, int version) throws IOException + { + t.serialize(out); + } + + @Override + public Ballot deserialize(DataInputPlus in, int version) throws IOException + { + return Ballot.deserialize(in); + } + + @Override + public long serializedSize(Ballot t, int version) + { + return sizeInBytes(); + } + } + +} diff --git a/src/java/org/apache/cassandra/service/paxos/BallotGenerator.java b/src/java/org/apache/cassandra/service/paxos/BallotGenerator.java index 2355bbe20b..f746c4edec 100644 --- a/src/java/org/apache/cassandra/service/paxos/BallotGenerator.java +++ b/src/java/org/apache/cassandra/service/paxos/BallotGenerator.java @@ -19,13 +19,13 @@ package org.apache.cassandra.service.paxos; import java.security.SecureRandom; -import java.util.UUID; import java.util.concurrent.ThreadLocalRandom; import org.apache.cassandra.service.ClientState; +import org.apache.cassandra.service.paxos.Ballot.Flag; import org.apache.cassandra.utils.Shared; -import org.apache.cassandra.utils.TimeUUID; +import static org.apache.cassandra.service.paxos.Ballot.atUnixMicrosWithLsb; import static org.apache.cassandra.utils.Shared.Scope.SIMULATION; @Shared(scope = SIMULATION) @@ -35,23 +35,29 @@ public interface BallotGenerator { private static final SecureRandom secureRandom = new SecureRandom(); - public TimeUUID randomBallot(long whenInMicros, boolean isSerial) + public Ballot atUnixMicros(long unixMicros, Flag flag) { - return TimeUUID.atUnixMicrosWithLsb(whenInMicros, secureRandom.nextLong(), isSerial); + return atUnixMicrosWithLsb(unixMicros, secureRandom.nextLong(), flag); } - public TimeUUID randomBallot(long fromInMicros, long toInMicros, boolean isSerial) + public Ballot next(long minUnixMicros, Flag flag) { - long timestampMicros = ThreadLocalRandom.current().nextLong(fromInMicros, toInMicros); - return randomBallot(timestampMicros, isSerial); + long unixMicros = ClientState.getTimestampForPaxos(minUnixMicros); + return atUnixMicros(unixMicros, flag); } - public long nextBallotTimestampMicros(long minTimestamp) + public Ballot stale(long fromInMicros, long toInMicros, Flag flag) + { + long unixMicros = ThreadLocalRandom.current().nextLong(fromInMicros, toInMicros); + return atUnixMicros(unixMicros, flag); + } + + public long next(long minTimestamp) { return ClientState.getTimestampForPaxos(minTimestamp); } - public long prevBallotTimestampMicros() + public long prevUnixMicros() { return ClientState.getLastTimestampMicros(); } @@ -60,10 +66,11 @@ public interface BallotGenerator static class Global { private static BallotGenerator instance = new Default(); - public static TimeUUID randomBallot(long whenInMicros, boolean isSerial) { return instance.randomBallot(whenInMicros, isSerial); } - public static TimeUUID randomBallot(long fromInMicros, long toInMicros, boolean isSerial) { return instance.randomBallot(fromInMicros, toInMicros, isSerial); } - public static long nextBallotTimestampMicros(long minWhenInMicros) { return instance.nextBallotTimestampMicros(minWhenInMicros); } - public static long prevBallotTimestampMicros() { return instance.prevBallotTimestampMicros(); } + public static Ballot atUnixMicros(long unixMicros, Flag flag) { return instance.atUnixMicros(unixMicros, flag); } + public static Ballot nextBallot(Flag flag) { return instance.next(Long.MIN_VALUE, flag); } + public static Ballot nextBallot(long minUnixMicros, Flag flag) { return instance.next(minUnixMicros, flag); } + public static Ballot staleBallot(long fromUnixMicros, long toUnixMicros, Flag flag) { return instance.stale(fromUnixMicros, toUnixMicros, flag); } + public static long prevUnixMicros() { return instance.prevUnixMicros(); } public static void unsafeSet(BallotGenerator newInstance) { @@ -71,8 +78,8 @@ public interface BallotGenerator } } - TimeUUID randomBallot(long whenInMicros, boolean isSerial); - TimeUUID randomBallot(long fromInMicros, long toInMicros, boolean isSerial); - long nextBallotTimestampMicros(long minWhenInMicros); - long prevBallotTimestampMicros(); + Ballot atUnixMicros(long unixMicros, Flag flag); + Ballot next(long minUnixMicros, Flag flag); + Ballot stale(long fromUnixMicros, long toUnixMicros, Flag flag); + long prevUnixMicros(); } \ No newline at end of file diff --git a/src/java/org/apache/cassandra/service/paxos/Commit.java b/src/java/org/apache/cassandra/service/paxos/Commit.java index 2ada45b080..88ab5ea29c 100644 --- a/src/java/org/apache/cassandra/service/paxos/Commit.java +++ b/src/java/org/apache/cassandra/service/paxos/Commit.java @@ -22,31 +22,235 @@ package org.apache.cassandra.service.paxos; import java.io.IOException; -import java.util.UUID; +import java.util.function.BiFunction; import javax.annotation.Nullable; import com.google.common.base.Objects; -import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.db.*; import org.apache.cassandra.db.rows.*; import org.apache.cassandra.db.partitions.PartitionUpdate; import org.apache.cassandra.io.IVersionedSerializer; import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputPlus; -import org.apache.cassandra.utils.TimeUUID; -import org.apache.cassandra.utils.UUIDGen; -import org.apache.cassandra.utils.UUIDSerializer; +import org.apache.cassandra.schema.TableMetadata; + +import static org.apache.cassandra.db.SystemKeyspace.*; +import static org.apache.cassandra.service.paxos.Commit.CompareResult.AFTER; +import static org.apache.cassandra.service.paxos.Commit.CompareResult.BEFORE; +import static org.apache.cassandra.service.paxos.Commit.CompareResult.IS_REPROPOSAL; +import static org.apache.cassandra.service.paxos.Commit.CompareResult.WAS_REPROPOSED_BY; +import static org.apache.cassandra.service.paxos.Commit.CompareResult.SAME; +import static org.apache.cassandra.utils.FBUtilities.nowInSeconds; public class Commit { - public static final CommitSerializer serializer = new CommitSerializer(); + enum CompareResult { SAME, BEFORE, AFTER, IS_REPROPOSAL, WAS_REPROPOSED_BY} - public final TimeUUID ballot; + public static final CommitSerializer serializer = new CommitSerializer<>(Commit::new); + + public static class Proposal extends Commit + { + public static final CommitSerializer serializer = new CommitSerializer<>(Proposal::new); + + public Proposal(Ballot ballot, PartitionUpdate update) + { + super(ballot, update); + } + + public String toString() + { + return toString("Proposal"); + } + + public static Proposal of(Ballot ballot, PartitionUpdate update) + { + update = withTimestamp(update, ballot.unixMicros()); + return new Proposal(ballot, update); + } + + public static Proposal empty(Ballot ballot, DecoratedKey partitionKey, TableMetadata metadata) + { + return new Proposal(ballot, PartitionUpdate.emptyUpdate(metadata, partitionKey)); + } + + public Accepted accepted() + { + return new Accepted(ballot, update); + } + + public Agreed agreed() + { + return new Agreed(ballot, update); + } + } + + public static class Accepted extends Proposal + { + public static final CommitSerializer serializer = new CommitSerializer<>(Accepted::new); + + public static Accepted none(DecoratedKey partitionKey, TableMetadata metadata) + { + return new Accepted(Ballot.none(), PartitionUpdate.emptyUpdate(metadata, partitionKey)); + } + + public Accepted(Ballot ballot, PartitionUpdate update) + { + super(ballot, update); + } + + public Accepted(Commit commit) + { + super(commit.ballot, commit.update); + } + + Committed committed() + { + return new Committed(ballot, update); + } + + boolean isExpired(int nowInSec) + { + return false; + } + + public String toString() + { + return toString("Accepted"); + } + + /** + * Like {@link #latest(Commit, Commit)} but also takes into account deletion time + */ + public static Accepted latestAccepted(Accepted a, Accepted b) + { + int c = compare(a, b); + if (c != 0) + return c > 0 ? a : b; + return a instanceof AcceptedWithTTL ? ((AcceptedWithTTL)a).lastDeleted(b) : a; + } + } + + public static class AcceptedWithTTL extends Accepted + { + public static AcceptedWithTTL withDefaultTTL(Commit copy) + { + return new AcceptedWithTTL(copy, nowInSeconds() + legacyPaxosTtlSec(copy.update.metadata())); + } + + public final int localDeletionTime; + + public AcceptedWithTTL(Commit copy, int localDeletionTime) + { + super(copy); + this.localDeletionTime = localDeletionTime; + } + + public AcceptedWithTTL(Ballot ballot, PartitionUpdate update, int localDeletionTime) + { + super(ballot, update); + this.localDeletionTime = localDeletionTime; + } + + boolean isExpired(int nowInSec) + { + return nowInSec >= localDeletionTime; + } + + Accepted lastDeleted(Accepted b) + { + return b instanceof AcceptedWithTTL && localDeletionTime >= ((AcceptedWithTTL) b).localDeletionTime + ? this : b; + } + } + + // might prefer to call this Commit, but would mean refactoring more legacy code + public static class Agreed extends Accepted + { + public static final CommitSerializer serializer = new CommitSerializer<>(Agreed::new); + + public Agreed(Ballot ballot, PartitionUpdate update) + { + super(ballot, update); + } + + public Agreed(Commit copy) + { + super(copy); + } + } + + public static class Committed extends Agreed + { + public static final CommitSerializer serializer = new CommitSerializer<>(Committed::new); + + public static Committed none(DecoratedKey partitionKey, TableMetadata metadata) + { + return new Committed(Ballot.none(), PartitionUpdate.emptyUpdate(metadata, partitionKey)); + } + + public Committed(Ballot ballot, PartitionUpdate update) + { + super(ballot, update); + } + + public Committed(Commit copy) + { + super(copy); + } + + public String toString() + { + return toString("Committed"); + } + + public static Committed latestCommitted(Committed a, Committed b) + { + int c = compare(a, b); + if (c != 0) + return c > 0 ? a : b; + return a instanceof CommittedWithTTL ? ((CommittedWithTTL)a).lastDeleted(b) : a; + } + } + + public static class CommittedWithTTL extends Committed + { + public static CommittedWithTTL withDefaultTTL(Commit copy) + { + return new CommittedWithTTL(copy, nowInSeconds() + legacyPaxosTtlSec(copy.update.metadata())); + } + + public final int localDeletionTime; + + public CommittedWithTTL(Ballot ballot, PartitionUpdate update, int localDeletionTime) + { + super(ballot, update); + this.localDeletionTime = localDeletionTime; + } + + public CommittedWithTTL(Commit copy, int localDeletionTime) + { + super(copy); + this.localDeletionTime = localDeletionTime; + } + + boolean isExpired(int nowInSec) + { + return nowInSec >= localDeletionTime; + } + + Committed lastDeleted(Committed b) + { + return b instanceof CommittedWithTTL && localDeletionTime >= ((CommittedWithTTL) b).localDeletionTime + ? this : b; + } + } + + public final Ballot ballot; public final PartitionUpdate update; - public Commit(TimeUUID ballot, PartitionUpdate update) + public Commit(Ballot ballot, PartitionUpdate update) { assert ballot != null; assert update != null; @@ -55,36 +259,51 @@ public class Commit this.update = update; } - public static Commit newPrepare(DecoratedKey key, TableMetadata metadata, TimeUUID ballot) + public static Commit newPrepare(DecoratedKey partitionKey, TableMetadata metadata, Ballot ballot) { - return new Commit(ballot, PartitionUpdate.emptyUpdate(metadata, key)); + return new Commit(ballot, PartitionUpdate.emptyUpdate(metadata, partitionKey)); } - public static Commit newProposal(TimeUUID ballot, PartitionUpdate update) + public static Commit emptyCommit(DecoratedKey partitionKey, TableMetadata metadata) { - PartitionUpdate withNewTimestamp = new PartitionUpdate.Builder(update, 0).updateAllTimestamp(ballot.unixMicros()).build(); - return new Commit(ballot, withNewTimestamp); + return new Commit(Ballot.none(), PartitionUpdate.emptyUpdate(metadata, partitionKey)); } - public static Commit emptyCommit(DecoratedKey key, TableMetadata metadata) + @Deprecated + public static Commit newProposal(Ballot ballot, PartitionUpdate update) { - return new Commit(TimeUUID.minAtUnixMillis(0), PartitionUpdate.emptyUpdate(metadata, key)); + update = withTimestamp(update, ballot.unixMicros()); + return new Commit(ballot, update); } public boolean isAfter(Commit other) { - return ballot.rawTimestamp() > other.ballot.rawTimestamp(); + return other == null || ballot.uuidTimestamp() > other.ballot.uuidTimestamp(); } - public boolean hasBallot(TimeUUID ballot) + public boolean isSameOrAfter(@Nullable Ballot otherBallot) + { + return otherBallot == null || otherBallot.equals(ballot) || ballot.uuidTimestamp() > otherBallot.uuidTimestamp(); + } + + public boolean isAfter(@Nullable Ballot otherBallot) + { + return otherBallot == null || ballot.uuidTimestamp() > otherBallot.uuidTimestamp(); + } + + public boolean isBefore(@Nullable Ballot otherBallot) + { + return otherBallot != null && ballot.uuidTimestamp() < otherBallot.uuidTimestamp(); + } + + public boolean hasBallot(Ballot ballot) { return this.ballot.equals(ballot); } - /** Whether this is an empty commit, that is one with no updates. */ - public boolean isEmpty() + public boolean hasSameBallot(Commit other) { - return update.isEmpty(); + return this.ballot.equals(other.ballot); } public Mutation makeMutation() @@ -112,7 +331,68 @@ public class Commit @Override public String toString() { - return String.format("Commit(%s, %s)", ballot, update); + return toString("Commit"); + } + + public String toString(String kind) + { + return String.format("%s(%d:%s, %d:%s)", kind, ballot.uuidTimestamp(), ballot, update.stats().minTimestamp, update.toString(false)); + } + + /** + * We can witness reproposals of the latest successful commit; we can detect this by comparing the timestamp of + * the update with our ballot; if it is the same, we are not a reproposal. If it is the same as either the + * ballot timestamp or update timestamp of the latest committed proposal, then we are reproposing it and can + * instead simpy commit it. + */ + public boolean isReproposalOf(Commit older) + { + return isReproposal(older, older.ballot.uuidTimestamp(), this, this.ballot.uuidTimestamp()); + } + + private boolean isReproposal(Commit older, long ballotOfOlder, Commit newer, long ballotOfNewer) + { + // NOTE: it would in theory be possible to just check + // newer.update.stats().minTimestamp == older.update.stats().minTimestamp + // however this could be brittle, if for some reason they don't get updated; + // the logic below is fail-safe, in that if minTimestamp is not set we will treat it as not a reproposal + // which is the safer way to get it wrong. + + // the timestamp of a mutation stays unchanged as we repropose it, so the timestamp of the mutation + // is the timestamp of the ballot that originally proposed it + long originalBallotOfNewer = newer.update.stats().minTimestamp; + + // so, if the mutation and ballot timestamps match, this is not a reproposal but a first proposal + if (ballotOfNewer == originalBallotOfNewer) + return false; + + // otherwise, if the original proposing ballot matches the older proposal's ballot, it is reproposing it + if (originalBallotOfNewer == ballotOfOlder) + return true; + + // otherwise, it could be that both are reproposals, so just check both for the "original" ballot timestamp + return originalBallotOfNewer == older.update.stats().minTimestamp; + } + + public CompareResult compareWith(Commit that) + { + long thisBallot = this.ballot.uuidTimestamp(); + long thatBallot = that.ballot.uuidTimestamp(); + // by the time we reach proposal and commit, timestamps are unique so we can assert identity + if (thisBallot == thatBallot) + return SAME; + + if (thisBallot < thatBallot) + return isReproposal(this, thisBallot, that, thatBallot) ? WAS_REPROPOSED_BY : BEFORE; + else + return isReproposal(that, thatBallot, this, thisBallot) ? IS_REPROPOSAL : AFTER; + } + + private static int compare(@Nullable Commit a, @Nullable Commit b) + { + if (a == null) return 1; + if (b == null) return -1; + return Long.compare(a.ballot.uuidTimestamp(), b.ballot.uuidTimestamp()); } /** @@ -126,46 +406,95 @@ public class Commit /** * @return testIfAfter.isAfter(testIfBefore), with non-null > null */ - public static boolean isAfter(@Nullable TimeUUID testIsAfter, @Nullable Commit testIsBefore) + public static boolean isAfter(@Nullable Ballot testIsAfter, @Nullable Commit testIsBefore) { - return testIsAfter != null && (testIsBefore == null || testIsAfter.rawTimestamp() > testIsBefore.ballot.rawTimestamp()); + return testIsAfter != null && (testIsBefore == null || testIsAfter.uuidTimestamp() > testIsBefore.ballot.uuidTimestamp()); } /** * @return testIfAfter.isAfter(testIfBefore), with non-null > null */ - public static boolean isAfter(@Nullable Commit testIsAfter, @Nullable TimeUUID testIsBefore) + public static boolean isAfter(@Nullable Commit testIsAfter, @Nullable Ballot testIsBefore) { - return testIsAfter != null && (testIsBefore == null || testIsAfter.ballot.rawTimestamp() > testIsBefore.rawTimestamp()); + return testIsAfter != null && (testIsBefore == null || testIsAfter.ballot.uuidTimestamp() > testIsBefore.uuidTimestamp()); } /** * @return testIfAfter.isAfter(testIfBefore), with non-null > null */ - public static boolean isAfter(@Nullable UUID testIsAfter, @Nullable UUID testIsBefore) + public static boolean isAfter(@Nullable Ballot testIsAfter, @Nullable Ballot testIsBefore) { - return testIsAfter != null && (testIsBefore == null || testIsAfter.timestamp() > testIsBefore.timestamp()); + return testIsAfter != null && (testIsBefore == null || testIsAfter.uuidTimestamp() > testIsBefore.uuidTimestamp()); } - public static class CommitSerializer implements IVersionedSerializer + /** + * the latest of two ballots, or the first ballot if equal timestamps + */ + public static C latest(@Nullable C a, @Nullable C b) { - public void serialize(Commit commit, DataOutputPlus out, int version) throws IOException + return (a == null | b == null) ? (a == null ? b : a) : a.ballot.uuidTimestamp() >= b.ballot.uuidTimestamp() ? a : b; + } + + /** + * the latest of two ballots, or the first ballot if equal timestamps + */ + public static Ballot latest(@Nullable Commit a, @Nullable Ballot b) + { + return (a == null | b == null) ? (a == null ? b : a.ballot) : a.ballot.uuidTimestamp() >= b.uuidTimestamp() ? a.ballot : b; + } + + /** + * the latest of two ballots, or the first ballot if equal timestamps + */ + public static Ballot latest(@Nullable Ballot a, @Nullable Ballot b) + { + return (a == null | b == null) ? (a == null ? b : a) : a.uuidTimestamp() >= b.uuidTimestamp() ? a : b; + } + + /** + * unequal ballots with same timestamp + */ + public static boolean timestampsClash(@Nullable Commit a, @Nullable Ballot b) + { + return a != null && b != null && !a.ballot.equals(b) && a.ballot.uuidTimestamp() == b.uuidTimestamp(); + } + + public static boolean timestampsClash(@Nullable Ballot a, @Nullable Ballot b) + { + return a != null && b != null && !a.equals(b) && a.uuidTimestamp() == b.uuidTimestamp(); + } + + private static PartitionUpdate withTimestamp(PartitionUpdate update, long timestamp) + { + return new PartitionUpdate.Builder(update, 0).updateAllTimestamp(timestamp).build(); + } + + public static class CommitSerializer implements IVersionedSerializer + { + final BiFunction constructor; + public CommitSerializer(BiFunction constructor) + { + this.constructor = constructor; + } + + public void serialize(T commit, DataOutputPlus out, int version) throws IOException { commit.ballot.serialize(out); PartitionUpdate.serializer.serialize(commit.update, out, version); } - public Commit deserialize(DataInputPlus in, int version) throws IOException + public T deserialize(DataInputPlus in, int version) throws IOException { - TimeUUID ballot = TimeUUID.deserialize(in); + Ballot ballot = Ballot.deserialize(in); PartitionUpdate update = PartitionUpdate.serializer.deserialize(in, version, DeserializationHelper.Flag.LOCAL); - return new Commit(ballot, update); + return constructor.apply(ballot, update); } - public long serializedSize(Commit commit, int version) + public long serializedSize(T commit, int version) { - return TimeUUID.sizeInBytes() - + PartitionUpdate.serializer.serializedSize(commit.update, version); + return Ballot.sizeInBytes() + + PartitionUpdate.serializer.serializedSize(commit.update, version); } } + } diff --git a/src/java/org/apache/cassandra/service/paxos/CommitVerbHandler.java b/src/java/org/apache/cassandra/service/paxos/CommitVerbHandler.java deleted file mode 100644 index 12466ddc69..0000000000 --- a/src/java/org/apache/cassandra/service/paxos/CommitVerbHandler.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * - */ -package org.apache.cassandra.service.paxos; - -import org.apache.cassandra.net.IVerbHandler; -import org.apache.cassandra.net.Message; -import org.apache.cassandra.net.MessagingService; -import org.apache.cassandra.tracing.Tracing; - -public class CommitVerbHandler implements IVerbHandler -{ - public static final CommitVerbHandler instance = new CommitVerbHandler(); - - public void doVerb(Message message) - { - PaxosState.commit(message.payload); - - Tracing.trace("Enqueuing acknowledge to {}", message.from()); - MessagingService.instance().send(message.emptyResponse(), message.from()); - } -} diff --git a/src/java/org/apache/cassandra/service/paxos/ContentionStrategy.java b/src/java/org/apache/cassandra/service/paxos/ContentionStrategy.java new file mode 100644 index 0000000000..7d0cb74924 --- /dev/null +++ b/src/java/org/apache/cassandra/service/paxos/ContentionStrategy.java @@ -0,0 +1,651 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.paxos; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableMap; + +import com.codahale.metrics.Snapshot; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.db.ConsistencyLevel; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.tracing.Tracing; +import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.utils.NoSpamLogger; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.DoubleSupplier; +import java.util.function.LongBinaryOperator; +import java.util.function.Supplier; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static java.lang.Double.parseDouble; +import static java.lang.Integer.parseInt; +import static java.lang.Math.*; +import static java.util.Arrays.stream; +import static java.util.concurrent.TimeUnit.*; +import static org.apache.cassandra.config.DatabaseDescriptor.*; +import static org.apache.cassandra.metrics.ClientRequestsMetricsHolder.casReadMetrics; +import static org.apache.cassandra.metrics.ClientRequestsMetricsHolder.casWriteMetrics; +import static org.apache.cassandra.utils.Clock.Global.nanoTime; +import static org.apache.cassandra.utils.Clock.waitUntil; + +/** + *

    A strategy for making back-off decisions for Paxos operations that fail to make progress because of other paxos operations. + * The strategy is defined by four factors:

      + *
    • {@link #min} + *
    • {@link #max} + *
    • {@link #minDelta} + *
    • {@link #waitRandomizer} + *
    + * + *

    The first three represent time periods, and may be defined dynamically based on a simple calculation over:

      + *
    • {@code pX()} recent experienced latency distribution for successful operations, + * e.g. {@code p50(rw)} the maximum of read and write median latencies, + * {@code p999(r)} the 99.9th percentile of read latencies + *
    • {@code attempts} the number of failed attempts made by the operation so far + *
    • {@code constant} a user provided floating point constant + *
    + * + *

    Their calculation may take any of these forms + *

  3. constant {@code $constant$[mu]s} + *
  4. dynamic constant {@code pX() * constant} + *
  5. dynamic linear {@code pX() * constant * attempts} + *
  6. dynamic exponential {@code pX() * constant ^ attempts} + * + *

    Furthermore, the dynamic calculations can be bounded with a min/max, like so: + * {@code min[mu]s <= dynamic expr <= max[mu]s} + * + * e.g. + *

  7. {@code 10ms <= p50(rw)*0.66} + *
  8. {@code 10ms <= p95(rw)*1.8^attempts <= 100ms} + *
  9. {@code 5ms <= p50(rw)*0.5} + * + *

    These calculations are put together to construct a range from which we draw a random number. + * The period we wait for {@code X} will be drawn so that {@code min <= X < max}. + * + *

    With the constraint that {@code max} must be {@code minDelta} greater than {@code min}, + * but no greater than its expression-defined maximum. {@code max} will be increased up until + * this point, after which {@code min} will be decreased until this gap is imposed. + * + *

    The {@link #waitRandomizer} property specifies the manner in which a random value is drawn from the range. + * It is defined using one of the following specifiers: + *

  10. uniform + *
  11. exp($power$) or exponential($power$) + *
  12. qexp($power$) or qexponential($power$) or quantizedexponential($power$) + * + * The uniform specifier is self-explanatory, selecting all values in the range with equal probability. + * The exponential specifier draws values towards the end of the range with higher probability, raising + * a floating point number in the range [0..1.0) to the power provided, and translating the resulting value + * to a uniform value in the range. + * The quantized exponential specifier partitions the range into {@code attempts} buckets, then applies the pure + * exponential approach to draw values from [0..attempts), before drawing a uniform value from the corresponding bucket + * + *

    Finally, there is also a {@link #traceAfterAttempts} property that permits initiating tracing of operations + * that experience a certain minimum number of failed paxos rounds due to contention. A setting of 0 or 1 will initiate + * a trace session after the first failed ballot. + */ +public class ContentionStrategy +{ + private static final Logger logger = LoggerFactory.getLogger(ContentionStrategy.class); + + private static final Pattern BOUND = Pattern.compile( + "(?0|[0-9]+[mu]s)" + + "|((?0|[0-9]+[mu]s) *<= *)?" + + "(p(?[0-9]+)\\((?r|w|rw|wr)\\)|(?0|[0-9]+[mu]s))" + + "\\s*([*]\\s*(?[0-9.]+)?\\s*(?[*^]\\s*attempts)?)?" + + "( *<= *(?0|[0-9]+[mu]s))?"); + private static final Pattern TIME = Pattern.compile( + "0|([0-9]+)ms|([0-9]+)us"); + private static final Pattern RANDOMIZER = Pattern.compile( + "uniform|exp(onential)?[(](?[0-9.]+)[)]|q(uantized)?exp(onential)?[(](?[0-9.]+)[)]"); + private static final String DEFAULT_WAIT_RANDOMIZER = "qexp(1.5)"; // at least 0ms, and at least 66% of median latency + private static final String DEFAULT_MIN = "0 <= p50(rw)*0.66"; // at least 0ms, and at least 66% of median latency + private static final String DEFAULT_MAX = "10ms <= p95(rw)*1.8^attempts <= 100ms"; // p95 latency with exponential back-off at rate of 1.8^attempts + private static final String DEFAULT_MIN_DELTA = "5ms <= p50(rw)*0.5"; // at least 5ms, and at least 50% of median latency + + private static volatile ContentionStrategy current; + + // Factories can be useful for testing purposes, to supply custom implementations of selectors and modifiers. + final static LatencySelectorFactory selectors = new LatencySelectorFactory(){}; + final static LatencyModifierFactory modifiers = new LatencyModifierFactory(){}; + final static WaitRandomizerFactory randomizers = new WaitRandomizerFactory(){}; + + static + { + current = new ContentionStrategy(defaultWaitRandomizer(), defaultMinWait(), defaultMaxWait(), defaultMinDelta(), Integer.MAX_VALUE); + } + + static interface LatencyModifierFactory + { + default LatencyModifier identity() { return (l, a) -> l; } + default LatencyModifier multiply(double constant) { return (l, a) -> saturatedCast(l * constant); } + default LatencyModifier multiplyByAttempts(double multiply) { return (l, a) -> saturatedCast(l * multiply * a); } + default LatencyModifier multiplyByAttemptsExp(double base) { return (l, a) -> saturatedCast(l * pow(base, a)); } + } + + static interface LatencySupplier + { + abstract long get(double percentile); + } + + static interface LatencySelector + { + abstract long select(LatencySupplier readLatencyHistogram, LatencySupplier writeLatencyHistogram); + } + + static interface LatencySelectorFactory + { + default LatencySelector constant(long latency) { return (read, write) -> latency; } + default LatencySelector read(double percentile) { return (read, write) -> read.get(percentile); } + default LatencySelector write(double percentile) { return (read, write) -> write.get(percentile); } + default LatencySelector maxReadWrite(double percentile) { return (read, write) -> max(read.get(percentile), write.get(percentile)); } + } + + static interface LatencyModifier + { + long modify(long latency, int attempts); + } + + static interface WaitRandomizer + { + abstract long wait(long min, long max, int attempts); + } + + static interface WaitRandomizerFactory + { + default LongBinaryOperator uniformLongSupplier() { return (min, max) -> ThreadLocalRandom.current().nextLong(min, max); } // DO NOT USE METHOD HANDLES (want to fetch afresh each time) + default DoubleSupplier uniformDoubleSupplier() { return () -> ThreadLocalRandom.current().nextDouble(); } + + default WaitRandomizer uniform() { return new Uniform(uniformLongSupplier()); } + default WaitRandomizer exponential(double power) { return new Exponential(uniformLongSupplier(), uniformDoubleSupplier(), power); } + default WaitRandomizer quantizedExponential(double power) { return new QuantizedExponential(uniformLongSupplier(), uniformDoubleSupplier(), power); } + + static class Uniform implements WaitRandomizer + { + final LongBinaryOperator uniformLong; + + public Uniform(LongBinaryOperator uniformLong) + { + this.uniformLong = uniformLong; + } + + @Override + public long wait(long min, long max, int attempts) + { + return uniformLong.applyAsLong(min, max); + } + } + + static abstract class AbstractExponential implements WaitRandomizer + { + final LongBinaryOperator uniformLong; + final DoubleSupplier uniformDouble; + final double power; + + public AbstractExponential(LongBinaryOperator uniformLong, DoubleSupplier uniformDouble, double power) + { + this.uniformLong = uniformLong; + this.uniformDouble = uniformDouble; + this.power = power; + } + } + + static class Exponential extends AbstractExponential + { + public Exponential(LongBinaryOperator uniformLong, DoubleSupplier uniformDouble, double power) + { + super(uniformLong, uniformDouble, power); + } + + @Override + public long wait(long min, long max, int attempts) + { + if (attempts == 1) + return uniformLong.applyAsLong(min, max); + + double p = uniformDouble.getAsDouble(); + long delta = max - min; + delta *= Math.pow(p, power); + return max - delta; + } + } + + static class QuantizedExponential extends AbstractExponential + { + public QuantizedExponential(LongBinaryOperator uniformLong, DoubleSupplier uniformDouble, double power) + { + super(uniformLong, uniformDouble, power); + } + + @Override + public long wait(long min, long max, int attempts) + { + long quanta = (max - min) / attempts; + if (attempts == 1 || quanta == 0) + return uniformLong.applyAsLong(min, max); + + double p = uniformDouble.getAsDouble(); + int base = (int) (attempts * Math.pow(p, power)); + return max - ThreadLocalRandom.current().nextLong(quanta * base, quanta * (base + 1)); + } + } + } + + static class SnapshotAndTime + { + final long validUntil; + final Snapshot snapshot; + + SnapshotAndTime(long validUntil, Snapshot snapshot) + { + this.validUntil = validUntil; + this.snapshot = snapshot; + } + } + + static class TimeLimitedLatencySupplier extends AtomicReference implements LatencySupplier + { + final Supplier snapshotSupplier; + final long validForNanos; + + TimeLimitedLatencySupplier(Supplier snapshotSupplier, long time, TimeUnit units) + { + this.snapshotSupplier = snapshotSupplier; + this.validForNanos = units.toNanos(time); + } + + private Snapshot getSnapshot() + { + long now = nanoTime(); + + SnapshotAndTime cur = get(); + if (cur != null && cur.validUntil > now) + return cur.snapshot; + + Snapshot newSnapshot = snapshotSupplier.get(); + SnapshotAndTime next = new SnapshotAndTime(now + validForNanos, newSnapshot); + if (compareAndSet(cur, next)) + return next.snapshot; + + return accumulateAndGet(next, (a, b) -> a.validUntil > b.validUntil ? a : b).snapshot; + } + + @Override + public long get(double percentile) + { + return (long)getSnapshot().getValue(percentile); + } + } + + static class Bound + { + final long min, max, onFailure; + final LatencyModifier modifier; + final LatencySelector selector; + final LatencySupplier reads, writes; + + Bound(long min, long max, long onFailure, LatencyModifier modifier, LatencySelector selector) + { + Preconditions.checkArgument(min<=max, "min (%s) must be less than or equal to max (%s)", min, max); + this.min = min; + this.max = max; + this.onFailure = onFailure; + this.modifier = modifier; + this.selector = selector; + this.reads = new TimeLimitedLatencySupplier(casReadMetrics.latency::getSnapshot, 10L, SECONDS); + this.writes = new TimeLimitedLatencySupplier(casWriteMetrics.latency::getSnapshot, 10L, SECONDS); + } + + long get(int attempts) + { + try + { + long base = selector.select(reads, writes); + return max(min, min(max, modifier.modify(base, attempts))); + } + catch (Throwable t) + { + NoSpamLogger.getLogger(logger, 1L, MINUTES).info("", t); + return onFailure; + } + } + + public String toString() + { + return "Bound{" + + "min=" + min + + ", max=" + max + + ", onFailure=" + onFailure + + ", modifier=" + modifier + + ", selector=" + selector + + '}'; + } + } + + final WaitRandomizer waitRandomizer; + final Bound min, max, minDelta; + final int traceAfterAttempts; + + public ContentionStrategy(String waitRandomizer, String min, String max, String minDelta, int traceAfterAttempts) + { + this.waitRandomizer = parseWaitRandomizer(waitRandomizer); + this.min = parseBound(min, true); + this.max = parseBound(max, false); + this.minDelta = parseBound(minDelta, true); + this.traceAfterAttempts = traceAfterAttempts; + } + + public enum Type + { + READ("Contended Paxos Read"), WRITE("Contended Paxos Write"), REPAIR("Contended Paxos Repair"); + + final String traceTitle; + final String lowercase; + + Type(String traceTitle) + { + this.traceTitle = traceTitle; + this.lowercase = name().toLowerCase(); + } + } + + private long computeWaitUntilForContention(int attempts, TableMetadata table, DecoratedKey partitionKey, ConsistencyLevel consistency, Type type) + { + if (attempts >= traceAfterAttempts && !Tracing.isTracing()) + { + Tracing.instance.newSession(Tracing.TraceType.QUERY); + Tracing.instance.begin(type.traceTitle, + ImmutableMap.of( + "keyspace", table.keyspace, + "table", table.name, + "partitionKey", table.partitionKeyType.getString(partitionKey.getKey()), + "consistency", consistency.name(), + "kind", type.lowercase + )); + + logger.info("Tracing contended paxos {} for key {} on {}.{} with trace id {}", + type.lowercase, + ByteBufferUtil.bytesToHex(partitionKey.getKey()), + table.keyspace, table.name, + Tracing.instance.getSessionId()); + } + + long minWaitMicros = min.get(attempts); + long maxWaitMicros = max.get(attempts); + long minDeltaMicros = minDelta.get(attempts); + + if (minWaitMicros + minDeltaMicros > maxWaitMicros) + { + maxWaitMicros = minWaitMicros + minDeltaMicros; + if (maxWaitMicros > this.max.max) + { + maxWaitMicros = this.max.max; + minWaitMicros = max(this.min.min, min(this.min.max, maxWaitMicros - minDeltaMicros)); + } + } + + long wait = waitRandomizer.wait(minWaitMicros, maxWaitMicros, attempts); + return nanoTime() + wait; + } + + private boolean doWaitForContention(long deadline, int attempts, TableMetadata table, DecoratedKey partitionKey, ConsistencyLevel consistency, Type type) + { + long until = computeWaitUntilForContention(attempts, table, partitionKey, consistency, type); + if (until >= deadline) + return false; + + try + { + waitUntil(until); + } + catch (InterruptedException e) + { + Thread.currentThread().interrupt(); + return false; + } + return true; + } + + static boolean waitForContention(long deadline, int attempts, TableMetadata table, DecoratedKey partitionKey, ConsistencyLevel consistency, Type type) + { + return current.doWaitForContention(deadline, attempts, table, partitionKey, consistency, type); + } + + static long waitUntilForContention(int attempts, TableMetadata table, DecoratedKey partitionKey, ConsistencyLevel consistency, Type type) + { + return current.computeWaitUntilForContention(attempts, table, partitionKey, consistency, type); + } + + static class ParsedStrategy + { + final String waitRandomizer, min, max, minDelta; + final ContentionStrategy strategy; + + ParsedStrategy(String waitRandomizer, String min, String max, String minDelta, ContentionStrategy strategy) + { + this.waitRandomizer = waitRandomizer; + this.min = min; + this.max = max; + this.minDelta = minDelta; + this.strategy = strategy; + } + } + + @VisibleForTesting + static ParsedStrategy parseStrategy(String spec) + { + String[] args = spec.split(","); + String waitRandomizer = find(args, "random"); + String min = find(args, "min"); + String max = find(args, "max"); + String minDelta = find(args, "delta"); + String trace = find(args, "trace"); + + if (waitRandomizer == null) waitRandomizer = defaultWaitRandomizer(); + if (min == null) min = defaultMinWait(); + if (max == null) max = defaultMaxWait(); + if (minDelta == null) minDelta = defaultMinDelta(); + int traceAfterAttempts = trace == null ? current.traceAfterAttempts: Integer.parseInt(trace); + + ContentionStrategy strategy = new ContentionStrategy(waitRandomizer, min, max, minDelta, traceAfterAttempts); + return new ParsedStrategy(waitRandomizer, min, max, minDelta, strategy); + } + + + public static void setStrategy(String spec) + { + ParsedStrategy parsed = parseStrategy(spec); + current = parsed.strategy; + setPaxosContentionWaitRandomizer(parsed.waitRandomizer); + setPaxosContentionMinWait(parsed.min); + setPaxosContentionMaxWait(parsed.max); + setPaxosContentionMinDelta(parsed.minDelta); + } + + public static String getStrategySpec() + { + return "min=" + defaultMinWait() + + ",max=" + defaultMaxWait() + + ",delta=" + defaultMinDelta() + + ",random=" + defaultWaitRandomizer() + + ",trace=" + current.traceAfterAttempts; + } + + private static String find(String[] args, String param) + { + return stream(args).filter(s -> s.startsWith(param + '=')) + .map(s -> s.substring(param.length() + 1)) + .findFirst().orElse(null); + } + + private static LatencySelector parseLatencySelector(Matcher m, LatencySelectorFactory selectors) + { + String perc = m.group("perc"); + if (perc == null) + return selectors.constant(parseInMicros(m.group("constbase"))); + + double percentile = parseDouble("0." + perc); + String rw = m.group("rw"); + if (rw.length() == 2) + return selectors.maxReadWrite(percentile); + else if ("r".equals(rw)) + return selectors.read(percentile); + else + return selectors.write(percentile); + } + + private static LatencyModifier parseLatencyModifier(Matcher m, LatencyModifierFactory modifiers) + { + String mod = m.group("mod"); + if (mod == null) + return modifiers.identity(); + + double modifier = parseDouble(mod); + + String modkind = m.group("modkind"); + if (modkind == null) + return modifiers.multiply(modifier); + + if (modkind.startsWith("*")) + return modifiers.multiplyByAttempts(modifier); + else if (modkind.startsWith("^")) + return modifiers.multiplyByAttemptsExp(modifier); + else + throw new IllegalArgumentException("Unrecognised attempt modifier: " + modkind); + } + + static long saturatedCast(double v) + { + if (v > Long.MAX_VALUE) + return Long.MAX_VALUE; + return (long) v; + } + + static WaitRandomizer parseWaitRandomizer(String input) + { + return parseWaitRandomizer(input, randomizers); + } + + static WaitRandomizer parseWaitRandomizer(String input, WaitRandomizerFactory randomizers) + { + Matcher m = RANDOMIZER.matcher(input); + if (!m.matches()) + throw new IllegalArgumentException(input + " does not match" + RANDOMIZER); + + String exp; + exp = m.group("exp"); + if (exp != null) + return randomizers.exponential(Double.parseDouble(exp)); + exp = m.group("qexp"); + if (exp != null) + return randomizers.quantizedExponential(Double.parseDouble(exp)); + return randomizers.uniform(); + } + + static Bound parseBound(String input, boolean isMin) + { + return parseBound(input, isMin, selectors, modifiers); + } + + @VisibleForTesting + static Bound parseBound(String input, boolean isMin, LatencySelectorFactory selectors, LatencyModifierFactory modifiers) + { + Matcher m = BOUND.matcher(input); + if (!m.matches()) + throw new IllegalArgumentException(input + " does not match " + BOUND); + + String maybeConst = m.group("const"); + if (maybeConst != null) + { + long v = parseInMicros(maybeConst); + return new Bound(v, v, v, modifiers.identity(), selectors.constant(v)); + } + + long min = parseInMicros(m.group("min"), 0); + long max = parseInMicros(m.group("max"), maxQueryTimeoutMicros() / 2); + return new Bound(min, max, isMin ? min : max, parseLatencyModifier(m, modifiers), parseLatencySelector(m, selectors)); + } + + private static long parseInMicros(String input, long orElse) + { + if (input == null) + return orElse; + + return parseInMicros(input); + } + + private static long parseInMicros(String input) + { + Matcher m = TIME.matcher(input); + if (!m.matches()) + throw new IllegalArgumentException(input + " does not match " + TIME); + + String text; + if (null != (text = m.group(1))) + return parseInt(text) * 1000; + else if (null != (text = m.group(2))) + return parseInt(text); + else + return 0; + } + + @VisibleForTesting + static String defaultWaitRandomizer() + { + return orElse(DatabaseDescriptor::getPaxosContentionWaitRandomizer, DEFAULT_WAIT_RANDOMIZER); + } + + @VisibleForTesting + static String defaultMinWait() + { + return orElse(DatabaseDescriptor::getPaxosContentionMinWait, DEFAULT_MIN); + } + + @VisibleForTesting + static String defaultMaxWait() + { + return orElse(DatabaseDescriptor::getPaxosContentionMaxWait, DEFAULT_MAX); + } + + @VisibleForTesting + static String defaultMinDelta() + { + return orElse(DatabaseDescriptor::getPaxosContentionMinDelta, DEFAULT_MIN_DELTA); + } + + @VisibleForTesting + static long maxQueryTimeoutMicros() + { + return max(max(getCasContentionTimeout(MICROSECONDS), getWriteRpcTimeout(MICROSECONDS)), getReadRpcTimeout(MICROSECONDS)); + } + + private static String orElse(Supplier get, String orElse) + { + String result = get.get(); + return result != null ? result : orElse; + } +} diff --git a/src/java/org/apache/cassandra/service/paxos/Paxos.java b/src/java/org/apache/cassandra/service/paxos/Paxos.java index 97734f8147..8ae26f7c01 100644 --- a/src/java/org/apache/cassandra/service/paxos/Paxos.java +++ b/src/java/org/apache/cassandra/service/paxos/Paxos.java @@ -18,12 +18,1184 @@ package org.apache.cassandra.service.paxos; -import com.google.common.base.Preconditions; -import org.apache.cassandra.config.Config; +import java.io.IOException; +import java.util.Collection; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; +import java.util.function.Function; +import java.util.function.Supplier; +import javax.annotation.Nullable; + +import com.google.common.base.Preconditions; +import com.google.common.collect.Maps; + +import org.apache.commons.httpclient.methods.multipart.Part; + +import com.codahale.metrics.Meter; +import org.apache.cassandra.exceptions.CasWriteTimeoutException; +import org.apache.cassandra.exceptions.ExceptionCode; +import org.apache.cassandra.gms.FailureDetector; +import org.apache.cassandra.locator.AbstractReplicationStrategy; +import org.apache.cassandra.locator.EndpointsForToken; +import org.apache.cassandra.locator.InOurDc; +import org.apache.cassandra.locator.Replica; +import org.apache.cassandra.locator.ReplicaLayout; +import org.apache.cassandra.locator.ReplicaLayout.ForTokenWrite; +import org.apache.cassandra.locator.ReplicaPlan.ForRead; +import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.config.Config; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.db.ConsistencyLevel; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.Keyspace; +import org.apache.cassandra.db.SinglePartitionReadCommand; +import org.apache.cassandra.db.WriteType; +import org.apache.cassandra.db.partitions.FilteredPartition; +import org.apache.cassandra.db.partitions.PartitionIterator; +import org.apache.cassandra.db.partitions.PartitionIterators; +import org.apache.cassandra.db.partitions.PartitionUpdate; +import org.apache.cassandra.db.rows.RowIterator; +import org.apache.cassandra.dht.Token; +import org.apache.cassandra.exceptions.InvalidRequestException; +import org.apache.cassandra.exceptions.IsBootstrappingException; +import org.apache.cassandra.exceptions.ReadFailureException; +import org.apache.cassandra.exceptions.ReadTimeoutException; +import org.apache.cassandra.exceptions.RequestExecutionException; +import org.apache.cassandra.exceptions.RequestFailureException; +import org.apache.cassandra.exceptions.RequestFailureReason; +import org.apache.cassandra.exceptions.RequestTimeoutException; +import org.apache.cassandra.exceptions.UnavailableException; +import org.apache.cassandra.exceptions.WriteFailureException; +import org.apache.cassandra.exceptions.WriteTimeoutException; +import org.apache.cassandra.gms.EndpointState; +import org.apache.cassandra.gms.Gossiper; +import org.apache.cassandra.io.IVersionedSerializer; +import org.apache.cassandra.io.util.DataInputPlus; +import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.metrics.ClientRequestMetrics; +import org.apache.cassandra.service.CASRequest; +import org.apache.cassandra.service.ClientState; +import org.apache.cassandra.service.FailureRecordingCallback; +import org.apache.cassandra.service.FailureRecordingCallback.AsMap; +import org.apache.cassandra.service.paxos.Commit.Proposal; +import org.apache.cassandra.service.reads.DataResolver; +import org.apache.cassandra.service.reads.repair.NoopReadRepair; +import org.apache.cassandra.service.paxos.cleanup.PaxosTableRepairs; +import org.apache.cassandra.tracing.Tracing; +import org.apache.cassandra.triggers.TriggerExecutor; +import org.apache.cassandra.utils.CassandraVersion; +import org.apache.cassandra.utils.CollectionSerializer; +import org.apache.cassandra.utils.FBUtilities; +import org.apache.cassandra.service.paxos.PaxosPrepare.FoundIncompleteAccepted; +import org.apache.cassandra.service.paxos.PaxosPrepare.FoundIncompleteCommitted; + +import static java.util.Collections.emptyMap; +import static java.util.concurrent.TimeUnit.NANOSECONDS; +import static java.util.concurrent.TimeUnit.SECONDS; +import static org.apache.cassandra.config.Config.PaxosVariant.v2_without_linearizable_reads_or_rejected_writes; +import static org.apache.cassandra.db.Keyspace.openAndGetStore; +import static org.apache.cassandra.exceptions.RequestFailureReason.TIMEOUT; +import static org.apache.cassandra.gms.ApplicationState.RELEASE_VERSION; +import static org.apache.cassandra.config.DatabaseDescriptor.*; +import static org.apache.cassandra.db.ConsistencyLevel.*; +import static org.apache.cassandra.locator.InetAddressAndPort.Serializer.inetAddressAndPortSerializer; +import static org.apache.cassandra.locator.ReplicaLayout.forTokenWriteLiveAndDown; +import static org.apache.cassandra.metrics.ClientRequestsMetricsHolder.casReadMetrics; +import static org.apache.cassandra.metrics.ClientRequestsMetricsHolder.casWriteMetrics; +import static org.apache.cassandra.metrics.ClientRequestsMetricsHolder.readMetrics; +import static org.apache.cassandra.metrics.ClientRequestsMetricsHolder.readMetricsMap; +import static org.apache.cassandra.metrics.ClientRequestsMetricsHolder.writeMetricsMap; +import static org.apache.cassandra.service.paxos.Ballot.Flag.GLOBAL; +import static org.apache.cassandra.service.paxos.Ballot.Flag.LOCAL; +import static org.apache.cassandra.service.paxos.BallotGenerator.Global.nextBallot; +import static org.apache.cassandra.service.paxos.BallotGenerator.Global.staleBallot; +import static org.apache.cassandra.service.paxos.ContentionStrategy.*; +import static org.apache.cassandra.service.paxos.ContentionStrategy.Type.READ; +import static org.apache.cassandra.service.paxos.ContentionStrategy.Type.WRITE; +import static org.apache.cassandra.service.paxos.PaxosCommit.commit; +import static org.apache.cassandra.service.paxos.PaxosCommitAndPrepare.commitAndPrepare; +import static org.apache.cassandra.service.paxos.PaxosPrepare.prepare; +import static org.apache.cassandra.service.paxos.PaxosPropose.propose; +import static org.apache.cassandra.utils.Clock.Global.nanoTime; +import static org.apache.cassandra.utils.CollectionSerializer.newHashSet; +import static org.apache.cassandra.utils.FBUtilities.getBroadcastAddressAndPort; + +/** + *

    This class serves as an entry-point to Cassandra's implementation of Paxos Consensus. + * Note that Cassandra does not utilise the distinguished proposer (Multi Paxos) optimisation; + * each operation executes its own instance of Paxos Consensus. Instead Cassandra employs + * various optimisations to reduce the overhead of operations. This may lead to higher throughput + * and lower overhead read operations, at the expense of contention during mixed or write-heavy workloads. + * + * Firstly, note that we do not follow Lamport's formulation, instead following the more common approach in + * literature (see e.g. Dr. Heidi Howard's dissertation) of permitting any acceptor to vote on a proposal, + * not only those who issued a promise. + * + *

    No Commit of Empty Proposals

    + *

    If a proposal is empty, there can be no effect to the state, so once this empty proposal has poisoned any earlier + * proposal it is safe to stop processing. An empty proposal effectively scrubs the instance of consensus being + * performed once it has reached a quorum, as no earlier incomplete proposal (that may perhaps have reached a minority) + * may now be completed. + * + *

    Fast Read / Failed Write

    + *

    This optimisation relies on every voter having no incomplete promises, i.e. their commit register must be greater + * than or equal to their promise and proposal registers (or there must be such an empty proposal). + * Since the operation we are performing must invalidate any nascent operation that has reached a minority, and will + * itself be invalidated by any newer write it might race with, we are only concerned about operations that might be + * in-flight and incomplete. If we reach a quorum without any incomplete proposal, we prevent any incomplete proposal + * that might have come before us from being committed, and so are correctly ordered. + * + *

    NOTE: we could likely weaken this further, permitting a fast operation if we witness a stale incomplete operation + * on one or more of the replicas, so long as we witness _some_ response that had knowledge of that operation's decision, + * however we might waste more time performing optimistic reads (which we skip if we witness any in progress promise) + * + *

    Read Commutativity Optimisation

    + *

    We separate read and write promises into distinct registers. Since reads are commutative they do not need to be + * ordered with respect to each other, so read promises consult only the write promise register to find competing + * operations, whereas writes consult both read and write registers. This permits better utilisation of the Fast Read + * optimisation, permitting arbitrarily many fast reads to execute concurrently. + * + *

    A read will use its promise to finish any in progress write it encounters, but note that this is safe for multiple + * reads to attempt simultaneously. If a write operation has not reached a quorum of promises then it has no effect, + * so while some read operations may attempt to complete it and others may not, the operation will only be invalidated + * and these actions will be equivalent. If the write had reached a quorum of promises then every reads will attempt + * to complete the write. At the accept phase, only the most recent read promise will be accepted so whether the write + * proposal had reached a quorum or not, a consistent outcome will result. + * + *

    Reproposal Avoidance

    + *

    It can occur that two (or more) commands begin competing to re-propose the same incomplete command even after it + * has already committed - this can occur when an in progress command that has reached the commit condition (but not yet + * committed) is encountered by a promise, so that it is re-proposed. If the original coordinator does not fail this + * original command will be committed normally, but the re-proposal can take on a life of its own, and become contended + * and re-proposed indefinitely. By having reproposals use the original proposal ballot's timestamp we spot this situation + * and consider re-proposals of a command we have seen committed to be (in effect) empty proposals. + * + *

    Durability of Asynchronous Commit

    + * To permit asynchronous commit (and also because we should) we ensure commits are durable once a proposal has been + * accepted by a majority. + * + * Replicas track commands that have *locally* been witnessed but not committed. They may clear this log by performing + * a round of Paxos Repair for each key in the log (which is simply a round of Paxos that tries not to interfere with + * future rounds of Paxos, while aiming to complete any earlier incomplete round). + * + * By selecting some quorum of replicas for a range to perform this operation on, once successful we guarantee that + * any transaction that had previously been accepted by a majority has been committed, and any transaction that had been + * previously witnessed by a majority has been either committed or invalidated. + * + * To ensure durability across range movements, once a joining node becomes pending such a coordinated paxos repair + * is performed prior to performing bootstrap, so that commands initiated before joining will either be bootstrapped + * or completed by paxos repair to be committed to a majority that includes the new node in its calculations, and + * commands initiated after will anyway do so due to being pending. + * + * Finally, for greater guarantees across range movements despite the uncertainty of gossip, paxos operations validate + * ring information with each other while seeking a quorum of promises. Any inconsistency is resolved by synchronising + * gossip state between the coordinator and the peers in question. + * + *

    Clearing of Paxos State

    + * Coordinated paxos repairs as described above are preceded by an preparation step that determines a ballot below + * which we agree to reject new promises. By deciding and disseminating this point prior to performing a coordinated + * paxos repair, once complete we have ensured that all commands with a lower ballot are either committed or invalidated, + * and so we are then able to disseminate this ballot as a bound below which may expunge all data for the range. + * + * For consistency of execution coordinators seek this latter ballot bound from each replica and, using the maximum of + * these, ignore all data received associated with ballots lower than this bound. + */ public class Paxos { - private static volatile Config.PaxosVariant PAXOS_VARIANT = Config.PaxosVariant.v1; + private static volatile Config.PaxosVariant PAXOS_VARIANT = DatabaseDescriptor.getPaxosVariant(); + private static final CassandraVersion MODERN_PAXOS_RELEASE = new CassandraVersion(System.getProperty("cassandra.paxos.modern_release", "4.1")); + static final boolean LOG_TTL_LINEARIZABILITY_VIOLATIONS = Boolean.parseBoolean(System.getProperty("cassandra.paxos.log_ttl_linearizability_violations", "true")); + + static class Electorate + { + static final Serializer serializer = new Serializer(); + + // all replicas, including pending, but without those in a remote DC if consistency is local + final Collection natural; + + // pending subset of electorate + final Collection pending; + + public Electorate(Collection natural, Collection pending) + { + this.natural = natural; + this.pending = pending; + } + + public int size() + { + return natural.size() + pending.size(); + } + + public void forEach(Consumer forEach) + { + natural.forEach(forEach); + pending.forEach(forEach); + } + + static Electorate get(TableMetadata table, DecoratedKey key, ConsistencyLevel consistency) + { + return get(consistency, forTokenWriteLiveAndDown(Keyspace.open(table.keyspace), key.getToken())); + } + + static Electorate get(ConsistencyLevel consistency, ForTokenWrite all) + { + ForTokenWrite electorate = all; + if (consistency == LOCAL_SERIAL) + electorate = all.filter(InOurDc.replicas()); + + return new Electorate(electorate.natural().endpointList(), electorate.pending().endpointList()); + } + + boolean hasPending() + { + return !pending.isEmpty(); + } + + boolean isPending(InetAddressAndPort endpoint) + { + return hasPending() && pending.contains(endpoint); + } + + public boolean equals(Object o) + { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Electorate that = (Electorate) o; + return natural.equals(that.natural) && pending.equals(that.pending); + } + + public int hashCode() + { + return Objects.hash(natural, pending); + } + + public String toString() + { + return "{" + natural + ", " + pending + '}'; + } + + static class Serializer implements IVersionedSerializer + { + public void serialize(Electorate electorate, DataOutputPlus out, int version) throws IOException + { + CollectionSerializer.serializeCollection(inetAddressAndPortSerializer, electorate.natural, out, version); + CollectionSerializer.serializeCollection(inetAddressAndPortSerializer, electorate.pending, out, version); + } + + public Electorate deserialize(DataInputPlus in, int version) throws IOException + { + Set endpoints = CollectionSerializer.deserializeCollection(inetAddressAndPortSerializer, newHashSet(), in, version); + Set pending = CollectionSerializer.deserializeCollection(inetAddressAndPortSerializer, newHashSet(), in, version); + return new Electorate(endpoints, pending); + } + + public long serializedSize(Electorate electorate, int version) + { + return CollectionSerializer.serializedSizeCollection(inetAddressAndPortSerializer, electorate.natural, version) + + CollectionSerializer.serializedSizeCollection(inetAddressAndPortSerializer, electorate.pending, version); + } + } + } + + /** + * Encapsulates the peers we will talk to for this operation. + */ + static class Participants implements ForRead, Supplier + { + final Keyspace keyspace; + + final AbstractReplicationStrategy replicationStrategy; + + /** + * SERIAL or LOCAL_SERIAL + */ + final ConsistencyLevel consistencyForConsensus; + + /** + * Those members that vote for {@link #consistencyForConsensus} + */ + final Electorate electorate; + + /** + * Those members of {@link #electorate} that we will 'poll' for their vote + * i.e. {@link #electorate} with down nodes removed + */ + + private final EndpointsForToken electorateNatural; + final EndpointsForToken electorateLive; + + final EndpointsForToken all; + final EndpointsForToken allLive; + final EndpointsForToken allDown; + final EndpointsForToken pending; + + /** + * The number of responses we require to reach desired consistency from members of {@code contact} + */ + final int sizeOfConsensusQuorum; + + /** + * The number of read responses we require to reach desired consistency from members of {@code contact} + * Note that this should always be met if {@link #sizeOfConsensusQuorum} is met, but we supply it separately + * for corroboration. + */ + final int sizeOfReadQuorum; + + Participants(Keyspace keyspace, ConsistencyLevel consistencyForConsensus, ReplicaLayout.ForTokenWrite all, ReplicaLayout.ForTokenWrite electorate, EndpointsForToken live) + { + this.keyspace = keyspace; + this.replicationStrategy = all.replicationStrategy(); + this.consistencyForConsensus = consistencyForConsensus; + this.all = all.all(); + this.pending = all.pending(); + this.allDown = all.all() == live ? EndpointsForToken.empty(all.token()) : all.all().without(live.endpoints()); + this.electorate = new Electorate(electorate.natural().endpointList(), electorate.pending().endpointList()); + this.electorateNatural = electorate.natural(); + this.electorateLive = electorate.all() == live ? live : electorate.all().keep(live.endpoints()); + this.allLive = live; + this.sizeOfReadQuorum = electorate.natural().size() / 2 + 1; + this.sizeOfConsensusQuorum = sizeOfReadQuorum + electorate.pending().size(); + } + + @Override + public int readQuorum() + { + return sizeOfReadQuorum; + } + + @Override + public EndpointsForToken readCandidates() + { + // Note: we could probably return electorateLive here and save a reference, but it's not strictly correct + return electorateNatural; + } + + static Participants get(TableMetadata table, Token token, ConsistencyLevel consistencyForConsensus) + { + Keyspace keyspace = Keyspace.open(table.keyspace); + ReplicaLayout.ForTokenWrite all = forTokenWriteLiveAndDown(keyspace, token); + ReplicaLayout.ForTokenWrite electorate = consistencyForConsensus.isDatacenterLocal() + ? all.filter(InOurDc.replicas()) : all; + + EndpointsForToken live = all.all().filter(FailureDetector.isReplicaAlive); + + return new Participants(keyspace, consistencyForConsensus, all, electorate, live); + } + + static Participants get(TableMetadata cfm, DecoratedKey key, ConsistencyLevel consistency) + { + return get(cfm, key.getToken(), consistency); + } + + int sizeOfPoll() + { + return electorateLive.size(); + } + + InetAddressAndPort voter(int i) + { + return electorateLive.endpoint(i); + } + + void assureSufficientLiveNodes(boolean isWrite) throws UnavailableException + { + if (sizeOfConsensusQuorum > sizeOfPoll()) + { + mark(isWrite, m -> m.unavailables, consistencyForConsensus); + throw new UnavailableException("Cannot achieve consistency level " + consistencyForConsensus, consistencyForConsensus, sizeOfConsensusQuorum, sizeOfPoll()); + } + } + + void assureSufficientLiveNodesForRepair() throws UnavailableException + { + if (sizeOfConsensusQuorum > sizeOfPoll()) + { + throw UnavailableException.create(consistencyForConsensus, sizeOfConsensusQuorum, sizeOfPoll()); + } + } + + int requiredFor(ConsistencyLevel consistency) + { + if (consistency == Paxos.nonSerial(consistencyForConsensus)) + return sizeOfConsensusQuorum; + + return consistency.blockForWrite(replicationStrategy(), pending); + } + + public boolean hasOldParticipants() + { + return electorateLive.anyMatch(Paxos::isOldParticipant); + } + + @Override + public Participants get() + { + return this; + } + + @Override + public Keyspace keyspace() + { + return keyspace; + } + + @Override + public AbstractReplicationStrategy replicationStrategy() + { + return replicationStrategy; + } + + @Override + public ConsistencyLevel consistencyLevel() + { + return nonSerial(consistencyForConsensus); + } + + @Override + public EndpointsForToken contacts() + { + return electorateLive; + } + + @Override + public Replica lookup(InetAddressAndPort endpoint) + { + return all.lookup(endpoint); + } + + @Override + public Participants withContacts(EndpointsForToken newContacts) + { + throw new UnsupportedOperationException(); + } + } + + /** + * Encapsulates information about a failure to reach Success, either because of explicit failure responses + * or insufficient responses (in which case the status is not final) + */ + static class MaybeFailure + { + final boolean isFailure; + final String serverError; + final int contacted; + final int required; + final int successes; + final Map failures; + + static MaybeFailure noResponses(Participants contacted) + { + return new MaybeFailure(false, contacted.sizeOfPoll(), contacted.sizeOfConsensusQuorum, 0, emptyMap()); + } + + MaybeFailure(Participants contacted, int successes, AsMap failures) + { + this(contacted.sizeOfPoll() - failures.failureCount() < contacted.sizeOfConsensusQuorum, contacted.sizeOfPoll(), contacted.sizeOfConsensusQuorum, successes, failures); + } + + MaybeFailure(int contacted, int required, int successes, AsMap failures) + { + this(contacted - failures.failureCount() < required, contacted, required, successes, failures); + } + + MaybeFailure(boolean isFailure, int contacted, int required, int successes, Map failures) + { + this(isFailure, null, contacted, required, successes, failures); + } + + MaybeFailure(boolean isFailure, String serverError, int contacted, int required, int successes, Map failures) + { + this.isFailure = isFailure; + this.serverError = serverError; + this.contacted = contacted; + this.required = required; + this.successes = successes; + this.failures = failures; + } + + private static int failureCount(Map failures) + { + int count = 0; + for (RequestFailureReason reason : failures.values()) + count += reason != TIMEOUT ? 1 : 0; + return count; + } + + /** + * update relevant counters and throw the relevant exception + */ + RequestExecutionException markAndThrowAsTimeoutOrFailure(boolean isWrite, ConsistencyLevel consistency, int failedAttemptsDueToContention) + { + if (isFailure) + { + mark(isWrite, m -> m.failures, consistency); + throw serverError != null ? new RequestFailureException(ExceptionCode.SERVER_ERROR, serverError, consistency, successes, required, failures) + : isWrite + ? new WriteFailureException(consistency, successes, required, WriteType.CAS, failures) + : new ReadFailureException(consistency, successes, required, false, failures); + } + else + { + mark(isWrite, m -> m.timeouts, consistency); + throw isWrite + ? new CasWriteTimeoutException(WriteType.CAS, consistency, successes, required, failedAttemptsDueToContention) + : new ReadTimeoutException(consistency, successes, required, false); + } + } + + public String toString() + { + return (isFailure ? "Failure(" : "Timeout(") + successes + ',' + failures + ')'; + } + } + + public interface Async + { + Result awaitUntil(long until); + } + + /** + * Apply @param updates if and only if the current values in the row for @param key + * match the provided @param conditions. The algorithm is "raw" Paxos: that is, Paxos + * minus leader election -- any node in the cluster may propose changes for any partition. + * + * The Paxos electorate consists only of the replicas for the partition key. + * We expect performance to be reasonable, but CAS is still intended to be used + * "when you really need it," not for all your updates. + * + * There are three phases to Paxos: + * 1. Prepare: the coordinator generates a ballot (Ballot in our case) and asks replicas to + * - promise not to accept updates from older ballots and + * - tell us about the latest ballots it has already _promised_, _accepted_, or _committed_ + * - reads the necessary data to evaluate our CAS condition + * + * 2. Propose: if a majority of replicas reply, the coordinator asks replicas to accept the value of the + * highest proposal ballot it heard about, or a new value if no in-progress proposals were reported. + * 3. Commit (Learn): if a majority of replicas acknowledge the accept request, we can commit the new + * value. + * + * Commit procedure is not covered in "Paxos Made Simple," and only briefly mentioned in "Paxos Made Live," + * so here is our approach: + * 3a. The coordinator sends a commit message to all replicas with the ballot and value. + * 3b. Because of 1-2, this will be the highest-seen commit ballot. The replicas will note that, + * and send it with subsequent promise replies. This allows us to discard acceptance records + * for successfully committed replicas, without allowing incomplete proposals to commit erroneously + * later on. + * + * Note that since we are performing a CAS rather than a simple update, when nodes respond positively to + * Prepare, they include read response of commited values that will be reconciled on the coordinator + * and checked against CAS precondition between the prepare and accept phases. This gives us a slightly + * longer window for another coordinator to come along and trump our own promise with a newer one but + * is otherwise safe. + * + * Any successful prepare phase yielding a read that rejects the condition must be followed by the proposal of + * an empty update, to ensure the evaluation of the condition is linearized with respect to other reads and writes. + * + * @param key the row key for the row to CAS + * @param request the conditions for the CAS to apply as well as the update to perform if the conditions hold. + * @param consistencyForConsensus the consistency for the paxos prepare and propose round. This can only be either SERIAL or LOCAL_SERIAL. + * @param consistencyForCommit the consistency for write done during the commit phase. This can be anything, except SERIAL or LOCAL_SERIAL. + * + * @return null if the operation succeeds in updating the row, or the current values corresponding to conditions. + * (since, if the CAS doesn't succeed, it means the current value do not match the conditions). + */ + public static RowIterator cas(DecoratedKey key, + CASRequest request, + ConsistencyLevel consistencyForConsensus, + ConsistencyLevel consistencyForCommit, + ClientState clientState) + throws UnavailableException, IsBootstrappingException, RequestFailureException, RequestTimeoutException, InvalidRequestException + { + final long start = nanoTime(); + final long proposeDeadline = start + getCasContentionTimeout(NANOSECONDS); + final long commitDeadline = Math.max(proposeDeadline, start + getWriteRpcTimeout(NANOSECONDS)); + return cas(key, request, consistencyForConsensus, consistencyForCommit, clientState, start, proposeDeadline, commitDeadline); + } + public static RowIterator cas(DecoratedKey key, + CASRequest request, + ConsistencyLevel consistencyForConsensus, + ConsistencyLevel consistencyForCommit, + ClientState clientState, + long proposeDeadline, + long commitDeadline + ) + throws UnavailableException, IsBootstrappingException, RequestFailureException, RequestTimeoutException, InvalidRequestException + { + return cas(key, request, consistencyForConsensus, consistencyForCommit, clientState, nanoTime(), proposeDeadline, commitDeadline); + } + private static RowIterator cas(DecoratedKey partitionKey, + CASRequest request, + ConsistencyLevel consistencyForConsensus, + ConsistencyLevel consistencyForCommit, + ClientState clientState, + long start, + long proposeDeadline, + long commitDeadline + ) + throws UnavailableException, IsBootstrappingException, RequestFailureException, RequestTimeoutException, InvalidRequestException + { + SinglePartitionReadCommand readCommand = request.readCommand(FBUtilities.nowInSeconds()); + TableMetadata metadata = readCommand.metadata(); + + consistencyForConsensus.validateForCas(); + consistencyForCommit.validateForCasCommit(Keyspace.open(metadata.keyspace).getReplicationStrategy()); + + Ballot minimumBallot = null; + int failedAttemptsDueToContention = 0; + try (PaxosOperationLock lock = PaxosState.lock(partitionKey, metadata, proposeDeadline, consistencyForConsensus, true)) + { + Paxos.Async commit = null; + done: while (true) + { + // read the current values and check they validate the conditions + Tracing.trace("Reading existing values for CAS precondition"); + + BeginResult begin = begin(proposeDeadline, readCommand, consistencyForConsensus, + true, minimumBallot, failedAttemptsDueToContention); + Ballot ballot = begin.ballot; + Participants participants = begin.participants; + failedAttemptsDueToContention = begin.failedAttemptsDueToContention; + + FilteredPartition current; + try (RowIterator iter = PartitionIterators.getOnlyElement(begin.readResponse, readCommand)) + { + current = FilteredPartition.create(iter); + } + + Proposal proposal; + boolean conditionMet = request.appliesTo(current); + if (!conditionMet) + { + if (getPaxosVariant() == v2_without_linearizable_reads_or_rejected_writes) + { + Tracing.trace("CAS precondition rejected", current); + casWriteMetrics.conditionNotMet.inc(); + return current.rowIterator(); + } + + // If we failed to meet our condition, it does not mean we can do nothing: if we do not propose + // anything that is accepted by a quorum, it is possible for our !conditionMet state + // to not be serialized wrt other operations. + // If a later read encounters an "in progress" write that did not reach a majority, + // but that would have permitted conditionMet had it done so (and hence we evidently did not witness), + // that operation will complete the in-progress proposal before continuing, so that this and future + // reads will perceive conditionMet without any intervening modification from the time at which we + // assured a conditional write that !conditionMet. + // So our evaluation is only serialized if we invalidate any in progress operations by proposing an empty update + // See also CASSANDRA-12126 + if (begin.isLinearizableRead) + { + Tracing.trace("CAS precondition does not match current values {}; read is already linearizable; aborting", current); + return conditionNotMet(current); + } + + Tracing.trace("CAS precondition does not match current values {}; proposing empty update", current); + proposal = Proposal.empty(ballot, partitionKey, metadata); + } + else if (begin.isPromised) + { + // finish the paxos round w/ the desired updates + // TODO "turn null updates into delete?" - what does this TODO even mean? + PartitionUpdate updates = request.makeUpdates(current, clientState, begin.ballot); + + // Apply triggers to cas updates. A consideration here is that + // triggers emit Mutations, and so a given trigger implementation + // may generate mutations for partitions other than the one this + // paxos round is scoped for. In this case, TriggerExecutor will + // validate that the generated mutations are targetted at the same + // partition as the initial updates and reject (via an + // InvalidRequestException) any which aren't. + updates = TriggerExecutor.instance.execute(updates); + + proposal = Proposal.of(ballot, updates); + Tracing.trace("CAS precondition is met; proposing client-requested updates for {}", ballot); + } + else + { + // must retry, as only achieved read success in begin + Tracing.trace("CAS precondition is met, but ballot stale for proposal; retrying", current); + continue; + } + + PaxosPropose.Status propose = propose(proposal, participants, conditionMet).awaitUntil(proposeDeadline); + switch (propose.outcome) + { + default: throw new IllegalStateException(); + + case MAYBE_FAILURE: + throw propose.maybeFailure().markAndThrowAsTimeoutOrFailure(true, consistencyForConsensus, failedAttemptsDueToContention); + + case SUCCESS: + { + if (!conditionMet) + return conditionNotMet(current); + + // no need to commit a no-op; either it + // 1) reached a majority, in which case it was agreed, had no effect and we can do nothing; or + // 2) did not reach a majority, was not agreed, and was not user visible as a result so we can ignore it + if (!proposal.update.isEmpty()) + commit = commit(proposal.agreed(), participants, consistencyForConsensus, consistencyForCommit, true); + + break done; + } + + case SUPERSEDED: + { + switch (propose.superseded().hadSideEffects) + { + default: throw new IllegalStateException(); + + case MAYBE: + // We don't know if our update has been applied, as the competing ballot may have completed + // our proposal. We yield our uncertainty to the caller via timeout exception. + // TODO: should return more useful result to client, and should also avoid this situation where possible + throw new MaybeFailure(false, participants.sizeOfPoll(), participants.sizeOfConsensusQuorum, 0, emptyMap()) + .markAndThrowAsTimeoutOrFailure(true, consistencyForConsensus, failedAttemptsDueToContention); + + case NO: + minimumBallot = propose.superseded().by; + // We have been superseded without our proposal being accepted by anyone, so we can safely retry + Tracing.trace("Paxos proposal not accepted (pre-empted by a higher ballot)"); + if (!waitForContention(proposeDeadline, ++failedAttemptsDueToContention, metadata, partitionKey, consistencyForConsensus, WRITE)) + throw MaybeFailure.noResponses(participants).markAndThrowAsTimeoutOrFailure(true, consistencyForConsensus, failedAttemptsDueToContention); + } + } + } + // continue to retry + } + + if (commit != null) + { + PaxosCommit.Status result = commit.awaitUntil(commitDeadline); + if (!result.isSuccess()) + throw result.maybeFailure().markAndThrowAsTimeoutOrFailure(true, consistencyForCommit, failedAttemptsDueToContention); + } + Tracing.trace("CAS successful"); + return null; + + } + finally + { + final long latency = nanoTime() - start; + + if (failedAttemptsDueToContention > 0) + { + casWriteMetrics.contention.update(failedAttemptsDueToContention); + openAndGetStore(metadata).metric.topCasPartitionContention.addSample(partitionKey.getKey(), failedAttemptsDueToContention); + } + + + casWriteMetrics.addNano(latency); + writeMetricsMap.get(consistencyForConsensus).addNano(latency); + } + } + + private static RowIterator conditionNotMet(FilteredPartition read) + { + Tracing.trace("CAS precondition rejected", read); + casWriteMetrics.conditionNotMet.inc(); + return read.rowIterator(); + } + + public static PartitionIterator read(SinglePartitionReadCommand.Group group, ConsistencyLevel consistencyForConsensus) + throws InvalidRequestException, UnavailableException, ReadFailureException, ReadTimeoutException + { + long start = nanoTime(); + long deadline = start + DatabaseDescriptor.getReadRpcTimeout(NANOSECONDS); + return read(group, consistencyForConsensus, start, deadline); + } + + public static PartitionIterator read(SinglePartitionReadCommand.Group group, ConsistencyLevel consistencyForConsensus, long deadline) + throws InvalidRequestException, UnavailableException, ReadFailureException, ReadTimeoutException + { + return read(group, consistencyForConsensus, nanoTime(), deadline); + } + + private static PartitionIterator read(SinglePartitionReadCommand.Group group, ConsistencyLevel consistencyForConsensus, long start, long deadline) + throws InvalidRequestException, UnavailableException, ReadFailureException, ReadTimeoutException + { + if (group.queries.size() > 1) + throw new InvalidRequestException("SERIAL/LOCAL_SERIAL consistency may only be requested for one partition at a time"); + + int failedAttemptsDueToContention = 0; + Ballot minimumBallot = null; + SinglePartitionReadCommand read = group.queries.get(0); + try (PaxosOperationLock lock = PaxosState.lock(read.partitionKey(), read.metadata(), deadline, consistencyForConsensus, false)) + { + while (true) + { + // does the work of applying in-progress writes; throws UAE or timeout if it can't + final BeginResult begin = begin(deadline, read, consistencyForConsensus, false, minimumBallot, failedAttemptsDueToContention); + failedAttemptsDueToContention = begin.failedAttemptsDueToContention; + + switch (PAXOS_VARIANT) + { + default: throw new AssertionError(); + + case v2_without_linearizable_reads_or_rejected_writes: + case v2_without_linearizable_reads: + return begin.readResponse; + + case v2: + // no need to submit an empty proposal, as the promise will be treated as complete for future optimistic reads + if (begin.isLinearizableRead) + return begin.readResponse; + } + + Proposal proposal = Proposal.empty(begin.ballot, read.partitionKey(), read.metadata()); + PaxosPropose.Status propose = propose(proposal, begin.participants, false).awaitUntil(deadline); + switch (propose.outcome) + { + default: throw new IllegalStateException(); + + case MAYBE_FAILURE: + throw propose.maybeFailure().markAndThrowAsTimeoutOrFailure(false, consistencyForConsensus, failedAttemptsDueToContention); + + case SUCCESS: + return begin.readResponse; + + case SUPERSEDED: + switch (propose.superseded().hadSideEffects) + { + default: throw new IllegalStateException(); + + case MAYBE: + // We don't know if our update has been applied, as the competing ballot may have completed + // our proposal. We yield our uncertainty to the caller via timeout exception. + // TODO: should return more useful result to client, and should also avoid this situation where possible + throw new MaybeFailure(false, begin.participants.sizeOfPoll(), begin.participants.sizeOfConsensusQuorum, 0, emptyMap()) + .markAndThrowAsTimeoutOrFailure(true, consistencyForConsensus, failedAttemptsDueToContention); + + case NO: + minimumBallot = propose.superseded().by; + // We have been superseded without our proposal being accepted by anyone, so we can safely retry + Tracing.trace("Paxos proposal not accepted (pre-empted by a higher ballot)"); + if (!waitForContention(deadline, ++failedAttemptsDueToContention, group.metadata(), group.queries.get(0).partitionKey(), consistencyForConsensus, READ)) + throw MaybeFailure.noResponses(begin.participants).markAndThrowAsTimeoutOrFailure(true, consistencyForConsensus, failedAttemptsDueToContention); + } + } + } + } + finally + { + long latency = nanoTime() - start; + readMetrics.addNano(latency); + casReadMetrics.addNano(latency); + readMetricsMap.get(consistencyForConsensus).addNano(latency); + TableMetadata table = read.metadata(); + Keyspace.open(table.keyspace).getColumnFamilyStore(table.name).metric.coordinatorReadLatency.update(latency, TimeUnit.NANOSECONDS); + if (failedAttemptsDueToContention > 0) + casReadMetrics.contention.update(failedAttemptsDueToContention); + } + } + + static class BeginResult + { + final Ballot ballot; + final Participants participants; + final int failedAttemptsDueToContention; + final PartitionIterator readResponse; + final boolean isLinearizableRead; + final boolean isPromised; + final Ballot retryWithAtLeast; + + public BeginResult(Ballot ballot, Participants participants, int failedAttemptsDueToContention, PartitionIterator readResponse, boolean isLinearizableRead, boolean isPromised, Ballot retryWithAtLeast) + { + assert isPromised || isLinearizableRead; + this.ballot = ballot; + this.participants = participants; + this.failedAttemptsDueToContention = failedAttemptsDueToContention; + this.readResponse = readResponse; + this.isLinearizableRead = isLinearizableRead; + this.isPromised = isPromised; + this.retryWithAtLeast = retryWithAtLeast; + } + } + + /** + * Begin a Paxos operation by seeking promises from our electorate to be completed with proposals by our caller; and: + * + * - Completing any in-progress proposals witnessed, that are not known to have reached the commit phase + * - Completing any in-progress commits witnessed, that are not known to have reached a quorum of the electorate + * - Retrying and backing-off under contention + * - Detecting electorate mismatches with our peers and retrying to avoid non-overlapping + * electorates agreeing operations + * - Returning a resolved read response, and knowledge of if it is linearizable to read without proposing an empty update + * + * Optimisations: + * - If the promises report an incomplete commit (but have been able to witness it in a read response) + * we will submit the commit to those nodes that have not witnessed while waiting for those that have, + * returning as soon as a quorum is known to have witnessed the commit + * - If we witness an in-progress commit to complete, we batch the commit together with a new prepare + * restarting our operation. + * - If we witness an in-progress proposal to complete, after successfully proposing it we batch its + * commit together with a new prepare restarting our operation. + * + * @return the Paxos ballot promised by the replicas if no in-progress requests were seen and a quorum of + * nodes have seen the mostRecentCommit. Otherwise, return null. + */ + @SuppressWarnings("resource") + private static BeginResult begin(long deadline, + SinglePartitionReadCommand query, + ConsistencyLevel consistencyForConsensus, + final boolean isWrite, + Ballot minimumBallot, + int failedAttemptsDueToContention) + throws WriteTimeoutException, WriteFailureException, ReadTimeoutException, ReadFailureException + { + boolean acceptEarlyReadPermission = !isWrite; // if we're reading, begin by assuming a read permission is sufficient + Participants initialParticipants = Participants.get(query.metadata(), query.partitionKey(), consistencyForConsensus); + initialParticipants.assureSufficientLiveNodes(isWrite); + PaxosPrepare preparing = prepare(minimumBallot, initialParticipants, query, isWrite, acceptEarlyReadPermission); + while (true) + { + // prepare + PaxosPrepare retry = null; + PaxosPrepare.Status prepare = preparing.awaitUntil(deadline); + boolean isPromised = false; + retry: switch (prepare.outcome) + { + default: throw new IllegalStateException(); + + case FOUND_INCOMPLETE_COMMITTED: + { + FoundIncompleteCommitted incomplete = prepare.incompleteCommitted(); + Tracing.trace("Repairing replicas that missed the most recent commit"); + retry = commitAndPrepare(incomplete.committed, incomplete.participants, query, isWrite, acceptEarlyReadPermission); + break; + } + case FOUND_INCOMPLETE_ACCEPTED: + { + FoundIncompleteAccepted inProgress = prepare.incompleteAccepted(); + Tracing.trace("Finishing incomplete paxos round {}", inProgress.accepted); + if (isWrite) + casWriteMetrics.unfinishedCommit.inc(); + else + casReadMetrics.unfinishedCommit.inc(); + + // we DO NOT need to change the timestamp of this commit - either we or somebody else will finish it + // and the original timestamp is correctly linearised. By not updatinig the timestamp we leave enough + // information for nodes to avoid competing re-proposing the same proposal; if an in progress accept + // is equal to the latest commit (even if the ballots aren't) we're done and can abort earlier, + // and in fact it's possible for a CAS to sometimes determine if side effects occurred by reading + // the underlying data and not witnessing the timestamp of its ballot (or any newer for the relevant data). + Proposal repropose = new Proposal(inProgress.ballot, inProgress.accepted.update); + PaxosPropose.Status proposeResult = propose(repropose, inProgress.participants, false).awaitUntil(deadline); + switch (proposeResult.outcome) + { + default: throw new IllegalStateException(); + + case MAYBE_FAILURE: + throw proposeResult.maybeFailure().markAndThrowAsTimeoutOrFailure(isWrite, consistencyForConsensus, failedAttemptsDueToContention); + + case SUCCESS: + retry = commitAndPrepare(repropose.agreed(), inProgress.participants, query, isWrite, acceptEarlyReadPermission); + break retry; + + case SUPERSEDED: + // since we are proposing a previous value that was maybe superseded by us before completion + // we don't need to test the side effects, as we just want to start again, and fall through + // to the superseded section below + prepare = new PaxosPrepare.Superseded(proposeResult.superseded().by, inProgress.participants); + + } + } + + case SUPERSEDED: + { + Tracing.trace("Some replicas have already promised a higher ballot than ours; aborting"); + // sleep a random amount to give the other proposer a chance to finish + if (!waitForContention(deadline, ++failedAttemptsDueToContention, query.metadata(), query.partitionKey(), consistencyForConsensus, isWrite ? WRITE : READ)) + throw MaybeFailure.noResponses(prepare.participants).markAndThrowAsTimeoutOrFailure(true, consistencyForConsensus, failedAttemptsDueToContention); + retry = prepare(prepare.retryWithAtLeast(), prepare.participants, query, isWrite, acceptEarlyReadPermission); + break; + } + case PROMISED: isPromised = true; + case READ_PERMITTED: + { + // We have received a quorum of promises (or read permissions) that have all witnessed the commit of the prior paxos + // round's proposal (if any). + PaxosPrepare.Success success = prepare.success(); + + DataResolver resolver = new DataResolver(query, success.participants, NoopReadRepair.instance, query.creationTimeNanos()); + for (int i = 0 ; i < success.responses.size() ; ++i) + resolver.preprocess(success.responses.get(i)); + + class WasRun implements Runnable { boolean v; public void run() { v = true; } } + WasRun hadShortRead = new WasRun(); + PartitionIterator result = resolver.resolve(hadShortRead); + + if (!isPromised && hadShortRead.v) + { + // we need to propose an empty update to linearize our short read, but only had read success + // since we may continue to perform short reads, we ask our prepare not to accept an early + // read permission, when a promise may yet be obtained + // TODO: increase read size each time this happens? + acceptEarlyReadPermission = false; + break; + } + + return new BeginResult(success.ballot, success.participants, failedAttemptsDueToContention, result, !hadShortRead.v && success.isReadSafe, isPromised, success.supersededBy); + } + + case MAYBE_FAILURE: + throw prepare.maybeFailure().markAndThrowAsTimeoutOrFailure(isWrite, consistencyForConsensus, failedAttemptsDueToContention); + + case ELECTORATE_MISMATCH: + Participants participants = Participants.get(query.metadata(), query.partitionKey(), consistencyForConsensus); + participants.assureSufficientLiveNodes(isWrite); + retry = prepare(participants, query, isWrite, acceptEarlyReadPermission); + break; + + } + + if (retry == null) + { + Tracing.trace("Some replicas have already promised a higher ballot than ours; retrying"); + // sleep a random amount to give the other proposer a chance to finish + if (!waitForContention(deadline, ++failedAttemptsDueToContention, query.metadata(), query.partitionKey(), consistencyForConsensus, isWrite ? WRITE : READ)) + throw MaybeFailure.noResponses(prepare.participants).markAndThrowAsTimeoutOrFailure(true, consistencyForConsensus, failedAttemptsDueToContention); + retry = prepare(prepare.retryWithAtLeast(), prepare.participants, query, isWrite, acceptEarlyReadPermission); + } + + preparing = retry; + } + } + + public static boolean isInRangeAndShouldProcess(InetAddressAndPort from, DecoratedKey key, TableMetadata table, boolean includesRead) + { + Keyspace keyspace = Keyspace.open(table.keyspace); + return (includesRead ? EndpointsForToken.natural(keyspace, key.getToken()) + : ReplicaLayout.forTokenWriteLiveAndDown(keyspace, key.getToken()).all() + ).contains(getBroadcastAddressAndPort()); + } + + static ConsistencyLevel nonSerial(ConsistencyLevel serial) + { + switch (serial) + { + default: throw new IllegalStateException(); + case SERIAL: return QUORUM; + case LOCAL_SERIAL: return LOCAL_QUORUM; + } + } + + private static void mark(boolean isWrite, Function toMark, ConsistencyLevel consistency) + { + if (isWrite) + { + toMark.apply(casWriteMetrics).mark(); + toMark.apply(writeMetricsMap.get(consistency)).mark(); + } + else + { + toMark.apply(casReadMetrics).mark(); + toMark.apply(readMetricsMap.get(consistency)).mark(); + } + } + + public static Ballot newBallot(@Nullable Ballot minimumBallot, ConsistencyLevel consistency) + { + // We want a timestamp that is guaranteed to be unique for that node (so that the ballot is globally unique), but if we've got a prepare rejected + // already we also want to make sure we pick a timestamp that has a chance to be promised, i.e. one that is greater that the most recently known + // in progress (#5667). Lastly, we don't want to use a timestamp that is older than the last one assigned by ClientState or operations may appear + // out-of-order (#7801). + long minTimestampMicros = minimumBallot == null ? Long.MIN_VALUE : 1 + minimumBallot.unixMicros(); + // Note that ballotMicros is not guaranteed to be unique if two proposal are being handled concurrently by the same coordinator. But we still + // need ballots to be unique for each proposal so we have to use getRandomTimeUUIDFromMicros. + return nextBallot(minTimestampMicros, flag(consistency)); + } + + static Ballot staleBallotNewerThan(Ballot than, ConsistencyLevel consistency) + { + long minTimestampMicros = 1 + than.unixMicros(); + long maxTimestampMicros = BallotGenerator.Global.prevUnixMicros(); + maxTimestampMicros -= Math.min((maxTimestampMicros - minTimestampMicros) / 2, SECONDS.toMicros(5L)); + if (maxTimestampMicros <= minTimestampMicros) + return nextBallot(minTimestampMicros, flag(consistency)); + + return staleBallot(minTimestampMicros, maxTimestampMicros, flag(consistency)); + } + + /** + * Create a ballot uuid with the consistency level encoded in the timestamp. + * + * UUIDGen.getRandomTimeUUIDFromMicros timestamps are always a multiple of 10, so we add a 1 or 2 to indicate + * the consistency level of the operation. This should have no effect in practice (except preferring a serial + * operation over a local serial if there's a timestamp collision), but lets us avoid adding CL to the paxos + * table and messages, which should make backcompat easier if a different solution is committed upstream. + */ + public static Ballot ballotForConsistency(long whenInMicros, ConsistencyLevel consistency) + { + Preconditions.checkArgument(consistency.isSerialConsistency()); + return nextBallot(whenInMicros, flag(consistency)); + } + + private static Ballot.Flag flag(ConsistencyLevel consistency) + { + return consistency == SERIAL ? GLOBAL : LOCAL; + } + + public static ConsistencyLevel consistency(Ballot ballot) + { + switch (ballot.flag()) + { + default: return null; + case LOCAL: return LOCAL_SERIAL; + case GLOBAL: return SERIAL; + } + } + + static Map verifyElectorate(Electorate remoteElectorate, Electorate localElectorate) + { + // verify electorates; if they differ, send back gossip info for superset of two participant sets + if (remoteElectorate.equals(localElectorate)) + return emptyMap(); + + Map endpoints = Maps.newHashMapWithExpectedSize(remoteElectorate.size() + localElectorate.size()); + remoteElectorate.forEach(host -> endpoints.put(host, Gossiper.instance.getEndpointStateForEndpoint(host))); + localElectorate.forEach(host -> endpoints.putIfAbsent(host, Gossiper.instance.getEndpointStateForEndpoint(host))); + + return endpoints; + } + + public static boolean useV2() + { + switch (PAXOS_VARIANT) + { + case v2_without_linearizable_reads_or_rejected_writes: + case v2_without_linearizable_reads: + case v2: + return true; + case v1: + case v1_without_linearizable_reads_or_rejected_writes: + return false; + default: + throw new AssertionError(); + } + } + + public static boolean isLinearizable() + { + switch (PAXOS_VARIANT) + { + case v2: + case v1: + return true; + case v2_without_linearizable_reads_or_rejected_writes: + case v2_without_linearizable_reads: + case v1_without_linearizable_reads_or_rejected_writes: + return false; + default: + throw new AssertionError(); + } + } public static void setPaxosVariant(Config.PaxosVariant paxosVariant) { @@ -35,4 +1207,25 @@ public class Paxos { return PAXOS_VARIANT; } + + static boolean isOldParticipant(Replica replica) + { + String version = Gossiper.instance.getForEndpoint(replica.endpoint(), RELEASE_VERSION); + if (version == null) + return false; + + try + { + return new CassandraVersion(version).compareTo(MODERN_PAXOS_RELEASE) < 0; + } + catch (Throwable t) + { + return false; + } + } + + public static void evictHungRepairs() + { + PaxosTableRepairs.evictHungRepairs(); + } } diff --git a/src/java/org/apache/cassandra/service/paxos/Paxos.md b/src/java/org/apache/cassandra/service/paxos/Paxos.md new file mode 100644 index 0000000000..e8f1991d81 --- /dev/null +++ b/src/java/org/apache/cassandra/service/paxos/Paxos.md @@ -0,0 +1,345 @@ +# Light-weight transactions and Paxos algorithm + +Our implementation of light-weight transactions (LWT) is loosely based on the classic Paxos single-decision algorithm. +It contains a range of modifications that make it different from a direct application of a sequence of independent +decisions. + +Below we will describe the basic process (used by both V1 and V2 implementations) with the actions that each participant +takes and sketch a proof why the approach is safe, and then talk about various later improvements that do not change the +correctness of the basic scheme. + +A key consideration in the design was the fact that we apply LWTs independently to the partitions of a table, and the +necessary additional infrastructure (such as improved failure detection, client-side routing, etc) to efficiently +utilise a stable leader (i.e. Multi-Paxos). Instead of solving this we prefer to better support independent writes by +a leaderless scheme, where each coordinator attempts to direct its operations to completion. On congestion this runs +the chance of repeated clashes between coordinators attempting to perform writes to the same partition, in which case +we use randomized exponential backoff to achieve progress. + +The descriptions and proofs below assume no non-LWT modifications to the partition. Any other write (regardless of the +requested consistency level) may reach a minority of replicas and leave the partition in an inconsistent state where the +following operations can go different ways depending on which set of replicas respond to a quorum read. In particular, +if two compare-and-set (CAS) operations with the same condition are executed with no intervening operations, such a +state would make it possible for the first to fail and the second to succeed. + +## Basic scheme + +For each partition with LWT modification, we maintain the following registers: + +- The current `promised` ballot number. +- The latest `accepted` proposal together with its ballot number and a `committed` flag. + +We define an ordering on the latter by ballot number and committed flag, where `committed` true is interpreted as +greater on equal ballot numbers. Two proposals are equal when their ballot numbers and committed flag match. + +The basic scheme includes the following stages: + +1. A coordinator selects a fresh ballot number (based on time and made unique by including a source node identifier). +2. The coordinator sends a `prepare` message to all replicas responsible for the targeted partition with the given + ballot number and a request to read the data required by the operation. +3. By accepting the message, a replica declares that it will not accept any `prepare` or `propose` message with smaller + ballot number: + 1. If the replica's `promised` register is greater than the number included in the message, the replica rejects it, + sending back its current `promised` number in a `rejected` message. + 2. Otherwise, the replica stores the ballot number from the message in its `promised` register and replies with a + `promise` message which includes the contents of the current `accepted` register together with the requested data + from the database. +4. On receipt of `promise`s from a quorum of nodes, the coordinator compiles the "most recently accepted" (`MRA`) value as + the greatest among the accepted values in the promises and then: + 1. If the `MRA` is not null and its committed flag is not true, there is an in-progress Paxos session that needs to be + completed and committed. Controller reproposes the value with the new ballot (i.e. follows the steps from (iv) + below with a proposed value equal to the `MRA`'s) and then restarts the process. + 2. If the `MRA` is not null, its committed flag is true and it is not a match for the `accepted` value of a quorum of + promises, controller sends a `commit` message to all replicas whose value did not match and awaits responses + until they form a quorum of replicas with a matching `accepted` value. + 3. The coordinator then creates a proposal, a partition update which is the result of applying the operation, using + a read result obtained by resolving the quorum of read responses; the partition update has a timestamp that + corresponds to the proposal's ballot number, and is empty if the operation was a read or the CAS failed. + 4. It then sends the proposal as a `propose` message with the value and the current ballot number to all replicas. +5. A replica accepts the proposal if it has not promised that it will not do so: + 1. If its `promised` ballot number is not higher than the proposal's, it sets its `accepted` register to the + proposal with its ballot number and a false `committed` flag, updates its `promised` register to the proposal's + ballot and sends back an `accepted` message. + 2. Otherwise it rejects the proposal by sending the current `promised` number in a `rejected` message. +6. On receipt of `accepted` messages from a quorum of nodes, the Paxos session has reached a decision. The coordinator + completes it by sending `commit` messages to all replicas, attaching the proposal value with its ballot number. + 1. It can return completion without waiting for receipt of any commit messages. +7. A replica accepts a commit unconditionally, by applying the attached partition update. If the commit's ballot number + is greater than the replica's `accepted` ballot number, it sets its `accepted` register to the message's value and + ballot with true `committed` flag, and the `promised` register to its ballot number. +8. If at any stage of the process that requires a quorum the quorum is not reached, the coordinator backs off and then + restarts the process from the beginning using a fresh ballot that is greater than the ballot contained in any + `rejected` message received. + +We can't directly map multi-instance classic Paxos onto this, but we can follow its correctness proofs to ensure the +following propositions: + +1. Once a value (possibly empty) has been decided (i.e. accepted by a majority), no earlier proposal can be reproposed. +2. Once a value has been decided, in any further round (i.e. action taken by any proposer) it will either be accepted + and/or committed, or it will have been committed in an earlier round and will be witnessed by at least one member of + the quorum. + +Suppose the value V was decided in the round with ballot number E0. + +Proposition 1 is true because new proposals can only be made after a successful promise round with ballot number E > E0. +To be successful, it needs a quorum which must intersect in at least one replica with E0's decision quorum. Since that +replica accepted that proposal, it cannot have made a promise on E before that and must thus return an accepted value +whose ballot number is at least E0. This is true because both `propose` and `commit` actions can only replace the +`accepted` value with one with a higher or equal ballot number. + +Proposition 2 can be proven true by induction on the following invariant: for any successful ballot number E >= E0, +either: + +1. For all quorums of replicas, commit V with some ballot number G < E, G >= E0 was witnessed by some replica in the + quorum. +2. For all quorums of replicas, the `accepted` value with the highest ballot number among the replicas in the quorum is + V with some ballot number G where G <= E, G >= E0. + +For round E == E0 the invariant is true because all quorums contain a replica that accepted V with ballot E0, and E0 is +the newest ballot number. + +Suppose the invariant is true for some round E and examine the behaviour of the algorithm on F = succ(E). + +- If 1. was true for E, it remains true for F. +- Otherwise 2. was true for E and: + - If the promise pass did not reach a quorum, no accepted values change and hence 2. is still true. + - If the promise reached a quorum, the collected `MRA` is V with ballot G. + - If the `MRA`'s committed flag is not true, the value V is reproposed with ballot F. + - If the proposal reaches no node, no accepted values change and hence 2. is still true. + - If the proposal reaches a minority of nodes, any quorum that includes one of these nodes will have V with + ballot F as their highest `accepted` value. All other quorums will not have changed and still have V as their + highest accepted value with an earlier ballot >= E0. In any case, 2. is still true. + - If the proposal reaches a majority of nodes, all quorums have V with ballot F as the highest `accepted` value + and thus satisfy 2. + - Any `commit` message that is issued after a majority can only change the accepted value's `committed` flag -- + all quorums will still satisfy 2. + - If the `MRA`'s committed flag is true but it is not matched in all responses, a `commit` with this `MRA` is sent to + all replicas. By the reasoning above, 2. is still true regardless how many of them (if any) are received and + processed. + - If the committed `MRA` matches in all responses (initially or because of commits issued in the last step), then 1. + is true for G <= E < F regardless of any further action taken by the coordinator. + +Proposition 1 ensures that we can only commit one value in any concurrent operation. Proposition 2 means that any +accepted proposal must have started its promise after the previous value was committed to at least one replica in any +quorum, and hence must be able to see its effects in its read responses. This is also true for every other value that +was accepted in any previous ballot. + +Note that each commit may modify separate parts of the partition or overwrite previous values. Each of these updates may +be witnessed by a different replica, but they must all be witnessed in the responses the coordinator receives prior to +making a proposal or completing a read. By virtue of having their timestamps reflect ballot order, the read resolution +process can correctly restore the combined state. + +As writes are only done in the `commit` stage after a value has been accepted, no undecided writes can be reflected in +read responses. + +## Insubstantial differences with the actual code + +Instead of using a unique node identifier as part of the ballot number, we generate a 64-bit random integer. This has an +extremely low chance of collision that can be assumed to be immaterial. + +Instead of storing an `accepted` proposal with a committed flag, for historical reasons the actual implementation +separately stores the latest `accepted` and `committed` values. The coordinator computes most recent values for both +after receiving promises, and acts on the higher of the two. In other words, instead of checking the `committed` flag on +the most recently accepted, it checks if whether the `committed` value has a higher ballot than the `accepted` one. + +When accepting a proposal (which is conditional on having no newer promise or accepted/committed proposal), it stores it +in the `accepted` register. When accepting a commit (which is unconditional), it replaces the `commit` register only if +the commit has a newer ballot. It will clear the `accepted` value to null if that value does not have a newer ballot. + +Additionally, the `promised` value is not adjusted with accepted proposals and commits. Instead, whenever the code +decides whether to promise or accept, it collects the maximum of the promised, accepted and committed ballots. + +Version 2 of the Paxos implementation performs reads as described here, by combining them with the `prepare`/`promise` +messages. Version 1 runs quorum reads separately after receiving a promise; the read cannot complete if any write +reaches consensus after that promise and, if successful, it will in turn invalidate and proposals that it may fail to +see. + +These differences do not materially change the logic, but make correctness a little harder to prove directly. + +## Skipping commit for empty proposals (Versions 1 and 2) + +Since empty proposals make no modifications to the database state, it is not necessary to commit them. + +More precisely, in the basic scheme above we can treat the case of an empty partition update as the most-recently +accepted value in the coordinator's preparation phase like we would treat a null `MRA`, skipping phases i and ii. In other +words, we can change 4(i) to: + +4. + 1. If the `MRA` is not null or empty, and its committed flag is not true, there is an in-progress Paxos session that + needs to be completed and committed. Controller reproposes the value with the new ballot (i.e. follow the steps + below with a proposed value equal to the `MRA`'s) and then restart the process. + +With this modified step Proposition 1 is still true, as is Proposition 2 restricted to committing and witnessing +non-empty proposals. Their combination still ensures that no update made concurrently with any operation (including a +read or a failing CAS) can resurface after that operation is decided, and that any operation will see the results of +applying any previous. + +## Skipping proposal for reads (Version 2 only) + +To ensure correct ordering of reads and unsuccessful CAS operations, the algorithm above forces invalidation of any +concurrent operation by issuing an empty update. It is possible, however, to recognize if a concurrent operation may +have started at the time a `promise` is given. If no such operation is present, the read may proceed without issuing an +empty update. + +To recognize this, the current `promised` value is returned with `promise` messages. During the proposal generation +phase, the coordinator compiles the maximum returned `promised` number and compares it against the `MRA`'s. If higher, a +concurrent operation is in place and all reads must issue an empty update to ensure proper ordering. If not, the empty +update may be skipped. + +In other words, step 3(ii) changes to the following: + +3. + 2. Otherwise, the replica stores the ballot number from the message in its `promised` register and replies with a + `promise` message which includes the contents of the current `accepted` and previous `promised` registers together + with the requested data from the database. + +and a new step is inserted before 4(iv) (which becomes 4(v)): + +4. + 4. If the proposal is empty (i.e. the operation was a read or the CAS failed), the coordinator checks the maximum of + the quorum's `promised` values agains the `MRA`'s ballot number. If that maximum isn't higher, the operation + completes. + +Since we do not change issued proposals or make new ones, Proposition 1 is still in force. For Proposition 2 we must +consider the possibility of a no-propose read missing an update with an earlier ballot number that was decided on +concurrently. The difference in the new scheme is that this read will not invalidate an incomplete concurrent write, and +thus an undecided entry could be decided after the read executes. However, to propose a new entry, a coordinator must +first obtain a quorum of promises using a ballot number greater than the last committed or empty value's. Given such a +promise and a read executing with higher ballot, at least one of the reader's quorum replicas must return its ballot +number or higher in the `promised` field (otherwise the read's promise will have executed before the write's and the +preparation would have been rejected). As a result, the coordinator will see a concurrent operation (either in a +non-committed `MRA`, or a `promised` value higher than a committed or empty `MRA`) and will proceed to issue an invalidating +empty proposal. + +## Concurrent reads (Version 2 only) + +As stated, the above optimization is only useful once per successful proposal, because a read executed in this way does +not complete and will be treated as concurrent with any operation started after it. To improve this, we can use the fact +that reads do not affect other reads, i.e. they are commutative operations and the order of execution of a set of reads +with no concurrent writes is not significant, and separately store and issue read-only and write promises. + +More precisely, `prepare` messages are augmented with an `isWrite` flag, and an additional register called +`promisedWrite` is maintained. The latter is updated when a promise is given for a `prepare` message with a true +`isWrite` field, and is returned with all `promise` messages (in addition to `promised` as above). When a promise is +requested with a ballot number lower than the current `promised` but higher than `promisedWrite`, the replica does not +reject the request, but issues a "read-only promise" (note that this can be a normal `promise` message, recognized by +`promised` being greater than the coordinator-issued ballot number), which cannot be used for making any proposal. + +The condition for making a no-proposal read is that the maximum returned `promisedWrite` number is not higher than the +`MRA`'s (i.e. concurrent reads are permitted, but not concurrent writes). Provided that this is the case, the coordinator +can use a read-only promise to execute no-proposal reads and failing CAS's. If they are not, it must restart the +process, treating the read-only promises as rejections. + +Steps 2, 3, 4 and 8 are changed to accommodate this. The modified algorithm becomes: + +1. A coordinator selects a fresh ballot number (based on time and made unique by including a source node identifier). +2. The coordinator sends a `prepare` message to all replicas responsible for the targeted partition with the given + ballot number, `isWrite` set to true if the operation is a CAS and false if it is a read, and a request to read the + data required by the operation. +3. By accepting the message, a replica declares that it will not accept any `propose` message or issue write promises + with smaller ballot number: + 1. If the replica's `promisedWrite` register is greater than the number included in the message, the replica rejects + it, sending back its current `promised` number in a `rejected` message. + 2. A `read-only` flag is initialized to false. + 3. If the `promised` register contains a lower value than the one supplied by the message, the `promised` register + is updated. Otherwise, the `read-only` flag is set to true. + 4. If the message's `isWrite` flag is true and `read-only` is still false, the `promisedWrite` register is updated + to the passed ballot number. Otherwise, `read-only` is set to true. + 5. The replica replies with a `promise` message which includes the `read-only` flag, the contents of the current + `accepted` and previous `promised` and `promisedWrite` registers together with the requested data from the + database. +4. On receipt of `promise`s from a quorum of nodes, the coordinator compiles the "most recently accepted" (`MRA`) value as + the greatest among the accepted values in the promises and then: + 1. If the `MRA` is not null or empty, and its committed flag is not true, there is an in-progress Paxos session that + needs to be completed and committed. The coordinator prepares a reproposal of the value with the new ballot, + continuing with step (v) below, and then restarts the process. + 2. If the `MRA` is not null, its committed flag is true and it is not a match for the `accepted` value of a quorum of + promises, controller sends a `commit` message to all replicas whose value did not match and awaits responses + until they form a quorum of replicas with a matching `accepted` value. + 3. The coordinator then creates a proposal, a partition update which is the result of applying the operation, using + a read result obtained by resolving the quorum of read responses; the partition update is empty if the operation + was a read or the CAS failed. + 4. If the proposal is empty (i.e. the operation was a read or the CAS failed), the coordinator checks the maximum of + the quorum's `promisedWrite` values agains the `MRA`'s ballot number. If that maximum isn't higher, the operation + completes. + 5. If there was no quorum of promises with false `read-only` flag, the coordinator restarts the process (step 8). + 6. Otherwise, it sends the proposal as a `propose` message with the value and the current ballot number to all + replicas. +5. A replica accepts the proposal if it has not promised that it will not do so: + 1. If its `promised` ballot number is not higher than the proposal's, it sets its `accepted` register to the + proposal with its ballot number and a false `committed` flag, updates its `promised` register to the proposal's + ballot and sends back an `accepted` message. + 2. Otherwise, it rejects the proposal by sending the current `promised` number in a `rejected` message. +6. On receipt of `accepted` messages from a quorum of nodes, the Paxos session has reached a decision. The coordinator + completes it by sending `commit` messages to all replicas, attaching the proposal value and its ballot number. + 1. It can return completion without waiting for receipt of any commit messages. +7. A replica accepts a commit unconditionally, by applying the attached partition update. If the commit's ballot number + is greater than the replica's `accepted` ballot number, it sets its `accepted` register to the message's value and + ballot with true `committed` flag, and the `promised` register to its ballot number. +8. If at any stage of the process that requires a quorum the quorum is not reached, the coordinator backs off and then + restarts the process from the beginning using a fresh ballot that is greater than the `promised` ballot contained in + any `rejected` and `promise` message received. + +With respect to any operation that issues a proposal, this algorithm fully matches the earlier version. For operations +that do not (including all operations executing with a read-only promise), it allows for multiple to execute +concurrently as long as no write promise quorum has been reached after the last commit. The reasoning of the previous +paragraph is still valid and proves that no older proposal can be agreed on after a no-proposal read. + +## Paxos system table expiration (Version 1 only) + +The Paxos state registers used by the algorithm are persisted in the Paxos system table. For every partition with LWTs, +this table will contain an entry specifying the current values of all registers (promised, promisedWrite, accepted, +committed). Because this information is per-partition, there is a high chance that this table will quickly become very +large if LWTs are used with many independent partitions. + +To make sure the overhead of the Paxos system table remains limited, Version 1 of the Cassandra Paxos implementation +specifies a time-to-live (TTL) for all writes. That is, after a certain period of time with no LWT to a partition, the +replica will forget the partition's Paxos state. + +If this data expires, any in-progress operations may fail to be brought to completion. With the algorithm as described +above, one of the effects of this is that some writes that are reported complete may fail to ever be committed on a +majority of replicas, or even on any replica (if e.g. connection with the replicas is lost before commits are sent, and +the TTL expires before any new LWT operation on the permition is initiated). + +To avoid this problem, Version 1 of the implementation only reports success on writes after the commit stage has reached +a requested consistency level. This solves the problem of reporting success, but only guarantees LWT writes to behave +like non-LWT ones: a write may still reach a minority of nodes and leave the partition in an inconsistent state, which +permits linearity guarantees to be violated. + +## Paxos repair (Version 2 only) + +In the second version of the implementation the Paxos system table does not expire. Instead, clearing up state is +performed by a "Paxos repair" process which actively processes unfinished Paxos sessions and only deletes state that is +known to have been brought to completion (i.e. successful majority commit). + +The repair process starts with taking a snapshot of the uncommitted operations in the Paxos system table. It then takes +their ballots' maximum, which is then distributed to all nodes to be used as a lower bound on all promises, i.e. +replicas stop accepting messages with earlier ballots. The process then proceeds to perform the equivalent of an empty +read on all partitions in the snapshot. Upon completion, it can guarantee that all proposals with a ballot lower than +the bound have been completed, i.e. either accepted and committed or superseded in a majority of nodes. + +What this means is that no LWT with earlier ballot can be incomplete, and thus no longer need any earlier state in the +Paxos system table. The actual data deletion happens on compaction, where we drop all data that has lower ballots than +what we know to have been repaired. + +## Correctness in the presence of range movements (Version 2 only) + +The crucial requirements for any Paxos-like scheme to work is to only make progress when we are guaranteed that all +earlier decision points have at least one representative among the replicas that reply to a message (in other words, +that all quorums intersect). When the node set is fixed this is achieved by requesting that a quorum contains more than +half the replicas for the given partition. + +Range movements (i.e. joining or leaving nodes), however, can change the set of replicas that are responsible for a +partition. In the exteme case, after multiple range movements it is possible to have a completely different set of +replicas responsible for the partition (i.e. no quorum can exist that contains a replica for all earlier transactions). +To deal with this problem, we must ensure that: + +- While operations in an incomplete state are ongoing, a quorum is formed in such a way that it contains a majority for + the current replica set _as well as_ for the replica set before the operation was started (as well as any intermediate + set, if it possible to perform multiple range movements in parallel). +- By the time we transition fully to a new set of replicas responsible for a partition, we have completed moving all + committed mutations from any source replica to its replacement. + +The Paxos repair process above gives us a way to complete ongoing operations and a point in time when we can assume that +all earlier LWT operations are complete. In combination with streaming, which moves all committed data to the new +replica, this means that from the point when both complete forward we can safely use the new replica in quorums in place +of the source. \ No newline at end of file diff --git a/src/java/org/apache/cassandra/service/paxos/PaxosCommit.java b/src/java/org/apache/cassandra/service/paxos/PaxosCommit.java new file mode 100644 index 0000000000..246fed7799 --- /dev/null +++ b/src/java/org/apache/cassandra/service/paxos/PaxosCommit.java @@ -0,0 +1,325 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.paxos; + +import java.util.concurrent.atomic.AtomicLongFieldUpdater; +import java.util.function.Consumer; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.concurrent.ExecutorPlus; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.db.ConsistencyLevel; +import org.apache.cassandra.db.Mutation; +import org.apache.cassandra.exceptions.RequestFailureReason; +import org.apache.cassandra.locator.EndpointsForToken; +import org.apache.cassandra.locator.InOurDc; +import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.locator.Replica; +import org.apache.cassandra.net.IVerbHandler; +import org.apache.cassandra.net.Message; +import org.apache.cassandra.net.MessagingService; +import org.apache.cassandra.net.NoPayload; +import org.apache.cassandra.service.paxos.Paxos.Participants; +import org.apache.cassandra.tracing.Tracing; +import org.apache.cassandra.utils.concurrent.ConditionAsConsumer; + +import static java.util.Collections.emptyMap; +import static org.apache.cassandra.exceptions.RequestFailureReason.NODE_DOWN; +import static org.apache.cassandra.exceptions.RequestFailureReason.UNKNOWN; +import static org.apache.cassandra.net.Verb.PAXOS2_COMMIT_REMOTE_REQ; +import static org.apache.cassandra.net.Verb.PAXOS_COMMIT_REQ; +import static org.apache.cassandra.service.StorageProxy.shouldHint; +import static org.apache.cassandra.service.StorageProxy.submitHint; +import static org.apache.cassandra.service.paxos.Commit.*; +import static org.apache.cassandra.utils.concurrent.ConditionAsConsumer.newConditionAsConsumer; + +// Does not support EACH_QUORUM, as no such thing as EACH_SERIAL +public class PaxosCommit> extends PaxosRequestCallback +{ + public static final RequestHandler requestHandler = new RequestHandler(); + private static final Logger logger = LoggerFactory.getLogger(PaxosCommit.class); + + private static volatile boolean ENABLE_DC_LOCAL_COMMIT = Boolean.parseBoolean(System.getProperty("cassandra.enable_dc_local_commit", "true")); + + public static boolean getEnableDcLocalCommit() + { + return ENABLE_DC_LOCAL_COMMIT; + } + + public static void setEnableDcLocalCommit(boolean enableDcLocalCommit) + { + ENABLE_DC_LOCAL_COMMIT = enableDcLocalCommit; + } + + /** + * Represents the current status of a commit action: it is a status rather than a result, + * as the result may be unknown without sufficient responses (though in most cases it is final status). + */ + static class Status + { + private final Paxos.MaybeFailure maybeFailure; + + Status(Paxos.MaybeFailure maybeFailure) + { + this.maybeFailure = maybeFailure; + } + + boolean isSuccess() { return maybeFailure == null; } + Paxos.MaybeFailure maybeFailure() { return maybeFailure; } + + public String toString() { return maybeFailure == null ? "Success" : maybeFailure.toString(); } + } + + private static final Status success = new Status(null); + + private static final AtomicLongFieldUpdater responsesUpdater = AtomicLongFieldUpdater.newUpdater(PaxosCommit.class, "responses"); + + final Agreed commit; + final boolean allowHints; + final ConsistencyLevel consistencyForConsensus; + final ConsistencyLevel consistencyForCommit; + + final EndpointsForToken replicas; + final int required; + final OnDone onDone; + + /** + * packs two 32-bit integers; + * bit 00-31: accepts + * bit 32-63: failures/timeouts + * + * {@link #accepts} + * {@link #failures} + */ + private volatile long responses; + + public PaxosCommit(Agreed commit, boolean allowHints, ConsistencyLevel consistencyForConsensus, ConsistencyLevel consistencyForCommit, Participants participants, OnDone onDone) + { + this.commit = commit; + this.allowHints = allowHints; + this.consistencyForConsensus = consistencyForConsensus; + this.consistencyForCommit = consistencyForCommit; + this.replicas = participants.all; + this.onDone = onDone; + this.required = participants.requiredFor(consistencyForCommit); + if (required == 0) + onDone.accept(status()); + } + + /** + * Submit the proposal for commit with all replicas, and wait synchronously until at most {@code deadline} for the result + */ + static Paxos.Async commit(Agreed commit, Participants participants, ConsistencyLevel consistencyForConsensus, ConsistencyLevel consistencyForCommit, @Deprecated boolean allowHints) + { + // to avoid unnecessary object allocations we extend PaxosPropose to implements Paxos.Async + class Async extends PaxosCommit> implements Paxos.Async + { + private Async(Agreed commit, boolean allowHints, ConsistencyLevel consistencyForConsensus, ConsistencyLevel consistencyForCommit, Participants participants) + { + super(commit, allowHints, consistencyForConsensus, consistencyForCommit, participants, newConditionAsConsumer()); + } + + public Status awaitUntil(long deadline) + { + try + { + onDone.awaitUntil(deadline); + } + catch (InterruptedException e) + { + Thread.currentThread().interrupt(); + return new Status(new Paxos.MaybeFailure(true, replicas.size(), required, 0, emptyMap())); + } + + return status(); + } + } + + Async async = new Async(commit, allowHints, consistencyForConsensus, consistencyForCommit, participants); + async.start(participants, false); + return async; + } + + /** + * Submit the proposal for commit with all replicas, and wait synchronously until at most {@code deadline} for the result + */ + static > T commit(Agreed commit, Participants participants, ConsistencyLevel consistencyForConsensus, ConsistencyLevel consistencyForCommit, @Deprecated boolean allowHints, T onDone) + { + new PaxosCommit<>(commit, allowHints, consistencyForConsensus, consistencyForCommit, participants, onDone) + .start(participants, true); + return onDone; + } + + /** + * Send commit messages to peers (or self) + */ + void start(Participants participants, boolean async) + { + boolean executeOnSelf = false; + Message commitMessage = Message.out(PAXOS_COMMIT_REQ, commit); + Message mutationMessage = ENABLE_DC_LOCAL_COMMIT && consistencyForConsensus.isDatacenterLocal() + ? Message.out(PAXOS2_COMMIT_REMOTE_REQ, commit.makeMutation()) : null; + + for (int i = 0, mi = participants.allLive.size(); i < mi ; ++i) + executeOnSelf |= isSelfOrSend(commitMessage, mutationMessage, participants.allLive.endpoint(i)); + + for (int i = 0, mi = participants.allDown.size(); i < mi ; ++i) + onFailure(participants.allDown.endpoint(i), NODE_DOWN); + + if (executeOnSelf) + { + ExecutorPlus executor = PAXOS_COMMIT_REQ.stage.executor(); + if (async) executor.execute(this::executeOnSelf); + else executor.maybeExecuteImmediately(this::executeOnSelf); + } + } + + /** + * If isLocal return true; otherwise if the destination is alive send our message, and if not mark the callback with failure + */ + private boolean isSelfOrSend(Message commitMessage, Message mutationMessage, InetAddressAndPort destination) + { + if (shouldExecuteOnSelf(destination)) + return true; + + // don't send commits to remote dcs for local_serial operations + if (mutationMessage != null && !isInLocalDc(destination)) + MessagingService.instance().sendWithCallback(mutationMessage, destination, this); + else + MessagingService.instance().sendWithCallback(commitMessage, destination, this); + + return false; + } + + private static boolean isInLocalDc(InetAddressAndPort destination) + { + return DatabaseDescriptor.getLocalDataCenter().equals(DatabaseDescriptor.getEndpointSnitch().getDatacenter(destination)); + } + + /** + * Record a failure or timeout, and maybe submit a hint to {@code from} + */ + @Override + public void onFailure(InetAddressAndPort from, RequestFailureReason reason) + { + if (logger.isTraceEnabled()) + logger.trace("{} {} from {}", commit, reason, from); + + response(false, from); + Replica replica = replicas.lookup(from); + + if (allowHints && shouldHint(replica)) + submitHint(commit.makeMutation(), replica, null); + } + + /** + * Record a success response + */ + public void onResponse(Message response) + { + logger.trace("{} Success from {}", commit, response.from()); + + response(true, response.from()); + } + + /** + * Execute locally and record response + */ + public void executeOnSelf() + { + executeOnSelf(commit, RequestHandler::execute); + } + + @Override + public void onResponse(NoPayload response, InetAddressAndPort from) + { + response(response != null, from); + } + + /** + * Record a failure or success response if {@code from} contributes to our consistency. + * If we have reached a final outcome of the commit, run {@code onDone}. + */ + private void response(boolean success, InetAddressAndPort from) + { + if (consistencyForCommit.isDatacenterLocal() && InOurDc.endpoints().test(from)) + return; + + long responses = responsesUpdater.addAndGet(this, success ? 0x1L : 0x100000000L); + // next two clauses mutually exclusive to ensure we only invoke onDone once, when either failed or succeeded + if (accepts(responses) == required) // if we have received _precisely_ the required accepts, we have succeeded + onDone.accept(status()); + else if (replicas.size() - failures(responses) == required - 1) // if we are _unable_ to receive the required accepts, we have failed + onDone.accept(status()); + } + + /** + * @return the Status as of now, which may be final or may indicate we have not received sufficient responses + */ + Status status() + { + long responses = this.responses; + if (isSuccessful(responses)) + return success; + + return new Status(new Paxos.MaybeFailure(replicas.size(), required, accepts(responses), failureReasonsAsMap())); + } + + private boolean isSuccessful(long responses) + { + return accepts(responses) >= required; + } + + private static int accepts(long responses) + { + return (int) (responses & 0xffffffffL); + } + + private static int failures(long responses) + { + return (int) (responses >>> 32); + } + + public static class RequestHandler implements IVerbHandler + { + @Override + public void doVerb(Message message) + { + NoPayload response = execute(message.payload, message.from()); + // NOTE: for correctness, this must be our last action, so that we cannot throw an error and send both a response and a failure response + if (response == null) + MessagingService.instance().respondWithFailure(UNKNOWN, message); + else + MessagingService.instance().respond(response, message); + } + + private static NoPayload execute(Agreed agreed, InetAddressAndPort from) + { + if (!Paxos.isInRangeAndShouldProcess(from, agreed.update.partitionKey(), agreed.update.metadata(), false)) + return null; + + PaxosState.commitDirect(agreed); + Tracing.trace("Enqueuing acknowledge to {}", from); + return NoPayload.noPayload; + } + } + +} diff --git a/src/java/org/apache/cassandra/service/paxos/PaxosCommitAndPrepare.java b/src/java/org/apache/cassandra/service/paxos/PaxosCommitAndPrepare.java new file mode 100644 index 0000000000..7270cc5eac --- /dev/null +++ b/src/java/org/apache/cassandra/service/paxos/PaxosCommitAndPrepare.java @@ -0,0 +1,145 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.paxos; + +import java.io.IOException; + +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.SinglePartitionReadCommand; +import org.apache.cassandra.io.util.DataInputPlus; +import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.net.IVerbHandler; +import org.apache.cassandra.net.Message; +import org.apache.cassandra.net.MessagingService; +import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.service.paxos.Commit.Agreed; +import org.apache.cassandra.tracing.Tracing; +import org.apache.cassandra.service.paxos.Ballot; + +import static org.apache.cassandra.exceptions.RequestFailureReason.UNKNOWN; +import static org.apache.cassandra.net.Verb.PAXOS2_COMMIT_AND_PREPARE_REQ; +import static org.apache.cassandra.service.paxos.Paxos.newBallot; +import static org.apache.cassandra.service.paxos.PaxosPrepare.start; + +public class PaxosCommitAndPrepare +{ + public static final RequestSerializer requestSerializer = new RequestSerializer(); + public static final RequestHandler requestHandler = new RequestHandler(); + + static PaxosPrepare commitAndPrepare(Agreed commit, Paxos.Participants participants, SinglePartitionReadCommand readCommand, boolean isWrite, boolean acceptEarlyReadSuccess) + { + Ballot ballot = newBallot(commit.ballot, participants.consistencyForConsensus); + Request request = new Request(commit, ballot, participants.electorate, readCommand, isWrite); + PaxosPrepare prepare = new PaxosPrepare(participants, request, acceptEarlyReadSuccess, null); + + Tracing.trace("Committing {}; Preparing {}", commit.ballot, ballot); + Message message = Message.out(PAXOS2_COMMIT_AND_PREPARE_REQ, request); +// .permitsArtificialDelay(participants.consistencyForConsensus); + start(prepare, participants, message, RequestHandler::execute); + return prepare; + } + + private static class Request extends PaxosPrepare.AbstractRequest + { + final Agreed commit; + + Request(Agreed commit, Ballot ballot, Paxos.Electorate electorate, SinglePartitionReadCommand read, boolean isWrite) + { + super(ballot, electorate, read, isWrite); + this.commit = commit; + } + + private Request(Agreed commit, Ballot ballot, Paxos.Electorate electorate, DecoratedKey partitionKey, TableMetadata table, boolean isWrite) + { + super(ballot, electorate, partitionKey, table, isWrite); + this.commit = commit; + } + + Request withoutRead() + { + return new Request(commit, ballot, electorate, partitionKey, table, isForWrite); + } + + public String toString() + { + return commit.toString("CommitAndPrepare(") + ", " + Ballot.toString(ballot) + ')'; + } + } + + public static class RequestSerializer extends PaxosPrepare.AbstractRequestSerializer + { + Request construct(Agreed param, Ballot ballot, Paxos.Electorate electorate, SinglePartitionReadCommand read, boolean isWrite) + { + return new Request(param, ballot, electorate, read, isWrite); + } + + Request construct(Agreed param, Ballot ballot, Paxos.Electorate electorate, DecoratedKey partitionKey, TableMetadata table, boolean isWrite) + { + return new Request(param, ballot, electorate, partitionKey, table, isWrite); + } + + @Override + public void serialize(Request request, DataOutputPlus out, int version) throws IOException + { + Agreed.serializer.serialize(request.commit, out, version); + super.serialize(request, out, version); + } + + @Override + public Request deserialize(DataInputPlus in, int version) throws IOException + { + Agreed committed = Agreed.serializer.deserialize(in, version); + return deserialize(committed, in, version); + } + + @Override + public long serializedSize(Request request, int version) + { + return Agreed.serializer.serializedSize(request.commit, version) + + super.serializedSize(request, version); + } + } + + public static class RequestHandler implements IVerbHandler + { + @Override + public void doVerb(Message message) + { + PaxosPrepare.Response response = execute(message.payload, message.from()); + if (response == null) + MessagingService.instance().respondWithFailure(UNKNOWN, message); + else + MessagingService.instance().respond(response, message); + } + + private static PaxosPrepare.Response execute(Request request, InetAddressAndPort from) + { + Agreed commit = request.commit; + if (!Paxos.isInRangeAndShouldProcess(from, commit.update.partitionKey(), commit.update.metadata(), request.read != null)) + return null; + + try (PaxosState state = PaxosState.get(commit)) + { + state.commit(commit); + return PaxosPrepare.RequestHandler.execute(request, state); + } + } + } +} diff --git a/test/simulator/main/org/apache/cassandra/simulator/debug/Capture.java b/src/java/org/apache/cassandra/service/paxos/PaxosOperationLock.java similarity index 69% rename from test/simulator/main/org/apache/cassandra/simulator/debug/Capture.java rename to src/java/org/apache/cassandra/service/paxos/PaxosOperationLock.java index 605866f784..b9f01e8bc0 100644 --- a/test/simulator/main/org/apache/cassandra/simulator/debug/Capture.java +++ b/src/java/org/apache/cassandra/service/paxos/PaxosOperationLock.java @@ -16,18 +16,15 @@ * limitations under the License. */ -package org.apache.cassandra.simulator.debug; +package org.apache.cassandra.service.paxos; -public class Capture +public interface PaxosOperationLock extends AutoCloseable { - public final boolean waitSites; - public final boolean wakeSites; - public final boolean nowSites; + @Override + public void close(); - public Capture(boolean waitSites, boolean wakeSites, boolean nowSites) + static PaxosOperationLock noOp() { - this.waitSites = waitSites; - this.wakeSites = wakeSites; - this.nowSites = nowSites; + return () -> {}; } } diff --git a/src/java/org/apache/cassandra/service/paxos/PaxosPrepare.java b/src/java/org/apache/cassandra/service/paxos/PaxosPrepare.java new file mode 100644 index 0000000000..373cee4c28 --- /dev/null +++ b/src/java/org/apache/cassandra/service/paxos/PaxosPrepare.java @@ -0,0 +1,1253 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.paxos; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.function.BiFunction; +import java.util.function.Consumer; +import java.util.stream.Collectors; +import javax.annotation.Nullable; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.concurrent.Stage; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.db.*; +import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator; +import org.apache.cassandra.exceptions.RequestFailureReason; +import org.apache.cassandra.exceptions.UnavailableException; +import org.apache.cassandra.gms.EndpointState; +import org.apache.cassandra.gms.Gossiper; +import org.apache.cassandra.io.IVersionedSerializer; +import org.apache.cassandra.io.util.DataInputPlus; +import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.metrics.PaxosMetrics; +import org.apache.cassandra.net.IVerbHandler; +import org.apache.cassandra.net.Message; +import org.apache.cassandra.net.MessagingService; +import org.apache.cassandra.schema.Schema; +import org.apache.cassandra.schema.TableId; +import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.service.PendingRangeCalculatorService; +import org.apache.cassandra.service.paxos.PaxosPrepare.Status.Outcome; +import org.apache.cassandra.tracing.Tracing; +import org.apache.cassandra.utils.vint.VIntCoding; + +import static java.util.Collections.emptyMap; +import static org.apache.cassandra.exceptions.RequestFailureReason.UNKNOWN; +import static org.apache.cassandra.locator.InetAddressAndPort.Serializer.inetAddressAndPortSerializer; +import static org.apache.cassandra.net.Verb.PAXOS2_PREPARE_REQ; +import static org.apache.cassandra.net.Verb.PAXOS2_PREPARE_RSP; +import static org.apache.cassandra.service.paxos.Ballot.Flag.NONE; +import static org.apache.cassandra.service.paxos.Commit.*; +import static org.apache.cassandra.service.paxos.Commit.CompareResult.WAS_REPROPOSED_BY; +import static org.apache.cassandra.service.paxos.Paxos.*; +import static org.apache.cassandra.service.paxos.PaxosPrepare.Status.Outcome.*; +import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis; +import static org.apache.cassandra.utils.Clock.Global.nanoTime; +import static org.apache.cassandra.service.paxos.PaxosState.*; +import static org.apache.cassandra.service.paxos.PaxosState.MaybePromise.Outcome.*; +import static org.apache.cassandra.utils.CollectionSerializer.deserializeMap; +import static org.apache.cassandra.utils.CollectionSerializer.newHashMap; +import static org.apache.cassandra.utils.CollectionSerializer.serializeMap; +import static org.apache.cassandra.utils.CollectionSerializer.serializedSizeMap; +import static org.apache.cassandra.utils.concurrent.Awaitable.SyncAwaitable.waitUntil; + +/** + * Perform one paxos "prepare" attempt, with various optimisations. + * + * The prepare step entails asking for a quorum of nodes to promise to accept our later proposal. It can + * yield one of five logical answers: + * + * 1) Success - we have received a quorum of promises, and we know that a quorum of nodes + * witnessed the prior round's commit (if any) + * 2) Timeout - we have not received enough responses at all before our deadline passed + * 3) Failure - we have received too many explicit failures to succeed + * 4) Superseded - we have been informed of a later ballot that has been promised + * 5) FoundInProgress - we have been informed of an earlier promise that has been accepted + * + * Success hinges on two distinct criteria being met, as the quorum of promises may not guarantee a quorum of + * witnesses of the prior round's commit. We track this separately by recording those nodes that have witnessed + * the prior round's commit. On receiving a quorum of promises, we submit the prior round's commit to any promisers + * that had not witnessed it, while continuing to wait for responses to our original request: as soon as we hear of + * a quorum that have witnessed it, either by our refresh request or by responses to the original request, we yield Success. + * + * Success is also accompanied by a quorum of read responses, avoiding another round-trip to obtain this result. + * + * This operation may be started either with a solo Prepare command, or with a prefixed Commit command. + * If we are completing an in-progress round we previously discovered, we save another round-trip by committing and + * preparing simultaneously. + */ +public class PaxosPrepare extends PaxosRequestCallback implements PaxosPrepareRefresh.Callbacks, Paxos.Async +{ + private static final Logger logger = LoggerFactory.getLogger(PaxosPrepare.class); + + private static Runnable onLinearizabilityViolation; + + public static final RequestHandler requestHandler = new RequestHandler(); + public static final RequestSerializer requestSerializer = new RequestSerializer(); + public static final ResponseSerializer responseSerializer = new ResponseSerializer(); + + /** + * Represents the current status of a prepare action: it is a status rather than a result, + * as the result may be unknown without sufficient responses (though in most cases it is final status). + */ + static class Status + { + enum Outcome { READ_PERMITTED, PROMISED, SUPERSEDED, FOUND_INCOMPLETE_ACCEPTED, FOUND_INCOMPLETE_COMMITTED, MAYBE_FAILURE, ELECTORATE_MISMATCH } + + final Outcome outcome; + final Participants participants; + + Status(Outcome outcome, Participants participants) + { + this.outcome = outcome; + this.participants = participants; + } + @Nullable + Ballot retryWithAtLeast() + { + switch (outcome) + { + case READ_PERMITTED: return ((Success) this).supersededBy; + case SUPERSEDED: return ((Superseded) this).by; + default: return null; + } + } + Success success() { return (Success) this; } + FoundIncompleteAccepted incompleteAccepted() { return (FoundIncompleteAccepted) this; } + FoundIncompleteCommitted incompleteCommitted() { return (FoundIncompleteCommitted) this; } + Paxos.MaybeFailure maybeFailure() { return ((MaybeFailure) this).info; } + } + + static class Success extends WithRequestedBallot + { + final List> responses; + final boolean isReadSafe; // read responses constitute a linearizable read (though short read protection would invalidate that) + final @Nullable + Ballot supersededBy; // if known and READ_SUCCESS + + Success(Outcome outcome, Ballot ballot, Participants participants, List> responses, boolean isReadSafe, @Nullable Ballot supersededBy) + { + super(outcome, participants, ballot); + this.responses = responses; + this.isReadSafe = isReadSafe; + this.supersededBy = supersededBy; + } + + static Success read(Ballot ballot, Participants participants, List> responses, @Nullable Ballot supersededBy) + { + return new Success(Outcome.READ_PERMITTED, ballot, participants, responses, true, supersededBy); + } + + static Success readOrWrite(Ballot ballot, Participants participants, List> responses, boolean isReadConsistent) + { + return new Success(Outcome.PROMISED, ballot, participants, responses, isReadConsistent, null); + } + + public String toString() { return "Success(" + ballot + ", " + participants.electorate + ')'; } + } + + /** + * The ballot we sought promises for has been superseded by another proposer's + * + * Note: we extend this for Success, so that supersededBy() can be called for ReadSuccess + */ + static class Superseded extends Status + { + final Ballot by; + + Superseded(Ballot by, Participants participants) + { + super(SUPERSEDED, participants); + this.by = by; + } + + public String toString() { return "Superseded(" + by + ')'; } + } + + static class WithRequestedBallot extends Status + { + final Ballot ballot; + + WithRequestedBallot(Outcome outcome, Participants participants, Ballot ballot) + { + super(outcome, participants); + this.ballot = ballot; + } + } + + static class FoundIncomplete extends WithRequestedBallot + { + private FoundIncomplete(Outcome outcome, Participants participants, Ballot promisedBallot) + { + super(outcome, participants, promisedBallot); + } + } + + /** + * We have been informed of a promise made by one of the replicas we contacted, that was not accepted by all replicas + * (though may have been accepted by a majority; we don't know). + * In this case we cannot readily know if we have prevented this proposal from being completed, so we attempt + * to finish it ourselves (unfortunately leaving the proposer to timeout, given the current semantics) + * TODO: we should consider waiting for more responses in case we encounter any successful commit, or a majority + * of acceptors? + */ + static class FoundIncompleteAccepted extends FoundIncomplete + { + final Accepted accepted; + + private FoundIncompleteAccepted(Ballot promisedBallot, Participants participants, Accepted accepted) + { + super(FOUND_INCOMPLETE_ACCEPTED, participants, promisedBallot); + this.accepted = accepted; + } + + public String toString() + { + return "FoundIncomplete" + accepted; + } + } + + /** + * We have been informed of a proposal that was accepted by a majority, but we do not know has been + * committed to a majority, and we failed to read from a single natural replica that had witnessed this + * commit when we performed the read. + * Since this is an edge case, we simply start again, to keep the control flow more easily understood; + * the commit shouldld be committed to a majority as part of our re-prepare. + */ + static class FoundIncompleteCommitted extends FoundIncomplete + { + final Committed committed; + + private FoundIncompleteCommitted(Ballot promisedBallot, Participants participants, Committed committed) + { + super(FOUND_INCOMPLETE_COMMITTED, participants, promisedBallot); + this.committed = committed; + } + + public String toString() + { + return "FoundIncomplete" + committed; + } + } + + static class MaybeFailure extends Status + { + final Paxos.MaybeFailure info; + private MaybeFailure(Paxos.MaybeFailure info, Participants participants) + { + super(MAYBE_FAILURE, participants); + this.info = info; + } + + public String toString() { return info.toString(); } + } + + static class ElectorateMismatch extends WithRequestedBallot + { + private ElectorateMismatch(Participants participants, Ballot ballot) + { + super(ELECTORATE_MISMATCH, participants, ballot); + } + } + + private final boolean acceptEarlyReadPermission; + private final AbstractRequest request; + private Ballot supersededBy; // cannot be promised, as a newer promise has been made + private Accepted latestAccepted; // the latest latestAcceptedButNotCommitted response we have received (which may still have been committed elsewhere) + private Committed latestCommitted; // latest actually committed proposal + + private final Participants participants; + + private final List> readResponses; + private boolean haveReadResponseWithLatest; + private boolean haveQuorumOfPermissions; // permissions => SUCCESS or READ_SUCCESS + private List withLatest; // promised and have latest commit + private List needLatest; // promised without having witnessed latest commit, nor yet been refreshed by us + private int failures; // failed either on initial request or on refresh + private boolean hasProposalStability = true; // no successful modifying proposal could have raced with us and not been seen + private boolean hasOnlyPromises = true; + private long maxLowBound; + + private Status outcome; + private final Consumer onDone; + + private PaxosPrepareRefresh refreshStaleParticipants; + private boolean linearizabilityViolationDetected = false; + + PaxosPrepare(Participants participants, AbstractRequest request, boolean acceptEarlyReadPermission, Consumer onDone) + { + this.acceptEarlyReadPermission = acceptEarlyReadPermission; + assert participants.sizeOfConsensusQuorum > 0; + this.participants = participants; + this.request = request; + this.readResponses = new ArrayList<>(participants.sizeOfConsensusQuorum); + this.withLatest = new ArrayList<>(participants.sizeOfConsensusQuorum); + this.latestAccepted = this.latestCommitted = Committed.none(request.partitionKey, request.table); + this.onDone = onDone; + } + + private boolean hasInProgressProposal() + { + // no need to commit a no-op; either it + // 1) reached a majority, in which case it was agreed, had no effect and we can do nothing; or + // 2) did not reach a majority, was not agreed, and was not user visible as a result so we can ignore it + if (latestAccepted.update.isEmpty()) + return false; + + // If we aren't newer than latestCommitted, then we're done + if (!latestAccepted.isAfter(latestCommitted)) + return false; + + if (latestAccepted.ballot.uuidTimestamp() <= maxLowBound) + return false; + + // We can be a re-proposal of latestCommitted, in which case we do not need to re-propose it + return !latestAccepted.isReproposalOf(latestCommitted); + } + + static PaxosPrepare prepare(Participants participants, SinglePartitionReadCommand readCommand, boolean isWrite, boolean acceptEarlyReadPermission) throws UnavailableException + { + return prepare(null, participants, readCommand, isWrite, acceptEarlyReadPermission); + } + + static PaxosPrepare prepare(Ballot minimumBallot, Participants participants, SinglePartitionReadCommand readCommand, boolean isWrite, boolean acceptEarlyReadPermission) throws UnavailableException + { + return prepareWithBallot(newBallot(minimumBallot, participants.consistencyForConsensus), participants, readCommand, isWrite, acceptEarlyReadPermission); + } + + static PaxosPrepare prepareWithBallot(Ballot ballot, Participants participants, SinglePartitionReadCommand readCommand, boolean isWrite, boolean acceptEarlyReadPermission) + { + Tracing.trace("Preparing {} with read", ballot); + Request request = new Request(ballot, participants.electorate, readCommand, isWrite); + return prepareWithBallotInternal(participants, request, acceptEarlyReadPermission, null); + } + + @SuppressWarnings("SameParameterValue") + static > T prepareWithBallot(Ballot ballot, Participants participants, DecoratedKey partitionKey, TableMetadata table, boolean isWrite, boolean acceptEarlyReadPermission, T onDone) + { + Tracing.trace("Preparing {}", ballot); + prepareWithBallotInternal(participants, new Request(ballot, participants.electorate, partitionKey, table, isWrite), acceptEarlyReadPermission, onDone); + return onDone; + } + + private static PaxosPrepare prepareWithBallotInternal(Participants participants, Request request, boolean acceptEarlyReadPermission, Consumer onDone) + { + PaxosPrepare prepare = new PaxosPrepare(participants, request, acceptEarlyReadPermission, onDone); + Message message = Message.out(PAXOS2_PREPARE_REQ, request); + start(prepare, participants, message, RequestHandler::execute); + return prepare; + } + + /** + * Submit the message to our peers, and submit it for local execution if relevant + */ + static > void start(PaxosPrepare prepare, Participants participants, Message send, BiFunction selfHandler) + { + boolean executeOnSelf = false; + for (int i = 0, size = participants.sizeOfPoll() ; i < size ; ++i) + { + InetAddressAndPort destination = participants.voter(i); + boolean isPending = participants.electorate.isPending(destination); + logger.trace("{} to {}", send.payload, destination); + if (shouldExecuteOnSelf(destination)) + executeOnSelf = true; + else + MessagingService.instance().sendWithCallback(isPending ? withoutRead(send) : send, destination, prepare); + } + + if (executeOnSelf) + send.verb().stage.execute(() -> prepare.executeOnSelf(send.payload, selfHandler)); + } + + // TODO: extend Sync? + public synchronized Status awaitUntil(long deadline) + { + try + { + //noinspection StatementWithEmptyBody + while (!isDone() && waitUntil(this, deadline)) {} + + if (!isDone()) + signalDone(MAYBE_FAILURE); + + return outcome; + } + catch (InterruptedException e) + { + // can only normally be interrupted if the system is shutting down; should rethrow as a write failure but propagate the interrupt + Thread.currentThread().interrupt(); + return new MaybeFailure(new Paxos.MaybeFailure(true, participants.sizeOfPoll(), participants.sizeOfConsensusQuorum, 0, emptyMap()), participants); + } + } + + private boolean isDone() + { + return outcome != null; + } + + private int withLatest() + { + return withLatest.size(); + } + + private int needLatest() + { + return needLatest == null ? 0 : needLatest.size(); + } + + private static boolean needsGossipUpdate(Map gossipInfo) + { + if (gossipInfo.isEmpty()) + return false; + + for (Map.Entry entry : gossipInfo.entrySet()) + { + EndpointState remote = entry.getValue(); + if (remote == null) + continue; + EndpointState local = Gossiper.instance.getEndpointStateForEndpoint(entry.getKey()); + if (local == null || local.isSupersededBy(remote)) + return true; + } + + return false; + } + + public synchronized void onResponse(Response response, InetAddressAndPort from) + { + if (logger.isTraceEnabled()) + logger.trace("{} for {} from {}", response, request.ballot, from); + + if (isDone()) + { + maybeCheckForLinearizabilityViolation(response, from); + return; + } + + if (response.isRejected()) + { + Rejected rejected = response.rejected(); + supersededBy = rejected.supersededBy; + signalDone(SUPERSEDED); + return; + } + + Permitted permitted = response.permitted(); + if (permitted.gossipInfo.isEmpty()) + // we agree about the electorate, so can simply accept the promise/permission + permitted(permitted, from); + else if (!needsGossipUpdate(permitted.gossipInfo)) + // our gossip is up-to-date, but our original electorate could have been built with stale gossip, so verify it + permittedOrTerminateIfElectorateMismatch(permitted, from); + else + // otherwise our beliefs about the ring potentially diverge, so update gossip with the peer's information + Stage.GOSSIP.executor().execute(() -> { + Gossiper.instance.notifyFailureDetector(permitted.gossipInfo); + Gossiper.instance.applyStateLocally(permitted.gossipInfo); + + // TODO: We should also wait for schema pulls/pushes, however this would be quite an involved change to MigrationManager + // (which currently drops some migration tasks on the floor). + // Note it would be fine for us to fail to complete the migration task and simply treat this response as a failure/timeout. + + // once any pending ranges have been calculated, refresh our Participants list and submit the promise + PendingRangeCalculatorService.instance.executeWhenFinished(() -> permittedOrTerminateIfElectorateMismatch(permitted, from)); + }); + } + + private synchronized void permittedOrTerminateIfElectorateMismatch(Permitted permitted, InetAddressAndPort from) + { + if (isDone()) // this execution is asynchronous wrt promise arrival, so must recheck done status + return; + + // if the electorate has changed, finish so we can retry with the updated view of the ring + if (!Electorate.get(request.table, request.partitionKey, consistency(request.ballot)).equals(participants.electorate)) + { + signalDone(ELECTORATE_MISMATCH); + return; + } + + // otherwise continue as normal + permitted(permitted, from); + } + + private void permitted(Permitted permitted, InetAddressAndPort from) + { + if (permitted.outcome != PROMISE) + { + hasOnlyPromises = false; + if (supersededBy == null) + supersededBy = permitted.supersededBy; + } + + if (permitted.lowBound > maxLowBound) + maxLowBound = permitted.lowBound; + + if (!haveQuorumOfPermissions) + { + CompareResult compareLatest = permitted.latestCommitted.compareWith(latestCommitted); + switch (compareLatest) + { + default: throw new IllegalStateException(); + case IS_REPROPOSAL: + latestCommitted = permitted.latestCommitted; + case WAS_REPROPOSED_BY: + case SAME: + withLatest.add(from); + haveReadResponseWithLatest |= permitted.readResponse != null; + break; + case BEFORE: + if (needLatest == null) + needLatest = new ArrayList<>(participants.sizeOfPoll() - withLatest.size()); + needLatest.add(from); + break; + case AFTER: + // move with->need + if (!withLatest.isEmpty()) + { + if (needLatest == null) + { + needLatest = withLatest; + withLatest = new ArrayList<>(Math.min(participants.sizeOfPoll() - needLatest.size(), participants.sizeOfConsensusQuorum)); + } + else + { + needLatest.addAll(withLatest); + withLatest.clear(); + } + } + + withLatest.add(from); + haveReadResponseWithLatest = permitted.readResponse != null; + latestCommitted = permitted.latestCommitted; + } + + if (isAfter(permitted.latestAcceptedButNotCommitted, latestAccepted)) + latestAccepted = permitted.latestAcceptedButNotCommitted; + + if (permitted.readResponse != null) + { + hasProposalStability &= permitted.hadProposalStability; + addReadResponse(permitted.readResponse, from); + } + } + else + { + switch (permitted.latestCommitted.compareWith(latestCommitted)) + { + default: throw new IllegalStateException(); + case SAME: + case IS_REPROPOSAL: + case WAS_REPROPOSED_BY: + withLatest.add(from); + break; + + case AFTER: + if (maybeCheckForLinearizabilityViolation(permitted, from)) + return; + // witnessing future commit doesn't imply have seen prior, so add to refresh list + + case BEFORE: + if (needLatest == null) + needLatest = new ArrayList<>(participants.sizeOfPoll() - withLatest.size()); + needLatest.add(from); + } + } + + haveQuorumOfPermissions |= withLatest() + needLatest() >= participants.sizeOfConsensusQuorum; + if (haveQuorumOfPermissions) + { + if (request.read != null && readResponses.size() < participants.sizeOfReadQuorum) + throw new IllegalStateException("Insufficient read responses: " + readResponses + "; need " + participants.sizeOfReadQuorum); + + if (!hasOnlyPromises && !hasProposalStability) + signalDone(SUPERSEDED); + + // We must be certain to have witnessed a quorum of responses before completing any in-progress proposal + // else we may complete a stale proposal that did not reach a quorum (and may do so in preference + // to a different in progress proposal that did reach a quorum). + + // We should also be sure to return any in progress proposal in preference to any incompletely committed + // earlier commits (since, while we should encounter it next round, any commit that is incomplete in the + // presence of an incomplete proposal can be ignored, as either the proposal is a re-proposal of the same + // commit or the commit has already reached a quorum + else if (hasInProgressProposal()) + signalDone(FOUND_INCOMPLETE_ACCEPTED); + + else if (withLatest() >= participants.sizeOfConsensusQuorum) + signalDone(hasOnlyPromises ? PROMISED : READ_PERMITTED); + + // otherwise if we have any read response with the latest commit, + // try to simply ensure it has been persisted to a consensus group + else if (haveReadResponseWithLatest) + { + refreshStaleParticipants(); + // if an optimistic read is possible, and we are performing a read, + // we can safely answer immediately without waiting for the refresh + if (hasProposalStability && acceptEarlyReadPermission) + signalDone(Outcome.READ_PERMITTED); + } + + // otherwise we need to run our reads again anyway, + // and the chance of receiving another response with latest may be slim. + // so we just start again + else + signalDone(FOUND_INCOMPLETE_COMMITTED); + } + } + + private boolean maybeCheckForLinearizabilityViolation(Response response, InetAddressAndPort from) + { + if (!response.isPromised() || !haveQuorumOfPermissions || !hasOnlyPromises) + return false; + + Permitted permitted = response.permitted(); + if (permitted.latestCommitted.compareWith(latestCommitted) == CompareResult.AFTER) + return checkForLinearizabilityViolation(permitted, from); + return false; + } + + private static boolean isRunningLegacyPaxos() + { + switch (getPaxosVariant()) + { + case v1: + case v1_without_linearizable_reads_or_rejected_writes: + return true; + default: + return false; + } + } + + private Ballot getLowBoundForKey() + { + ColumnFamilyStore cfs = Schema.instance.getColumnFamilyStoreInstance(request.table.id); + return cfs != null ? cfs.getPaxosRepairLowBound(request.partitionKey) : Ballot.none(); + } + + /** + * The linearizability check is incompatible with legacy paxos due to at least 2 issues: + * 1. The prepare phase doesn't evaluate accepted/committed ballots when promising ballots (excluding legacy_fixed) + * 2. Commits made at LOCAL_SERIAL are sent to all DCs + * Both issues will trigger linearizability violations, but are fixed by paxos repair. So we shouldn't do + * linearizability checks unless we're running v2 paxos and have had at least one paxos repair covering this + * operation's key. + */ + private boolean isCompatibleWithLinearizabilityCheck() + { + if (isRunningLegacyPaxos()) + return false; + + return getLowBoundForKey() != Ballot.none(); + } + + private boolean checkForLinearizabilityViolation(Permitted permitted, InetAddressAndPort from) + { + if (!isCompatibleWithLinearizabilityCheck()) + return false; + + if (linearizabilityViolationDetected) + return false; + // if we witness a newer commit AND are accepted something has gone wrong, except: + + // if we have raced with an ongoing commit, having missed all of them initially + if (permitted.latestCommitted.hasSameBallot(latestAccepted)) + return false; + + // or in the case that we have an empty proposal accepted, since that will not be committed + // in theory in this case we could now restart refreshStaleParticipants, but this would + // unnecessarily complicate the logic so instead we accept that we will unnecessarily re-propose + if (latestAccepted != null && latestAccepted.update.isEmpty() && latestAccepted.isAfter(permitted.latestCommitted)) + return false; + + // or in the case that both are older than the most recent repair low bound), in which case a topology change + // could have ocurred that means not all paxos state tables know of the accept/commit, though it is persistent + // in theory in this case we could ignore this entirely and call ourselves done + // TODO: consider this more; is it possible we cause problems by reproposing an old accept? + // shouldn't be, as any newer accept that reaches a quorum will supersede + if (permitted.latestCommitted.ballot.uuidTimestamp() <= maxLowBound) + return false; + + // if the lateset commit ballot doesn't have an encoded consistency level, it's from a legacy paxos operation. + // Legacy paxos operations would send commits to all replicas for LOCAL_SERIAL operations, which look like + // linearizability violations from datacenters the operation wasn't run in, so we ignore them here. + if (permitted.latestCommitted.ballot.flag() == NONE) + return false; + + // If we discovered an incomplete proposal, it could have since completed successfullly + if (latestAccepted != null && outcome.outcome == FOUND_INCOMPLETE_ACCEPTED) + { + switch (permitted.latestCommitted.compareWith(latestAccepted)) + { + case WAS_REPROPOSED_BY: + case SAME: + return false; + } + } + + long gcGraceMicros = TimeUnit.SECONDS.toMicros(permitted.latestCommitted.update.metadata().params.gcGraceSeconds); + // paxos repair uses stale ballots, so comparing against request.ballot time will not completely prevent false + // positives, since compaction may have removed paxos metadata on some nodes and not others. It's also possible + // clock skew has placed the ballot to repair in the future, so we use now or the ballot, whichever is higher. + long maxNowMicros = Math.max(currentTimeMillis() * 1000, request.ballot.unixMicros()); + long ageMicros = maxNowMicros - permitted.latestCommitted.ballot.unixMicros(); + + String modifier = ""; + boolean isTtlViolation; + if (isTtlViolation = (ageMicros >= gcGraceMicros)) + { + if (participants.hasOldParticipants()) + modifier = " (older than legacy TTL expiry with at least one legacy participant)"; + else + modifier = " (older than legacy TTL expiry)"; + } + String message = String.format("Linearizability violation%s: %s witnessed %s of latest %s (withLatest: %s, readResponses: %s, maxLowBound: %s, status: %s); %s promised with latest %s", + modifier, request.ballot, consistency(request.ballot), latestCommitted, + withLatest, readResponses + .stream() + .map(Message::from) + .map(Object::toString) + .collect(Collectors.joining(", ", "[", "]")), + maxLowBound, outcome, from, permitted.latestCommitted); + + PaxosMetrics.linearizabilityViolations.inc(); + linearizabilityViolationDetected = true; + + try + { + switch (DatabaseDescriptor.paxosOnLinearizabilityViolations()) + { + default: throw new AssertionError(); + case fail: + signalDone(new MaybeFailure(new Paxos.MaybeFailure(true, "A linearizability violation was detected", participants.sizeOfPoll(), participants.sizeOfConsensusQuorum, withLatest() + needLatest(), Collections.emptyMap()), participants)); + return true; + case log: + if (isTtlViolation && LOG_TTL_LINEARIZABILITY_VIOLATIONS) logger.warn(message); + else logger.error(message); + return false; + case ignore: + return false; + } + } + finally + { + Runnable run = onLinearizabilityViolation; + if (run != null) + run.run(); + } + } + + /** + * Save a read response from a node that we know to have witnessed the most recent commit + * + * Must be invoked while owning lock + */ + private void addReadResponse(ReadResponse response, InetAddressAndPort from) + { + readResponses.add(Message.synthetic(from, PAXOS2_PREPARE_RSP, response)); + } + + @Override + public synchronized void onFailure(InetAddressAndPort from, RequestFailureReason reason) + { + if (logger.isTraceEnabled()) + logger.trace("{} {} failure from {}", request, reason, from); + + if (isDone()) + return; + + super.onFailureWithMutex(from, reason); + ++failures; + + if (failures + participants.sizeOfConsensusQuorum == 1 + participants.sizeOfPoll()) + signalDone(MAYBE_FAILURE); + } + + private void signalDone(Outcome kindOfOutcome) + { + signalDone(toStatus(kindOfOutcome)); + } + + private void signalDone(Status status) + { + if (isDone()) + throw new IllegalStateException(); + + this.outcome = status; + if (onDone != null) + onDone.accept(outcome); + notifyAll(); + } + + private Status toStatus(Outcome outcome) + { + switch (outcome) + { + case ELECTORATE_MISMATCH: + return new ElectorateMismatch(participants, request.ballot); + case SUPERSEDED: + return new Superseded(supersededBy, participants); + case FOUND_INCOMPLETE_ACCEPTED: + return new FoundIncompleteAccepted(request.ballot, participants, latestAccepted); + case FOUND_INCOMPLETE_COMMITTED: + return new FoundIncompleteCommitted(request.ballot, participants, latestCommitted); + case PROMISED: + return Success.readOrWrite(request.ballot, participants, readResponses, hasProposalStability); + case READ_PERMITTED: + if (!hasProposalStability) + throw new IllegalStateException(); + return Success.read(request.ballot, participants, readResponses, supersededBy); + case MAYBE_FAILURE: + return new MaybeFailure(new Paxos.MaybeFailure(participants, withLatest(), failureReasonsAsMap()), participants); + default: + throw new IllegalStateException(); + } + } + + /** + * See {@link PaxosPrepareRefresh} + * + * Must be invoked while owning lock + */ + private void refreshStaleParticipants() + { + if (refreshStaleParticipants == null) + refreshStaleParticipants = new PaxosPrepareRefresh(request.ballot, participants, latestCommitted, this); + + refreshStaleParticipants.refresh(needLatest); + needLatest.clear(); + } + + @Override + public void onRefreshFailure(InetAddressAndPort from, RequestFailureReason reason) + { + onFailure(from, reason); + } + + public synchronized void onRefreshSuccess(Ballot isSupersededBy, InetAddressAndPort from) + { + if (logger.isTraceEnabled()) + logger.trace("Refresh {} from {}", isSupersededBy == null ? "Success" : "SupersededBy(" + isSupersededBy + ')', from); + + if (isDone()) + return; + + if (isSupersededBy != null) + { + supersededBy = isSupersededBy; + if (hasProposalStability) signalDone(Outcome.READ_PERMITTED); + else signalDone(SUPERSEDED); + } + else + { + withLatest.add(from); + if (withLatest.size() >= participants.sizeOfConsensusQuorum) + signalDone(hasOnlyPromises ? Outcome.PROMISED : Outcome.READ_PERMITTED); + } + } + + static abstract class AbstractRequest> + { + final Ballot ballot; + final Electorate electorate; + final SinglePartitionReadCommand read; + final boolean isForWrite; + final DecoratedKey partitionKey; + final TableMetadata table; + + AbstractRequest(Ballot ballot, Electorate electorate, SinglePartitionReadCommand read, boolean isForWrite) + { + this.ballot = ballot; + this.electorate = electorate; + this.read = read; + this.isForWrite = isForWrite; + this.partitionKey = read.partitionKey(); + this.table = read.metadata(); + } + + AbstractRequest(Ballot ballot, Electorate electorate, DecoratedKey partitionKey, TableMetadata table, boolean isForWrite) + { + this.ballot = ballot; + this.electorate = electorate; + this.partitionKey = partitionKey; + this.table = table; + this.read = null; + this.isForWrite = isForWrite; + } + + abstract R withoutRead(); + + public String toString() + { + return "Prepare(" + ballot + ')'; + } + } + + static class Request extends AbstractRequest + { + Request(Ballot ballot, Electorate electorate, SinglePartitionReadCommand read, boolean isWrite) + { + super(ballot, electorate, read, isWrite); + } + + private Request(Ballot ballot, Electorate electorate, DecoratedKey partitionKey, TableMetadata table, boolean isWrite) + { + super(ballot, electorate, partitionKey, table, isWrite); + } + + Request withoutRead() + { + return read == null ? this : new Request(ballot, electorate, partitionKey, table, isForWrite); + } + + public String toString() + { + return "Prepare(" + ballot + ')'; + } + } + + static class Response + { + final MaybePromise.Outcome outcome; + + Response(MaybePromise.Outcome outcome) + { + this.outcome = outcome; + } + Permitted permitted() { return (Permitted) this; } + Rejected rejected() { return (Rejected) this; } + + public boolean isRejected() + { + return outcome == REJECT; + } + + public boolean isPromised() + { + return outcome == PROMISE; + } + } + + static class Permitted extends Response + { + final long lowBound; + // a proposal that has been accepted but not committed, i.e. must be null or > latestCommit + @Nullable final Accepted latestAcceptedButNotCommitted; + final Committed latestCommitted; + @Nullable final ReadResponse readResponse; + // latestAcceptedButNotCommitted and latestCommitted were the same before and after the read occurred, and no incomplete promise was witnessed + final boolean hadProposalStability; + final Map gossipInfo; + @Nullable final Ballot supersededBy; + + Permitted(MaybePromise.Outcome outcome, long lowBound, @Nullable Accepted latestAcceptedButNotCommitted, Committed latestCommitted, @Nullable ReadResponse readResponse, boolean hadProposalStability, Map gossipInfo, @Nullable Ballot supersededBy) + { + super(outcome); + this.lowBound = lowBound; + this.latestAcceptedButNotCommitted = latestAcceptedButNotCommitted; + this.latestCommitted = latestCommitted; + this.hadProposalStability = hadProposalStability; + this.readResponse = readResponse; + this.gossipInfo = gossipInfo; + this.supersededBy = supersededBy; + } + + @Override + public String toString() + { + return "Promise(" + latestAcceptedButNotCommitted + ", " + latestCommitted + ", " + hadProposalStability + ", " + gossipInfo + ')'; + } + } + + static class Rejected extends Response + { + final Ballot supersededBy; + + Rejected(Ballot supersededBy) + { + super(REJECT); + this.supersededBy = supersededBy; + } + + @Override + public String toString() + { + return "RejectPromise(supersededBy=" + supersededBy + ')'; + } + } + + public static class RequestHandler implements IVerbHandler + { + @Override + public void doVerb(Message message) + { + Response response = execute(message.payload, message.from()); + if (response == null) + MessagingService.instance().respondWithFailure(UNKNOWN, message); + else + MessagingService.instance().respond(response, message); + } + + static Response execute(AbstractRequest request, InetAddressAndPort from) + { + if (!isInRangeAndShouldProcess(from, request.partitionKey, request.table, request.read != null)) + return null; + + long start = nanoTime(); + try (PaxosState state = get(request.partitionKey, request.table)) + { + return execute(request, state); + } + finally + { + Keyspace.openAndGetStore(request.table).metric.casPrepare.addNano(nanoTime() - start); + } + } + + static Response execute(AbstractRequest request, PaxosState state) + { + MaybePromise result = state.promiseIfNewer(request.ballot, request.isForWrite); + switch (result.outcome) + { + case PROMISE: + case PERMIT_READ: + // verify electorates; if they differ, send back gossip info for superset of two participant sets + Map gossipInfo = verifyElectorate(request.electorate, Electorate.get(request.table, request.partitionKey, consistency(request.ballot))); + ReadResponse readResponse = null; + + // Check we cannot race with a proposal, i.e. that we have not made a promise that + // could be in the process of making a proposal. If a majority of nodes have made no such promise + // then either we must have witnessed it (since it must have been committed), or the proposal + // will now be rejected by our promises. + + // This is logicaly complicated a bit by reading from a subset of the consensus group when there are + // pending nodes, however electorate verification we will cause us to retry if the pending status changes + // during execution; otherwise if the most recent commit we witnessed wasn't witnessed by a read response + // we will abort and retry, and we must witness it by the above argument. + + Ballot mostRecentCommit = result.before.accepted != null + && result.before.accepted.ballot.compareTo(result.before.committed.ballot) > 0 + && result.before.accepted.update.isEmpty() + ? result.before.accepted.ballot : result.before.committed.ballot; + + boolean hasProposalStability = mostRecentCommit.equals(result.before.promisedWrite) + || mostRecentCommit.compareTo(result.before.promisedWrite) > 0; + + if (request.read != null) + { + try (ReadExecutionController executionController = request.read.executionController(); + UnfilteredPartitionIterator iterator = request.read.executeLocally(executionController)) + { + readResponse = request.read.createResponse(iterator, executionController.getRepairedDataInfo()); + } + + if (hasProposalStability) + { + Snapshot now = state.current(request.ballot); + hasProposalStability = now.promisedWrite == result.after.promisedWrite + && now.committed == result.after.committed + && now.accepted == result.after.accepted; + } + } + + Ballot supersededBy = result.outcome == PROMISE ? null : result.after.latestWitnessedOrLowBound(); + Accepted acceptedButNotCommitted = result.after.accepted; + Committed committed = result.after.committed; + + ColumnFamilyStore cfs = Schema.instance.getColumnFamilyStoreInstance(request.table.id); + long lowBound = cfs.getPaxosRepairLowBound(request.partitionKey).uuidTimestamp(); + return new Permitted(result.outcome, lowBound, acceptedButNotCommitted, committed, readResponse, hasProposalStability, gossipInfo, supersededBy); + + case REJECT: + return new Rejected(result.supersededBy()); + + default: + throw new IllegalStateException(); + } + } + } + + static abstract class AbstractRequestSerializer, T> implements IVersionedSerializer + { + abstract R construct(T param, Ballot ballot, Electorate electorate, SinglePartitionReadCommand read, boolean isWrite); + abstract R construct(T param, Ballot ballot, Electorate electorate, DecoratedKey partitionKey, TableMetadata table, boolean isWrite); + + @Override + public void serialize(R request, DataOutputPlus out, int version) throws IOException + { + request.ballot.serialize(out); + Electorate.serializer.serialize(request.electorate, out, version); + out.writeByte((request.read != null ? 1 : 0) | (request.isForWrite ? 0 : 2)); + if (request.read != null) + { + + ReadCommand.serializer.serialize(request.read, out, version); + } + else + { + request.table.id.serialize(out); + DecoratedKey.serializer.serialize(request.partitionKey, out, version); + } + } + + public R deserialize(T param, DataInputPlus in, int version) throws IOException + { + Ballot ballot = Ballot.deserialize(in); + Electorate electorate = Electorate.serializer.deserialize(in, version); + byte flag = in.readByte(); + if ((flag & 1) != 0) + { + SinglePartitionReadCommand readCommand = (SinglePartitionReadCommand) ReadCommand.serializer.deserialize(in, version); + return construct(param, ballot, electorate, readCommand, (flag & 2) == 0); + } + else + { + TableMetadata table = Schema.instance.getExistingTableMetadata(TableId.deserialize(in)); + DecoratedKey partitionKey = (DecoratedKey) DecoratedKey.serializer.deserialize(in, table.partitioner, version); + return construct(param, ballot, electorate, partitionKey, table, (flag & 2) != 0); + } + } + + @Override + public long serializedSize(R request, int version) + { + return Ballot.sizeInBytes() + + Electorate.serializer.serializedSize(request.electorate, version) + + 1 + (request.read != null + ? ReadCommand.serializer.serializedSize(request.read, version) + : request.table.id.serializedSize() + + DecoratedKey.serializer.serializedSize(request.partitionKey, version)); + } + } + + public static class RequestSerializer extends AbstractRequestSerializer + { + Request construct(Object ignore, Ballot ballot, Electorate electorate, SinglePartitionReadCommand read, boolean isWrite) + { + return new Request(ballot, electorate, read, isWrite); + } + + Request construct(Object ignore, Ballot ballot, Electorate electorate, DecoratedKey partitionKey, TableMetadata table, boolean isWrite) + { + return new Request(ballot, electorate, partitionKey, table, isWrite); + } + + public Request deserialize(DataInputPlus in, int version) throws IOException + { + return deserialize(null, in, version); + } + } + + public static class ResponseSerializer implements IVersionedSerializer + { + public void serialize(Response response, DataOutputPlus out, int version) throws IOException + { + if (response.isRejected()) + { + out.writeByte(0); + Rejected rejected = (Rejected) response; + rejected.supersededBy.serialize(out); + } + else + { + Permitted promised = (Permitted) response; + out.writeByte(1 + | (promised.latestAcceptedButNotCommitted != null ? 2 : 0) + | (promised.readResponse != null ? 4 : 0) + | (promised.hadProposalStability ? 8 : 0) + | (promised.outcome == PERMIT_READ ? 16 : 0) + ); + out.writeUnsignedVInt(promised.lowBound); + if (promised.latestAcceptedButNotCommitted != null) + Accepted.serializer.serialize(promised.latestAcceptedButNotCommitted, out, version); + Committed.serializer.serialize(promised.latestCommitted, out, version); + if (promised.readResponse != null) + ReadResponse.serializer.serialize(promised.readResponse, out, version); + serializeMap(inetAddressAndPortSerializer, EndpointState.nullableSerializer, promised.gossipInfo, out, version); + if (promised.outcome == PERMIT_READ) + promised.supersededBy.serialize(out); + } + } + + public Response deserialize(DataInputPlus in, int version) throws IOException + { + byte flags = in.readByte(); + if (flags == 0) + { + Ballot supersededBy = Ballot.deserialize(in); + return new Rejected(supersededBy); + } + else + { + long lowBound = in.readUnsignedVInt(); + Accepted acceptedNotCommitted = (flags & 2) != 0 ? Accepted.serializer.deserialize(in, version) : null; + Committed committed = Committed.serializer.deserialize(in, version); + ReadResponse readResponse = (flags & 4) != 0 ? ReadResponse.serializer.deserialize(in, version) : null; + Map gossipInfo = deserializeMap(inetAddressAndPortSerializer, EndpointState.nullableSerializer, newHashMap(), in, version); + MaybePromise.Outcome outcome = (flags & 16) != 0 ? PERMIT_READ : PROMISE; + boolean hasProposalStability = (flags & 8) != 0; + Ballot supersededBy = null; + if (outcome == PERMIT_READ) + supersededBy = Ballot.deserialize(in); + return new Permitted(outcome, lowBound, acceptedNotCommitted, committed, readResponse, hasProposalStability, gossipInfo, supersededBy); + } + } + + public long serializedSize(Response response, int version) + { + if (response.isRejected()) + { + return 1 + Ballot.sizeInBytes(); + } + else + { + Permitted permitted = (Permitted) response; + return 1 + + VIntCoding.computeUnsignedVIntSize(permitted.lowBound) + + (permitted.latestAcceptedButNotCommitted == null ? 0 : Accepted.serializer.serializedSize(permitted.latestAcceptedButNotCommitted, version)) + + Committed.serializer.serializedSize(permitted.latestCommitted, version) + + (permitted.readResponse == null ? 0 : ReadResponse.serializer.serializedSize(permitted.readResponse, version)) + + serializedSizeMap(inetAddressAndPortSerializer, EndpointState.nullableSerializer, permitted.gossipInfo, version) + + (permitted.outcome == PERMIT_READ ? Ballot.sizeInBytes() : 0); + } + } + } + + static > Message withoutRead(Message send) + { + if (send.payload.read == null) + return send; + + return send.withPayload(send.payload.withoutRead()); + } + + public static void setOnLinearizabilityViolation(Runnable runnable) + { + assert onLinearizabilityViolation == null || runnable == null; + onLinearizabilityViolation = runnable; + } +} diff --git a/src/java/org/apache/cassandra/service/paxos/PaxosPrepareRefresh.java b/src/java/org/apache/cassandra/service/paxos/PaxosPrepareRefresh.java new file mode 100644 index 0000000000..6c13104960 --- /dev/null +++ b/src/java/org/apache/cassandra/service/paxos/PaxosPrepareRefresh.java @@ -0,0 +1,245 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.paxos; + +import java.io.IOException; +import java.util.List; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.exceptions.RequestFailureReason; +import org.apache.cassandra.exceptions.WriteTimeoutException; +import org.apache.cassandra.io.IVersionedSerializer; +import org.apache.cassandra.io.util.DataInputPlus; +import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.net.IVerbHandler; +import org.apache.cassandra.net.Message; +import org.apache.cassandra.net.MessagingService; +import org.apache.cassandra.net.RequestCallback; +import org.apache.cassandra.net.RequestCallbackWithFailure; +import org.apache.cassandra.service.paxos.Commit.Agreed; +import org.apache.cassandra.service.paxos.Commit.Committed; +import org.apache.cassandra.tracing.Tracing; + +import static org.apache.cassandra.exceptions.RequestFailureReason.TIMEOUT; +import static org.apache.cassandra.exceptions.RequestFailureReason.UNKNOWN; +import static org.apache.cassandra.net.Verb.PAXOS2_PREPARE_REFRESH_REQ; +import static org.apache.cassandra.service.paxos.Commit.isAfter; +import static org.apache.cassandra.service.paxos.PaxosRequestCallback.shouldExecuteOnSelf; +import static org.apache.cassandra.utils.FBUtilities.getBroadcastAddressAndPort; +import static org.apache.cassandra.utils.NullableSerializer.deserializeNullable; +import static org.apache.cassandra.utils.NullableSerializer.serializeNullable; +import static org.apache.cassandra.utils.NullableSerializer.serializedSizeNullable; + +/** + * Nodes that have promised in response to our prepare, may be missing the latestCommit, meaning we cannot be sure the + * prior round has been committed to the necessary quorum of participants, so that it will be visible to future quorums. + * + * To resolve this problem, we submit the latest commit we have seen, and wait for confirmation before continuing + * (verifying that we are still promised in the process). + */ +public class PaxosPrepareRefresh implements RequestCallbackWithFailure +{ + private static final Logger logger = LoggerFactory.getLogger(PaxosPrepareRefresh.class); + + public static final RequestHandler requestHandler = new RequestHandler(); + public static final RequestSerializer requestSerializer = new RequestSerializer(); + public static final ResponseSerializer responseSerializer = new ResponseSerializer(); + + interface Callbacks + { + void onRefreshFailure(InetAddressAndPort from, RequestFailureReason reason); + void onRefreshSuccess(Ballot isSupersededBy, InetAddressAndPort from); + } + + private final Message send; + private final Callbacks callbacks; + + public PaxosPrepareRefresh(Ballot prepared, Paxos.Participants participants, Committed latestCommitted, Callbacks callbacks) + { + this.callbacks = callbacks; + this.send = Message.out(PAXOS2_PREPARE_REFRESH_REQ, new Request(prepared, latestCommitted)); + } + + void refresh(List refresh) + { + boolean executeOnSelf = false; + for (int i = 0, size = refresh.size(); i < size ; ++i) + { + InetAddressAndPort destination = refresh.get(i); + + if (logger.isTraceEnabled()) + logger.trace("Refresh {} and Confirm {} to {}", send.payload.missingCommit, Ballot.toString(send.payload.promised, "Promise"), destination); + + if (Tracing.isTracing()) + Tracing.trace("Refresh {} and Confirm {} to {}", send.payload.missingCommit.ballot, send.payload.promised, destination); + + if (shouldExecuteOnSelf(destination)) + executeOnSelf = true; + else + MessagingService.instance().sendWithCallback(send, destination, this); + } + + if (executeOnSelf) + PAXOS2_PREPARE_REFRESH_REQ.stage.execute(this::executeOnSelf); + } + + @Override + public void onFailure(InetAddressAndPort from, RequestFailureReason reason) + { + callbacks.onRefreshFailure(from, reason); + } + + @Override + public void onResponse(Message message) + { + onResponse(message.payload, message.from()); + } + + private void executeOnSelf() + { + Response response; + try + { + response = RequestHandler.execute(send.payload, getBroadcastAddressAndPort()); + if (response == null) + return; + } + catch (Exception ex) + { + RequestFailureReason reason = UNKNOWN; + if (ex instanceof WriteTimeoutException) reason = TIMEOUT; + else logger.error("Failed to apply paxos refresh-prepare locally", ex); + + onFailure(getBroadcastAddressAndPort(), reason); + return; + } + onResponse(response, getBroadcastAddressAndPort()); + } + + private void onResponse(Response response, InetAddressAndPort from) + { + callbacks.onRefreshSuccess(response.isSupersededBy, from); + } + + private static class Request + { + final Ballot promised; + final Committed missingCommit; + + Request(Ballot promised, Committed missingCommit) + { + this.promised = promised; + this.missingCommit = missingCommit; + } + } + + static class Response + { + final Ballot isSupersededBy; + Response(Ballot isSupersededBy) + { + this.isSupersededBy = isSupersededBy; + } + } + + public static class RequestHandler implements IVerbHandler + { + @Override + public void doVerb(Message message) + { + Response response = execute(message.payload, message.from()); + if (response == null) + MessagingService.instance().respondWithFailure(UNKNOWN, message); + else + MessagingService.instance().respond(response, message); + } + + public static Response execute(Request request, InetAddressAndPort from) + { + Agreed commit = request.missingCommit; + + if (!Paxos.isInRangeAndShouldProcess(from, commit.update.partitionKey(), commit.update.metadata(), false)) + return null; + + try (PaxosState state = PaxosState.get(commit)) + { + state.commit(commit); + Ballot latest = state.current(request.promised).latestWitnessedOrLowBound(); + if (isAfter(latest, request.promised)) + { + Tracing.trace("Promise {} rescinded; latest is now {}", request.promised, latest); + return new Response(latest); + } + else + { + Tracing.trace("Promise confirmed for ballot {}", request.promised); + return new Response(null); + } + } + } + } + + public static class RequestSerializer implements IVersionedSerializer + { + @Override + public void serialize(Request request, DataOutputPlus out, int version) throws IOException + { + request.promised.serialize(out); + Committed.serializer.serialize(request.missingCommit, out, version); + } + + @Override + public Request deserialize(DataInputPlus in, int version) throws IOException + { + Ballot promise = Ballot.deserialize(in); + Committed missingCommit = Committed.serializer.deserialize(in, version); + return new Request(promise, missingCommit); + } + + @Override + public long serializedSize(Request request, int version) + { + return Ballot.sizeInBytes() + + Committed.serializer.serializedSize(request.missingCommit, version); + } + } + + public static class ResponseSerializer implements IVersionedSerializer + { + public void serialize(Response response, DataOutputPlus out, int version) throws IOException + { + serializeNullable(Ballot.Serializer.instance, response.isSupersededBy, out, version); + } + + public Response deserialize(DataInputPlus in, int version) throws IOException + { + Ballot isSupersededBy = deserializeNullable(Ballot.Serializer.instance, in, version); + return new Response(isSupersededBy); + } + + public long serializedSize(Response response, int version) + { + return serializedSizeNullable(Ballot.Serializer.instance, response.isSupersededBy, version); + } + } + +} diff --git a/src/java/org/apache/cassandra/service/paxos/PaxosPropose.java b/src/java/org/apache/cassandra/service/paxos/PaxosPropose.java new file mode 100644 index 0000000000..57d3459f40 --- /dev/null +++ b/src/java/org/apache/cassandra/service/paxos/PaxosPropose.java @@ -0,0 +1,479 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.paxos; + +import java.io.IOException; +import java.util.concurrent.atomic.AtomicLongFieldUpdater; +import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; +import java.util.function.Consumer; + +import com.google.common.annotations.VisibleForTesting; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.db.Keyspace; +import org.apache.cassandra.db.TypeSizes; +import org.apache.cassandra.exceptions.RequestFailureReason; +import org.apache.cassandra.io.IVersionedSerializer; +import org.apache.cassandra.io.util.DataInputPlus; +import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.net.IVerbHandler; +import org.apache.cassandra.net.Message; +import org.apache.cassandra.net.MessagingService; +import org.apache.cassandra.service.paxos.Commit.Proposal; +import org.apache.cassandra.utils.concurrent.ConditionAsConsumer; + +import static java.util.Collections.emptyMap; +import static org.apache.cassandra.exceptions.RequestFailureReason.UNKNOWN; +import static org.apache.cassandra.net.Verb.PAXOS2_PROPOSE_REQ; +import static org.apache.cassandra.service.paxos.PaxosPropose.Superseded.SideEffects.NO; +import static org.apache.cassandra.service.paxos.PaxosPropose.Superseded.SideEffects.MAYBE; +import static org.apache.cassandra.utils.Clock.Global.nanoTime; +import static org.apache.cassandra.utils.concurrent.ConditionAsConsumer.newConditionAsConsumer; + +/** + * In waitForNoSideEffect mode, we will not return failure to the caller until + * we have received a complete set of refusal responses, or at least one accept, + * indicating (respectively) that we have had no side effect, or that we cannot + * know if we our proposal produced a side effect. + */ +public class PaxosPropose> extends PaxosRequestCallback +{ + private static final Logger logger = LoggerFactory.getLogger(PaxosPropose.class); + + public static final RequestHandler requestHandler = new RequestHandler(); + public static final RequestSerializer requestSerializer = new RequestSerializer(); + public static final ResponseSerializer responseSerializer = new ResponseSerializer(); + + /** + * Represents the current status of a propose action: it is a status rather than a result, + * as the result may be unknown without sufficient responses (though in most cases it is final status). + */ + static class Status + { + enum Outcome { SUCCESS, SUPERSEDED, MAYBE_FAILURE } + final Outcome outcome; + + Status(Outcome outcome) + { + this.outcome = outcome; + } + Superseded superseded() { return (Superseded) this; } + Paxos.MaybeFailure maybeFailure() { return ((MaybeFailure) this).info; } + public String toString() { return "Success"; } + } + + static class Superseded extends Status + { + enum SideEffects { NO, MAYBE } + final Ballot by; + final SideEffects hadSideEffects; + Superseded(Ballot by, SideEffects hadSideEffects) + { + super(Outcome.SUPERSEDED); + this.by = by; + this.hadSideEffects = hadSideEffects; + } + + public String toString() { return "Superseded(" + by + ',' + hadSideEffects + ')'; } + } + + private static class MaybeFailure extends Status + { + final Paxos.MaybeFailure info; + MaybeFailure(Paxos.MaybeFailure info) + { + super(Outcome.MAYBE_FAILURE); + this.info = info; + } + + public String toString() { return info.toString(); } + } + + private static final Status success = new Status(Status.Outcome.SUCCESS); + + private static final AtomicLongFieldUpdater responsesUpdater = AtomicLongFieldUpdater.newUpdater(PaxosPropose.class, "responses"); + private static final AtomicReferenceFieldUpdater supersededByUpdater = AtomicReferenceFieldUpdater.newUpdater(PaxosPropose.class, Ballot.class, "supersededBy"); + + @VisibleForTesting public static final long ACCEPT_INCREMENT = 1; + private static final int REFUSAL_SHIFT = 21; + @VisibleForTesting public static final long REFUSAL_INCREMENT = 1L << REFUSAL_SHIFT; + private static final int FAILURE_SHIFT = 42; + @VisibleForTesting public static final long FAILURE_INCREMENT = 1L << FAILURE_SHIFT; + private static final long MASK = (1L << REFUSAL_SHIFT) - 1L; + + private final Proposal proposal; + /** Wait until we know if we may have had side effects */ + private final boolean waitForNoSideEffect; + /** Number of contacted nodes */ + final int participants; + /** Number of accepts required */ + final int required; + /** Invoke on reaching a terminal status */ + final OnDone onDone; + + /** + * bit 0-20: accepts + * bit 21-41: refusals/errors + * bit 42-62: timeouts + * bit 63: ambiguous signal bit (i.e. those states that cannot be certain to signal uniquely flip this bit to take signal responsibility) + * + * {@link #accepts} + * {@link #refusals} + * {@link #failures} + * {@link #notAccepts} (timeouts/errors+refusals) + */ + private volatile long responses; + + /** The newest superseding ballot from a refusal; only returned to the caller if we fail to reach a quorum */ + private volatile Ballot supersededBy; + + private PaxosPropose(Proposal proposal, int participants, int required, boolean waitForNoSideEffect, OnDone onDone) + { + this.proposal = proposal; + assert required > 0; + this.waitForNoSideEffect = waitForNoSideEffect; + this.participants = participants; + this.required = required; + this.onDone = onDone; + } + + /** + * Submit the proposal for commit with all replicas, and return an object that can be waited on synchronously for the result, + * or for the present status if the time elapses without a final result being reached. + * @param waitForNoSideEffect if true, on failure we will wait until we can say with certainty there are no side effects + * or until we know we will never be able to determine this with certainty + */ + static Paxos.Async propose(Proposal proposal, Paxos.Participants participants, boolean waitForNoSideEffect) + { + if (waitForNoSideEffect && proposal.update.isEmpty()) + waitForNoSideEffect = false; // by definition this has no "side effects" (besides linearizing the operation) + + // to avoid unnecessary object allocations we extend PaxosPropose to implements Paxos.Async + class Async extends PaxosPropose> implements Paxos.Async + { + private Async(Proposal proposal, int participants, int required, boolean waitForNoSideEffect) + { + super(proposal, participants, required, waitForNoSideEffect, newConditionAsConsumer()); + } + + public Status awaitUntil(long deadline) + { + try + { + onDone.awaitUntil(deadline); + } + catch (InterruptedException e) + { + Thread.currentThread().interrupt(); + return new MaybeFailure(new Paxos.MaybeFailure(true, participants, required, 0, emptyMap())); + } + + return status(); + } + } + + Async propose = new Async(proposal, participants.sizeOfPoll(), participants.sizeOfConsensusQuorum, waitForNoSideEffect); + propose.start(participants); + return propose; + } + + static > T propose(Proposal proposal, Paxos.Participants participants, boolean waitForNoSideEffect, T onDone) + { + if (waitForNoSideEffect && proposal.update.isEmpty()) + waitForNoSideEffect = false; // by definition this has no "side effects" (besides linearizing the operation) + + PaxosPropose propose = new PaxosPropose<>(proposal, participants.sizeOfPoll(), participants.sizeOfConsensusQuorum, waitForNoSideEffect, onDone); + propose.start(participants); + return onDone; + } + + void start(Paxos.Participants participants) + { + Message message = Message.out(PAXOS2_PROPOSE_REQ, new Request(proposal)); + + boolean executeOnSelf = false; + for (int i = 0, size = participants.sizeOfPoll(); i < size ; ++i) + { + InetAddressAndPort destination = participants.voter(i); + logger.trace("{} to {}", proposal, destination); + if (shouldExecuteOnSelf(destination)) executeOnSelf = true; + else MessagingService.instance().sendWithCallback(message, destination, this); + } + + if (executeOnSelf) + PAXOS2_PROPOSE_REQ.stage.execute(() -> executeOnSelf(proposal)); + } + + /** + * @return the result as of now; unless the result is definitive, it is only a snapshot of the present incomplete status + */ + Status status() + { + long responses = this.responses; + + if (isSuccessful(responses)) + return success; + + if (!canSucceed(responses) && supersededBy != null) + { + Superseded.SideEffects sideEffects = hasNoSideEffects(responses) ? NO : MAYBE; + return new Superseded(supersededBy, sideEffects); + } + + return new MaybeFailure(new Paxos.MaybeFailure(participants, required, accepts(responses), failureReasonsAsMap())); + } + + private void executeOnSelf(Proposal proposal) + { + executeOnSelf(proposal, RequestHandler::execute); + } + + public void onResponse(Response response, InetAddressAndPort from) + { + if (logger.isTraceEnabled()) + logger.trace("{} for {} from {}", response, proposal, from); + + Ballot supersededBy = response.supersededBy; + if (supersededBy != null) + supersededByUpdater.accumulateAndGet(this, supersededBy, (a, b) -> a == null ? b : b.uuidTimestamp() > a.uuidTimestamp() ? b : a); + + long increment = supersededBy == null + ? ACCEPT_INCREMENT + : REFUSAL_INCREMENT; + + update(increment); + } + + @Override + public void onFailure(InetAddressAndPort from, RequestFailureReason reason) + { + if (logger.isTraceEnabled()) + logger.trace("{} {} failure from {}", proposal, reason, from); + + super.onFailure(from, reason); + update(FAILURE_INCREMENT); + } + + private void update(long increment) + { + long responses = responsesUpdater.addAndGet(this, increment); + if (shouldSignal(responses)) + signalDone(); + } + + // returns true at most once for a given PaxosPropose, so we do not propagate a signal more than once + private boolean shouldSignal(long responses) + { + return shouldSignal(responses, required, participants, waitForNoSideEffect, responsesUpdater, this); + } + + @VisibleForTesting + public static boolean shouldSignal(long responses, int required, int participants, boolean waitForNoSideEffect, AtomicLongFieldUpdater responsesUpdater, T update) + { + if (responses <= 0L) // already signalled via ambiguous signal bit + return false; + + if (!isSuccessful(responses, required)) + { + if (canSucceed(responses, required, participants)) + return false; + + if (waitForNoSideEffect && !hasPossibleSideEffects(responses)) + return hasNoSideEffects(responses, participants); + } + + return responsesUpdater.getAndUpdate(update, x -> x | Long.MIN_VALUE) >= 0L; + } + + private void signalDone() + { + if (onDone != null) + onDone.accept(status()); + } + + private boolean isSuccessful(long responses) + { + return isSuccessful(responses, required); + } + + private static boolean isSuccessful(long responses, int required) + { + return accepts(responses) >= required; + } + + private boolean canSucceed(long responses) + { + return canSucceed(responses, required, participants); + } + + private static boolean canSucceed(long responses, int required, int participants) + { + return refusals(responses) == 0 && required <= participants - failures(responses); + } + + // Note: this is only reliable if !failFast + private boolean hasNoSideEffects(long responses) + { + return hasNoSideEffects(responses, participants); + } + + private static boolean hasNoSideEffects(long responses, int participants) + { + return refusals(responses) == participants; + } + + private static boolean hasPossibleSideEffects(long responses) + { + return accepts(responses) + failures(responses) > 0; + } + + /** {@link #responses} */ + private static int accepts(long responses) + { + return (int) (responses & MASK); + } + + /** {@link #responses} */ + private static int notAccepts(long responses) + { + return failures(responses) + refusals(responses); + } + + /** {@link #responses} */ + private static int refusals(long responses) + { + return (int) ((responses >>> REFUSAL_SHIFT) & MASK); + } + + /** {@link #responses} */ + private static int failures(long responses) + { + return (int) ((responses >>> FAILURE_SHIFT) & MASK); + } + + /** + * A Proposal to submit to another node + */ + static class Request + { + final Proposal proposal; + Request(Proposal proposal) + { + this.proposal = proposal; + } + + public String toString() + { + return proposal.toString("Propose"); + } + } + + /** + * The response to a proposal, indicating success (if {@code supersededBy == null}, + * or failure, alongside the ballot that beat us + */ + static class Response + { + final Ballot supersededBy; + Response(Ballot supersededBy) + { + this.supersededBy = supersededBy; + } + public String toString() { return supersededBy == null ? "Accept" : "RejectProposal(supersededBy=" + supersededBy + ')'; } + } + + /** + * The proposal request handler, i.e. receives a proposal from a peer and responds with either acccept/reject + */ + public static class RequestHandler implements IVerbHandler + { + @Override + public void doVerb(Message message) + { + Response response = execute(message.payload.proposal, message.from()); + if (response == null) + MessagingService.instance().respondWithFailure(UNKNOWN, message); + else + MessagingService.instance().respond(response, message); + } + + public static Response execute(Proposal proposal, InetAddressAndPort from) + { + if (!Paxos.isInRangeAndShouldProcess(from, proposal.update.partitionKey(), proposal.update.metadata(), false)) + return null; + + long start = nanoTime(); + try (PaxosState state = PaxosState.get(proposal)) + { + return new Response(state.acceptIfLatest(proposal)); + } + finally + { + Keyspace.openAndGetStore(proposal.update.metadata()).metric.casPropose.addNano(nanoTime() - start); + } + } + } + + public static class RequestSerializer implements IVersionedSerializer + { + @Override + public void serialize(Request request, DataOutputPlus out, int version) throws IOException + { + Proposal.serializer.serialize(request.proposal, out, version); + } + + @Override + public Request deserialize(DataInputPlus in, int version) throws IOException + { + Proposal propose = Proposal.serializer.deserialize(in, version); + return new Request(propose); + } + + @Override + public long serializedSize(Request request, int version) + { + return Proposal.serializer.serializedSize(request.proposal, version); + } + } + + public static class ResponseSerializer implements IVersionedSerializer + { + public void serialize(Response response, DataOutputPlus out, int version) throws IOException + { + out.writeBoolean(response.supersededBy != null); + if (response.supersededBy != null) + response.supersededBy.serialize(out); + } + + public Response deserialize(DataInputPlus in, int version) throws IOException + { + boolean isSuperseded = in.readBoolean(); + return isSuperseded ? new Response(Ballot.deserialize(in)) : new Response(null); + } + + public long serializedSize(Response response, int version) + { + return response.supersededBy != null + ? TypeSizes.sizeof(true) + Ballot.sizeInBytes() + : TypeSizes.sizeof(false); + } + } +} diff --git a/src/java/org/apache/cassandra/service/paxos/PaxosRepair.java b/src/java/org/apache/cassandra/service/paxos/PaxosRepair.java new file mode 100644 index 0000000000..d88323cbcf --- /dev/null +++ b/src/java/org/apache/cassandra/service/paxos/PaxosRepair.java @@ -0,0 +1,706 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.paxos; + + +import java.io.IOException; +import java.util.*; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.function.Function; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.collect.Iterables; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.concurrent.ScheduledExecutorPlus; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.db.ConsistencyLevel; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.Keyspace; +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.Token; +import org.apache.cassandra.exceptions.RequestFailureReason; +import org.apache.cassandra.exceptions.UnavailableException; +import org.apache.cassandra.gms.ApplicationState; +import org.apache.cassandra.gms.EndpointState; +import org.apache.cassandra.gms.Gossiper; +import org.apache.cassandra.gms.VersionedValue; +import org.apache.cassandra.io.IVersionedSerializer; +import org.apache.cassandra.io.util.DataInputPlus; +import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.locator.Replica; +import org.apache.cassandra.net.IVerbHandler; +import org.apache.cassandra.net.Message; +import org.apache.cassandra.net.MessagingService; +import org.apache.cassandra.net.RequestCallbackWithFailure; +import org.apache.cassandra.schema.Schema; +import org.apache.cassandra.schema.TableId; +import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.utils.CassandraVersion; +import org.apache.cassandra.utils.ExecutorUtils; +import org.apache.cassandra.utils.FBUtilities; +import org.apache.cassandra.utils.MonotonicClock; + +import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory; +import static org.apache.cassandra.config.CassandraRelevantProperties.PAXOS_REPAIR_RETRY_TIMEOUT_IN_MS; +import static org.apache.cassandra.exceptions.RequestFailureReason.UNKNOWN; +import static org.apache.cassandra.net.Verb.PAXOS2_REPAIR_REQ; +import static java.util.concurrent.TimeUnit.NANOSECONDS; +import static org.apache.cassandra.service.paxos.Commit.*; +import static org.apache.cassandra.service.paxos.ContentionStrategy.Type.REPAIR; +import static org.apache.cassandra.service.paxos.ContentionStrategy.waitUntilForContention; +import static org.apache.cassandra.service.paxos.Paxos.*; +import static org.apache.cassandra.service.paxos.PaxosPrepare.*; +import static org.apache.cassandra.utils.Clock.Global.nanoTime; +import static org.apache.cassandra.utils.NullableSerializer.deserializeNullable; +import static org.apache.cassandra.utils.NullableSerializer.serializeNullable; +import static org.apache.cassandra.utils.NullableSerializer.serializedSizeNullable; + +/** + * Facility to finish any in-progress paxos transaction, and ensure that a quorum of nodes agree on the most recent operation. + * Semantically, we simply ensure that any side effects that were "decided" before repair was initiated have been committed + * to a quorum of nodes. + * This means: + * - any prepare that has _possibly_ reached a quorum of nodes will be invalidated + * - any proposal that has been accepted by at least one node, but not known to be committed to any, will be proposed again + * - any proposal that has been committed to at least one node, but not committed to all, will be committed to a quorum + * + * Note that once started, this continues to try to repair any ongoing operations for the partition up to 4 times. + * In a functioning cluster this should always be possible, but during a network partition this might cause the repair + * to fail. + * + * Requirements for correction: + * - If performed during a range movement, we depend on a quorum (of the new topology) have been informed of the new + * topology _prior_ to initiating this repair, and this node to have been a member of a quorum of nodes verifying + * - If a quorum of nodes is unaware of the new topology prior to initiating repair, an operation could simply occur + * after repair completes that permits a linearization failure, such as with CASSANDRA-15745. + * their topology is up-to-date. + * - Paxos prepare rounds must also verify the topology being used with their peers + * - If prepare rounds do not verify their topology, a node that is not a member of the quorum who have agreed + * the latest topology could still perform an operation without being aware of the topology change, and permit a + * linearization failure, such as with CASSANDRA-15745. + * + * With these issues addressed elsewhere, our algorithm is fairly simple. + * In brief: + * 1) Query all replicas for any promises or proposals they have witnessed that have not been committed, + * and their most recent commit. Wait for a quorum of responses. + * 2) If this is the first time we have queried other nodes, we take a note of the most recent ballot we see; + * If this is not the first time we have queried other nodes, and we have committed a newer ballot than the one + * we previously recorded, we terminate (somebody else has done the work for us). + * 3) If we see an in-progress operation that is very recent, we wait for it to complete and try again + * 4) If we see a previously accepted operation, we attempt to complete it, or + * if we see a prepare with no proposal, we propose an empty update to invalidate it; + * otherwise we have nothing to do, as there is no operation that can have produced a side-effect before we began. + * 5) We prepare a paxos round to agree the new commit using a higher ballot than the one witnessed, + * but a lower than one we would propose a new operation with. This permits newer operations to "beat" us so + * that we do not interfere with normal paxos operations. + * 6) If we are "beaten" we start again (without delay, as (2) manages delays where necessary) + */ +public class PaxosRepair extends AbstractPaxosRepair +{ + private static final Logger logger = LoggerFactory.getLogger(PaxosRepair.class); + + public static final RequestSerializer requestSerializer = new RequestSerializer(); + public static final ResponseSerializer responseSerializer = new ResponseSerializer(); + public static final RequestHandler requestHandler = new RequestHandler(); + private static final long RETRY_TIMEOUT_NANOS = getRetryTimeoutNanos(); + + private static final ScheduledExecutorPlus RETRIES = executorFactory().scheduled("PaxosRepairRetries"); + + private static long getRetryTimeoutNanos() + { + long retryMillis = PAXOS_REPAIR_RETRY_TIMEOUT_IN_MS.getLong(); + return TimeUnit.MILLISECONDS.toNanos(retryMillis); + } + + private final TableMetadata table; + private final ConsistencyLevel paxosConsistency; + private Participants participants; + + private Ballot successCriteria; + private Ballot prevSupersededBy; + private int attempts; + + public String toString() + { + return "PaxosRepair{" + + "key=" + partitionKey() + + ", table=" + table.toString() + + ", consistency=" + paxosConsistency + + ", participants=" + participants.electorate + + ", state=" + state() + + ", startedMillis=" + MonotonicClock.Global.approxTime.translate().toMillisSinceEpoch(startedNanos()) + + ", started=" + isStarted() + + '}'; + } + + /** + * Waiting for responses to PAXOS_REPAIR messages. + * + * This state may be entered multiple times; every time we fail for any reason, we restart from this state + */ + private class Querying extends State implements RequestCallbackWithFailure, Runnable + { + private int successes; + private int failures; + + private Ballot latestWitnessed; + private @Nullable Accepted latestAccepted; + private Committed latestCommitted; + private Ballot oldestCommitted; + private Ballot clashingPromise; + + @Override + public void onFailure(InetAddressAndPort from, RequestFailureReason reason) + { + updateState(this, null, (i1, i2) -> i1.onFailure()); + } + + @Override + public void onResponse(Message msg) + { + logger.trace("PaxosRepair {} from {}", msg.payload, msg.from()); + updateState(this, msg, Querying::onResponseInternal); + } + + private State onFailure() + { + if (++failures + participants.sizeOfConsensusQuorum > participants.sizeOfPoll()) + return retry(this); + return this; + } + + private State onResponseInternal(Message msg) + { + latestWitnessed = latest(latestWitnessed, msg.payload.latestWitnessedOrLowBound); + latestAccepted = latest(latestAccepted, msg.payload.acceptedButNotCommitted); + latestCommitted = latest(latestCommitted, msg.payload.committed); + if (oldestCommitted == null || isAfter(oldestCommitted, msg.payload.committed)) + oldestCommitted = msg.payload.committed.ballot; + + if (isAfter(latestWitnessed, clashingPromise)) + clashingPromise = null; + if (timestampsClash(latestAccepted, msg.payload.latestWitnessedOrLowBound)) + clashingPromise = msg.payload.latestWitnessedOrLowBound; + if (timestampsClash(latestAccepted, latestWitnessed)) + clashingPromise = latestWitnessed; + + // once we receive the requisite number, we can simply proceed, and ignore future responses + if (++successes == participants.sizeOfConsensusQuorum) + return execute(); + + return this; + } + + private State execute() + { + // if we have a timestamp clash, always prefer the accepted ballot + latestWitnessed = latest(latestAccepted, latestWitnessed); + Ballot latestPreviouslyWitnessed = latest(successCriteria, prevSupersededBy); + + // Save as success criteria the latest promise seen by our first round; if we ever see anything + // newer committed, we know at least one paxos round has been completed since we started, which is all we need + // or newer than this committed we know we're done, so to avoid looping indefinitely in competition + // with others, we store this ballot for future retries so we can terminate based on other proposers' work + if (successCriteria == null || timestampsClash(successCriteria, latestWitnessed)) + { + if (logger.isTraceEnabled()) + logger.trace("PaxosRepair of {} setting success criteria to {}", partitionKey(), Ballot.toString(latestWitnessed)); + + successCriteria = latestWitnessed; + } + + boolean hasCommittedSuccessCriteria = isAfter(latestCommitted, successCriteria) || latestCommitted.hasBallot(successCriteria); + boolean isPromisedButNotAccepted = isAfter(latestWitnessed, latestAccepted); // not necessarily promised - may be lowBound + boolean isAcceptedButNotCommitted = isAfter(latestAccepted, latestCommitted); + boolean reproposalMayBeRejected = clashingPromise != null || !isAfter(latestWitnessed, latestPreviouslyWitnessed); + + if (hasCommittedSuccessCriteria) + { + if (logger.isTraceEnabled()) + logger.trace("PaxosRepair witnessed {} newer than success criteria {} (oldest: {})", latestCommitted, Ballot.toString(successCriteria), Ballot.toString(oldestCommitted)); + + // we have a new enough commit, but it might not have reached enough participants; make sure it has before terminating + // note: we could send to only those we know haven't witnessed it, but this is a rare operation so a small amount of redundant work is fine + return oldestCommitted.equals(latestCommitted.ballot) + ? DONE + : PaxosCommit.commit(latestCommitted, participants, paxosConsistency, commitConsistency(), true, + new CommittingRepair()); + } + else if (isAcceptedButNotCommitted && !isPromisedButNotAccepted && !reproposalMayBeRejected) + { + if (logger.isTraceEnabled()) + logger.trace("PaxosRepair of {} completing {}", partitionKey(), latestAccepted); + // We need to complete this in-progress accepted proposal, which may not have been seen by a majority + // However, since we have not sought any promises, we can simply complete the existing proposal + // since this is an idempotent operation - both us and the original proposer (and others) can + // all do it at the same time without incident + + // If ballots with same timestamp have been both accepted and rejected by different nodes, + // to avoid a livelock we simply try to poison, knowing we will fail but use a new ballot + // (note there are alternative approaches but this is conservative) + + return PaxosPropose.propose(latestAccepted, participants, false, + new ProposingRepair(latestAccepted)); + } + else if (isAcceptedButNotCommitted || isPromisedButNotAccepted || latestWitnessed.compareTo(latestPreviouslyWitnessed) < 0) + { + Ballot ballot = staleBallotNewerThan(latest(latestWitnessed, latestPreviouslyWitnessed), paxosConsistency); + // We need to propose a no-op > latestPromised, to ensure we don't later discover + // that latestPromised had already been accepted (by a minority) and repair it + // This means starting a new ballot, but we choose to use one that is likely to lose a contention battle + // Since this operation is not urgent, and we can piggy-back on other paxos operations + if (logger.isTraceEnabled()) + logger.trace("PaxosRepair of {} found incomplete promise or proposal; preparing stale ballot {}", partitionKey(), Ballot.toString(ballot)); + + return prepareWithBallot(ballot, participants, partitionKey(), table, false, false, + new PoisonProposals()); + } + else + { + logger.error("PaxosRepair illegal state latestWitnessed={}, latestAcceptedButNotCommitted={}, latestCommitted={}, oldestCommitted={}", latestWitnessed, latestAccepted, latestCommitted, oldestCommitted); + throw new IllegalStateException(); // should be logically impossible + } + } + + public void run() + { + Message message = Message.out(PAXOS2_REPAIR_REQ, new Request(partitionKey(), table)); + for (int i = 0, size = participants.sizeOfPoll(); i < size ; ++i) + MessagingService.instance().sendWithCallback(message, participants.voter(i), this); + } + } + + /** + * We found either an incomplete promise or proposal, so we need to start a new paxos round to complete them + */ + private class PoisonProposals extends ConsumerState + { + @Override + public State execute(Status input) throws Throwable + { + switch (input.outcome) + { + case MAYBE_FAILURE: + return retry(this); + + case READ_PERMITTED: + case SUPERSEDED: + prevSupersededBy = latest(prevSupersededBy, input.retryWithAtLeast()); + return retry(this); + + case FOUND_INCOMPLETE_ACCEPTED: + { + // finish the in-progress proposal + // cannot simply restart, as our latest promise is newer than the proposal + // so we require a promise before we decide which proposal to complete + // (else an "earlier" operation can sneak in and invalidate us while we're proposing + // with a newer ballot) + FoundIncompleteAccepted incomplete = input.incompleteAccepted(); + Proposal propose = new Proposal(incomplete.ballot, incomplete.accepted.update); + logger.trace("PaxosRepair of {} found incomplete {}", partitionKey(), incomplete.accepted); + return PaxosPropose.propose(propose, participants, false, + new ProposingRepair(propose)); // we don't know if we're done, so we must restart + } + + case FOUND_INCOMPLETE_COMMITTED: + { + // finish the in-progress commit + FoundIncompleteCommitted incomplete = input.incompleteCommitted(); + logger.trace("PaxosRepair of {} found in progress {}", partitionKey(), incomplete.committed); + return PaxosCommit.commit(incomplete.committed, participants, paxosConsistency, commitConsistency(), true, + new CommitAndRestart()); // we don't know if we're done, so we must restart + } + + case PROMISED: + { + // propose the empty ballot + logger.trace("PaxosRepair of {} submitting empty proposal", partitionKey()); + Proposal proposal = Proposal.empty(input.success().ballot, partitionKey(), table); + return PaxosPropose.propose(proposal, participants, false, + new ProposingRepair(proposal)); + } + + default: + throw new IllegalStateException(); + } + } + } + + private class ProposingRepair extends ConsumerState + { + final Proposal proposal; + private ProposingRepair(Proposal proposal) + { + this.proposal = proposal; + } + + @Override + public State execute(PaxosPropose.Status input) + { + switch (input.outcome) + { + case MAYBE_FAILURE: + return retry(this); + + case SUPERSEDED: + if (isAfter(input.superseded().by, prevSupersededBy)) + prevSupersededBy = input.superseded().by; + return retry(this); + + case SUCCESS: + if (proposal.update.isEmpty()) + { + logger.trace("PaxosRepair of {} complete after successful empty proposal", partitionKey()); + return DONE; + } + + logger.trace("PaxosRepair of {} committing successful proposal {}", partitionKey(), proposal); + return PaxosCommit.commit(proposal.agreed(), participants, paxosConsistency, commitConsistency(), true, + new CommittingRepair()); + + default: + throw new IllegalStateException(); + } + } + } + + private class CommittingRepair extends ConsumerState + { + @Override + public State execute(PaxosCommit.Status input) + { + logger.trace("PaxosRepair of {} {}", partitionKey(), input); + return input.isSuccess() ? DONE : retry(this); + } + } + + private class CommitAndRestart extends ConsumerState + { + @Override + public State execute(PaxosCommit.Status input) + { + return restart(this); + } + } + + private PaxosRepair(DecoratedKey partitionKey, Ballot incompleteBallot, TableMetadata table, ConsistencyLevel paxosConsistency) + { + super(partitionKey, incompleteBallot); + // TODO: move precondition into super ctor + Preconditions.checkArgument(paxosConsistency.isSerialConsistency()); + this.table = table; + this.paxosConsistency = paxosConsistency; + this.successCriteria = incompleteBallot; + } + + public static PaxosRepair create(ConsistencyLevel consistency, DecoratedKey partitionKey, Ballot incompleteBallot, TableMetadata table) + { + return new PaxosRepair(partitionKey, incompleteBallot, table, consistency); + } + + private State retry(State state) + { + Preconditions.checkState(isStarted()); + if (isResult(state)) + return state; + + return restart(state, waitUntilForContention(++attempts, table, partitionKey(), paxosConsistency, REPAIR)); + } + + @Override + public State restart(State state, long waitUntil) + { + if (isResult(state)) + return state; + + participants = Participants.get(table, partitionKey(), paxosConsistency); + + if (waitUntil > Long.MIN_VALUE && waitUntil - startedNanos() > RETRY_TIMEOUT_NANOS) + return new Failure(null); + + try + { + participants.assureSufficientLiveNodesForRepair(); + } + catch (UnavailableException e) + { + return new Failure(e); + } + + Querying querying = new Querying(); + long now; + if (waitUntil == Long.MIN_VALUE || waitUntil - (now = nanoTime()) < 0) querying.run(); + else RETRIES.schedule(querying, waitUntil - now, NANOSECONDS); + + return querying; + } + + private ConsistencyLevel commitConsistency() + { + Preconditions.checkState(paxosConsistency.isSerialConsistency()); + return paxosConsistency.isDatacenterLocal() ? ConsistencyLevel.LOCAL_QUORUM : ConsistencyLevel.QUORUM; + } + + static class Request + { + final DecoratedKey partitionKey; + final TableMetadata table; + Request(DecoratedKey partitionKey, TableMetadata table) + { + this.partitionKey = partitionKey; + this.table = table; + } + } + + /** + * The response to a proposal, indicating success (if {@code supersededBy == null}, + * or failure, alongside the ballot that beat us + */ + static class Response + { + @Nonnull final Ballot latestWitnessedOrLowBound; + @Nullable final Accepted acceptedButNotCommitted; + @Nonnull final Committed committed; + + Response(Ballot latestWitnessedOrLowBound, @Nullable Accepted acceptedButNotCommitted, Committed committed) + { + this.latestWitnessedOrLowBound = latestWitnessedOrLowBound; + this.acceptedButNotCommitted = acceptedButNotCommitted; + this.committed = committed; + } + + public String toString() + { + return String.format("Response(%s, %s, %s", latestWitnessedOrLowBound, acceptedButNotCommitted, committed); + } + } + + private static Map> mapToDc(Collection endpoints, Function dcFunc) + { + Map> map = new HashMap<>(); + endpoints.forEach(e -> map.computeIfAbsent(dcFunc.apply(e), k -> new HashSet<>()).add(e)); + return map; + } + + private static boolean hasQuorumOrSingleDead(Collection all, Collection live, boolean requireQuorum) + { + Preconditions.checkArgument(all.size() >= live.size()); + return live.size() >= (all.size() / 2) + 1 || (!requireQuorum && live.size() >= all.size() - 1); + } + + @VisibleForTesting + static boolean hasSufficientLiveNodesForTopologyChange(Collection allEndpoints, Collection liveEndpoints, Function dcFunc, boolean onlyQuorumRequired, boolean strictQuorum) + { + + Map> allDcMap = mapToDc(allEndpoints, dcFunc); + Map> liveDcMap = mapToDc(liveEndpoints, dcFunc); + + if (!hasQuorumOrSingleDead(allEndpoints, liveEndpoints, strictQuorum)) + return false; + + if (onlyQuorumRequired) + return true; + + for (Map.Entry> entry : allDcMap.entrySet()) + { + Set all = entry.getValue(); + Set live = liveDcMap.getOrDefault(entry.getKey(), Collections.emptySet()); + if (!hasQuorumOrSingleDead(all, live, strictQuorum)) + return false; + } + return true; + } + + /** + * checks if we have enough live nodes to perform a paxos repair for topology repair. Generally, this means that we need enough + * live participants to reach EACH_QUORUM, with a few exceptions. The EACH_QUORUM requirement is meant to support workload using either + * SERIAL or LOCAL_SERIAL + * + * if paxos_topology_repair_strict_each_quorum is set to false (the default), we will accept either a quorum or n-1 live nodes + * in the cluster and per dc. If paxos_topology_repair_no_dc_checks is true, we only check the live nodes in the cluster, + * and do not do any per-dc checks. + */ + public static boolean hasSufficientLiveNodesForTopologyChange(Keyspace keyspace, Range range, Collection liveEndpoints) + { + return hasSufficientLiveNodesForTopologyChange(keyspace.getReplicationStrategy().getNaturalReplicasForToken(range.right).endpoints(), + liveEndpoints, + DatabaseDescriptor.getEndpointSnitch()::getDatacenter, + DatabaseDescriptor.paxoTopologyRepairNoDcChecks(), + DatabaseDescriptor.paxoTopologyRepairStrictEachQuorum()); + } + + /** + * The proposal request handler, i.e. receives a proposal from a peer and responds with either acccept/reject + */ + public static class RequestHandler implements IVerbHandler + { + @Override + public void doVerb(Message message) + { + PaxosRepair.Request request = message.payload; + if (!isInRangeAndShouldProcess(message.from(), request.partitionKey, request.table, false)) + { + MessagingService.instance().respondWithFailure(UNKNOWN, message); + return; + } + + Ballot latestWitnessed; + Accepted acceptedButNotCommited; + Committed committed; + int nowInSec = FBUtilities.nowInSeconds(); + try (PaxosState state = PaxosState.get(request.partitionKey, request.table)) + { + PaxosState.Snapshot snapshot = state.current(nowInSec); + latestWitnessed = snapshot.latestWitnessedOrLowBound(); + acceptedButNotCommited = snapshot.accepted; + committed = snapshot.committed; + } + + Response response = new Response(latestWitnessed, acceptedButNotCommited, committed); + MessagingService.instance().respond(response, message); + } + } + + public static class RequestSerializer implements IVersionedSerializer + { + @Override + public void serialize(Request request, DataOutputPlus out, int version) throws IOException + { + request.table.id.serialize(out); + DecoratedKey.serializer.serialize(request.partitionKey, out, version); + } + + @Override + public Request deserialize(DataInputPlus in, int version) throws IOException + { + TableMetadata table = Schema.instance.getExistingTableMetadata(TableId.deserialize(in)); + DecoratedKey partitionKey = (DecoratedKey) DecoratedKey.serializer.deserialize(in, table.partitioner, version); + return new Request(partitionKey, table); + } + + @Override + public long serializedSize(Request request, int version) + { + return request.table.id.serializedSize() + + DecoratedKey.serializer.serializedSize(request.partitionKey, version); + } + } + + public static class ResponseSerializer implements IVersionedSerializer + { + public void serialize(Response response, DataOutputPlus out, int version) throws IOException + { + response.latestWitnessedOrLowBound.serialize(out); + serializeNullable(Accepted.serializer, response.acceptedButNotCommitted, out, version); + Committed.serializer.serialize(response.committed, out, version); + } + + public Response deserialize(DataInputPlus in, int version) throws IOException + { + Ballot latestWitnessed = Ballot.deserialize(in); + Accepted acceptedButNotCommitted = deserializeNullable(Accepted.serializer, in, version); + Committed committed = Committed.serializer.deserialize(in, version); + return new Response(latestWitnessed, acceptedButNotCommitted, committed); + } + + public long serializedSize(Response response, int version) + { + return Ballot.sizeInBytes() + + serializedSizeNullable(Accepted.serializer, response.acceptedButNotCommitted, version) + + Committed.serializer.serializedSize(response.committed, version); + } + } + + private static volatile boolean SKIP_VERSION_VALIDATION = Boolean.getBoolean("cassandra.skip_paxos_repair_version_validation"); + + public static void setSkipPaxosRepairCompatibilityCheck(boolean v) + { + SKIP_VERSION_VALIDATION = v; + } + + public static boolean getSkipPaxosRepairCompatibilityCheck() + { + return SKIP_VERSION_VALIDATION; + } + + static boolean validateVersionCompatibility(CassandraVersion version) + { + if (SKIP_VERSION_VALIDATION) + return true; + + if (version == null) + return false; + + // assume 4.0 is ok + return (version.major == 4 && version.minor > 0) || version.major > 4; + } + + static String getPeerVersion(InetAddressAndPort peer) + { + EndpointState epState = Gossiper.instance.getEndpointStateForEndpoint(peer); + if (epState == null) + return null; + + VersionedValue value = epState.getApplicationState(ApplicationState.RELEASE_VERSION); + if (value == null) + return null; + + try + { + return value.value; + } + catch (IllegalArgumentException e) + { + return null; + } + } + + static boolean validatePeerCompatibility(Replica peer) + { + String versionString = getPeerVersion(peer.endpoint()); + CassandraVersion version = versionString != null ? new CassandraVersion(versionString) : null; + boolean result = validateVersionCompatibility(version); + if (!result) + logger.info("PaxosRepair isn't supported by {} on version {}", peer, versionString); + return result; + } + + static boolean validatePeerCompatibility(TableMetadata table, Range range) + { + Participants participants = Participants.get(table, range.right, ConsistencyLevel.SERIAL); + return Iterables.all(participants.all, PaxosRepair::validatePeerCompatibility); + } + + public static boolean validatePeerCompatibility(TableMetadata table, Collection> ranges) + { + return Iterables.all(ranges, range -> validatePeerCompatibility(table, range)); + } + + public static void shutdownAndWait(long timeout, TimeUnit units) throws InterruptedException, TimeoutException + { + ExecutorUtils.shutdownAndWait(timeout, units, RETRIES); + } +} diff --git a/src/java/org/apache/cassandra/service/paxos/PaxosRepairHistory.java b/src/java/org/apache/cassandra/service/paxos/PaxosRepairHistory.java new file mode 100644 index 0000000000..a88e83136e --- /dev/null +++ b/src/java/org/apache/cassandra/service/paxos/PaxosRepairHistory.java @@ -0,0 +1,480 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.paxos; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.*; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableList; + +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.db.TypeSizes; +import org.apache.cassandra.db.marshal.BytesType; +import org.apache.cassandra.db.marshal.TimeUUIDType; +import org.apache.cassandra.db.marshal.TupleType; +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.Token; +import org.apache.cassandra.io.IVersionedSerializer; +import org.apache.cassandra.io.util.DataInputPlus; +import org.apache.cassandra.io.util.DataOutputPlus; + +import static java.lang.Math.min; +import static org.apache.cassandra.service.paxos.Commit.isAfter; +import static org.apache.cassandra.service.paxos.Commit.latest; + +public class PaxosRepairHistory +{ + public static final PaxosRepairHistory EMPTY = new PaxosRepairHistory(new Token[0], new Ballot[] { Ballot.none() }); + private static final Token.TokenFactory TOKEN_FACTORY = DatabaseDescriptor.getPartitioner().getTokenFactory(); + private static final Token MIN_TOKEN = DatabaseDescriptor.getPartitioner().getMinimumToken(); + private static final TupleType TYPE = new TupleType(ImmutableList.of(BytesType.instance, BytesType.instance)); + + /** + * The following two fields represent the mapping of ranges to ballot lower bounds, for example: + * + * ballotLowBound = [ none(), b2, none(), b4, none() ] + * tokenInclusiveUpperBound = [ t1, t2, t3, t4 ] + * + * Correspond to the following token bounds: + * + * (MIN_VALUE, t1] => none() + * (t1, t2] => b2 + * (t2, t3] => none() + * (t3, t4] => b4 + * (t4, MAX_VALUE) => none() + */ + + private final Token[] tokenInclusiveUpperBound; + private final Ballot[] ballotLowBound; // always one longer to capture values up to "MAX_VALUE" (which in some cases doesn't exist, as is infinite) + + PaxosRepairHistory(Token[] tokenInclusiveUpperBound, Ballot[] ballotLowBound) + { + assert ballotLowBound.length == tokenInclusiveUpperBound.length + 1; + this.tokenInclusiveUpperBound = tokenInclusiveUpperBound; + this.ballotLowBound = ballotLowBound; + } + + public Ballot maxLowBound() + { + Ballot maxBallot = Ballot.none(); + for (Ballot lowBound : ballotLowBound) + { + maxBallot = Commit.latest(maxBallot, lowBound); + } + return maxBallot; + } + + public String toString() + { + return "PaxosRepairHistory{" + + IntStream.range(0, ballotLowBound.length) + .filter(i -> !Ballot.none().equals(ballotLowBound[i])) + .mapToObj(i -> range(i) + "=" + ballotLowBound[i]) + .collect(Collectors.joining(", ")) + '}'; + } + + public boolean equals(Object o) + { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + PaxosRepairHistory that = (PaxosRepairHistory) o; + return Arrays.equals(ballotLowBound, that.ballotLowBound) && Arrays.equals(tokenInclusiveUpperBound, that.tokenInclusiveUpperBound); + } + + public int hashCode() + { + return Arrays.hashCode(ballotLowBound); + } + + public Ballot ballotForToken(Token token) + { + int idx = Arrays.binarySearch(tokenInclusiveUpperBound, token); + if (idx < 0) idx = -1 - idx; + return ballotLowBound[idx]; + } + + private Ballot ballotForIndex(int idx) + { + if (idx < 0 || idx > size()) + throw new IndexOutOfBoundsException(); + + return ballotLowBound[idx]; + } + + private int indexForToken(Token token) + { + int idx = Arrays.binarySearch(tokenInclusiveUpperBound, token); + if (idx < 0) idx = -1 - idx; + return idx; + } + + private boolean contains(int idx, Token token) + { + if (idx < 0 || idx > size()) + throw new IndexOutOfBoundsException(); + + return (idx == 0 || tokenInclusiveUpperBound[idx - 1].compareTo(token) < 0) + && (idx == size() || tokenInclusiveUpperBound[idx ].compareTo(token) >= 0); + } + + public int size() + { + return tokenInclusiveUpperBound.length; + } + + private RangeIterator rangeIterator() + { + return new RangeIterator(); + } + + private Range range(int i) + { + return new Range<>(tokenExclusiveLowerBound(i), tokenInclusiveUpperBound(i)); + } + + public Searcher searcher() + { + return new Searcher(); + } + + private Token tokenExclusiveLowerBound(int i) + { + return i == 0 ? MIN_TOKEN : tokenInclusiveUpperBound[i - 1]; + } + + private Token tokenInclusiveUpperBound(int i) + { + return i == tokenInclusiveUpperBound.length ? MIN_TOKEN : tokenInclusiveUpperBound[i]; + } + + public List toTupleBufferList() + { + List tuples = new ArrayList<>(size() + 1); + for (int i = 0 ; i < 1 + size() ; ++i) + tuples.add(TupleType.buildValue(new ByteBuffer[] { TOKEN_FACTORY.toByteArray(tokenInclusiveUpperBound(i)), ballotLowBound[i].toBytes() })); + return tuples; + } + + public static PaxosRepairHistory fromTupleBufferList(List tuples) + { + Token[] tokenInclusiveUpperBounds = new Token[tuples.size() - 1]; + Ballot[] ballotLowBounds = new Ballot[tuples.size()]; + for (int i = 0 ; i < tuples.size() ; ++i) + { + ByteBuffer[] split = TYPE.split(tuples.get(i)); + if (i < tokenInclusiveUpperBounds.length) + tokenInclusiveUpperBounds[i] = TOKEN_FACTORY.fromByteArray(split[0]); + ballotLowBounds[i] = Ballot.deserialize(split[1]); + } + + return new PaxosRepairHistory(tokenInclusiveUpperBounds, ballotLowBounds); + } + + // append the item to the given list, modifying the underlying list + // if the item makes previoud entries redundant + + public static PaxosRepairHistory merge(PaxosRepairHistory historyLeft, PaxosRepairHistory historyRight) + { + if (historyLeft == null) + return historyRight; + + if (historyRight == null) + return historyLeft; + + Builder builder = new Builder(historyLeft.size() + historyRight.size()); + + RangeIterator left = historyLeft.rangeIterator(); + RangeIterator right = historyRight.rangeIterator(); + while (left.hasUpperBound() && right.hasUpperBound()) + { + int cmp = left.tokenInclusiveUpperBound().compareTo(right.tokenInclusiveUpperBound()); + + Ballot ballot = latest(left.ballotLowBound(), right.ballotLowBound()); + if (cmp == 0) + { + builder.append(left.tokenInclusiveUpperBound(), ballot); + left.next(); + right.next(); + } + else + { + RangeIterator firstIter = cmp < 0 ? left : right; + builder.append(firstIter.tokenInclusiveUpperBound(), ballot); + firstIter.next(); + } + } + + while (left.hasUpperBound()) + { + builder.append(left.tokenInclusiveUpperBound(), latest(left.ballotLowBound(), right.ballotLowBound())); + left.next(); + } + + while (right.hasUpperBound()) + { + builder.append(right.tokenInclusiveUpperBound(), latest(left.ballotLowBound(), right.ballotLowBound())); + right.next(); + } + + builder.appendLast(latest(left.ballotLowBound(), right.ballotLowBound())); + return builder.build(); + } + + @VisibleForTesting + public static PaxosRepairHistory add(PaxosRepairHistory existing, Collection> ranges, Ballot ballot) + { + ranges = Range.normalize(ranges); + Builder builder = new Builder(ranges.size() * 2); + for (Range range : ranges) + { + // don't add a point for an opening min token, since it + // effectively leaves the bottom of the range unbounded + builder.appendMaybeMin(range.left, Ballot.none()); + builder.appendMaybeMax(range.right, ballot); + } + + return merge(existing, builder.build()); + } + + /** + * returns a copy of this PaxosRepairHistory limited to the ranges supplied, with all other ranges reporting Ballot.none() + */ + @VisibleForTesting + static PaxosRepairHistory trim(PaxosRepairHistory existing, Collection> ranges) + { + Builder builder = new Builder(existing.size()); + + ranges = Range.normalize(ranges); + for (Range select : ranges) + { + RangeIterator intersects = existing.intersects(select); + while (intersects.hasNext()) + { + if (Ballot.none().equals(intersects.ballotLowBound())) + { + intersects.next(); + continue; + } + + Token exclusiveLowerBound = maxExclusiveLowerBound(select.left, intersects.tokenExclusiveLowerBound()); + Token inclusiveUpperBound = minInclusiveUpperBound(select.right, intersects.tokenInclusiveUpperBound()); + assert exclusiveLowerBound.compareTo(inclusiveUpperBound) < 0 || inclusiveUpperBound.isMinimum(); + + builder.appendMaybeMin(exclusiveLowerBound, Ballot.none()); + builder.appendMaybeMax(inclusiveUpperBound, intersects.ballotLowBound()); + intersects.next(); + } + } + + return builder.build(); + } + + RangeIterator intersects(Range unwrapped) + { + int from = Arrays.binarySearch(tokenInclusiveUpperBound, unwrapped.left); + if (from < 0) from = -1 - from; else ++from; + int to = unwrapped.right.isMinimum() ? ballotLowBound.length - 1 : Arrays.binarySearch(tokenInclusiveUpperBound, unwrapped.right); + if (to < 0) to = -1 - to; + return new RangeIterator(from, min(1 + to, ballotLowBound.length)); + } + + private static Token maxExclusiveLowerBound(Token a, Token b) + { + return a.compareTo(b) < 0 ? b : a; + } + + private static Token minInclusiveUpperBound(Token a, Token b) + { + if (!a.isMinimum() && !b.isMinimum()) return a.compareTo(b) <= 0 ? a : b; + else if (!a.isMinimum()) return a; + else if (!b.isMinimum()) return b; + else return a; + } + + public static final IVersionedSerializer serializer = new IVersionedSerializer() + { + public void serialize(PaxosRepairHistory history, DataOutputPlus out, int version) throws IOException + { + out.writeUnsignedVInt(history.size()); + for (int i = 0; i < history.size() ; ++i) + { + Token.serializer.serialize(history.tokenInclusiveUpperBound[i], out, version); + history.ballotLowBound[i].serialize(out); + } + history.ballotLowBound[history.size()].serialize(out); + } + + public PaxosRepairHistory deserialize(DataInputPlus in, int version) throws IOException + { + int size = (int) in.readUnsignedVInt(); + Token[] tokenInclusiveUpperBounds = new Token[size]; + Ballot[] ballotLowBounds = new Ballot[size + 1]; + for (int i = 0; i < size; i++) + { + tokenInclusiveUpperBounds[i] = Token.serializer.deserialize(in, DatabaseDescriptor.getPartitioner(), version); + ballotLowBounds[i] = Ballot.deserialize(in); + } + ballotLowBounds[size] = Ballot.deserialize(in); + return new PaxosRepairHistory(tokenInclusiveUpperBounds, ballotLowBounds); + } + + public long serializedSize(PaxosRepairHistory history, int version) + { + long size = TypeSizes.sizeofUnsignedVInt(history.size()); + for (int i = 0; i < history.size() ; ++i) + { + size += Token.serializer.serializedSize(history.tokenInclusiveUpperBound[i], version); + size += Ballot.sizeInBytes(); + } + size += Ballot.sizeInBytes(); + return size; + } + }; + + public class Searcher + { + int idx = -1; + + public Ballot ballotForToken(Token token) + { + if (idx < 0 || !contains(idx, token)) + idx = indexForToken(token); + return ballotForIndex(idx); + } + } + + class RangeIterator + { + final int end; + int i; + + RangeIterator() + { + this.end = ballotLowBound.length; + } + + RangeIterator(int from, int to) + { + this.i = from; + this.end = to; + } + + boolean hasNext() + { + return i < end; + } + + boolean hasUpperBound() + { + return i < tokenInclusiveUpperBound.length; + } + + void next() + { + ++i; + } + + Token tokenExclusiveLowerBound() + { + return PaxosRepairHistory.this.tokenExclusiveLowerBound(i); + } + + Token tokenInclusiveUpperBound() + { + return PaxosRepairHistory.this.tokenInclusiveUpperBound(i); + } + + Ballot ballotLowBound() + { + return ballotLowBound[i]; + } + } + + static class Builder + { + final List tokenInclusiveUpperBounds; + final List ballotLowBounds; + + Builder(int capacity) + { + this.tokenInclusiveUpperBounds = new ArrayList<>(capacity); + this.ballotLowBounds = new ArrayList<>(capacity + 1); + } + + void appendMaybeMin(Token inclusiveLowBound, Ballot ballotLowBound) + { + if (inclusiveLowBound.isMinimum()) + assert ballotLowBound.equals(Ballot.none()) && ballotLowBounds.isEmpty(); + else + append(inclusiveLowBound, ballotLowBound); + } + + void appendMaybeMax(Token inclusiveLowBound, Ballot ballotLowBound) + { + if (inclusiveLowBound.isMinimum()) + appendLast(ballotLowBound); + else + append(inclusiveLowBound, ballotLowBound); + } + + void append(Token inclusiveLowBound, Ballot ballotLowBound) + { + int tailIdx = tokenInclusiveUpperBounds.size() - 1; + + assert tokenInclusiveUpperBounds.size() == ballotLowBounds.size(); + assert tailIdx < 0 || inclusiveLowBound.compareTo(tokenInclusiveUpperBounds.get(tailIdx)) >= 0; + + boolean sameAsTailToken = tailIdx >= 0 && inclusiveLowBound.equals(tokenInclusiveUpperBounds.get(tailIdx)); + boolean sameAsTailBallot = tailIdx >= 0 && ballotLowBound.equals(ballotLowBounds.get(tailIdx)); + if (sameAsTailToken || sameAsTailBallot) + { + if (sameAsTailBallot) + tokenInclusiveUpperBounds.set(tailIdx, inclusiveLowBound); + else if (isAfter(ballotLowBound, ballotLowBounds.get(tailIdx))) + ballotLowBounds.set(tailIdx, ballotLowBound); + } + else + { + tokenInclusiveUpperBounds.add(inclusiveLowBound); + ballotLowBounds.add(ballotLowBound); + } + } + + void appendLast(Ballot ballotLowBound) + { + assert ballotLowBounds.size() == tokenInclusiveUpperBounds.size(); + int tailIdx = tokenInclusiveUpperBounds.size() - 1; + if (!ballotLowBounds.isEmpty() && ballotLowBound.equals(ballotLowBounds.get(tailIdx))) + tokenInclusiveUpperBounds.remove(tailIdx); + else + ballotLowBounds.add(ballotLowBound); + } + + PaxosRepairHistory build() + { + if (tokenInclusiveUpperBounds.size() == ballotLowBounds.size()) + ballotLowBounds.add(Ballot.none()); + return new PaxosRepairHistory(tokenInclusiveUpperBounds.toArray(new Token[0]), ballotLowBounds.toArray(new Ballot[0])); + } + } +} diff --git a/src/java/org/apache/cassandra/service/paxos/PaxosRequestCallback.java b/src/java/org/apache/cassandra/service/paxos/PaxosRequestCallback.java new file mode 100644 index 0000000000..282aeb2dc1 --- /dev/null +++ b/src/java/org/apache/cassandra/service/paxos/PaxosRequestCallback.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.paxos; + +import java.util.function.BiFunction; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.config.CassandraRelevantProperties; +import org.apache.cassandra.exceptions.RequestFailureReason; +import org.apache.cassandra.exceptions.WriteTimeoutException; +import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.net.Message; +import org.apache.cassandra.service.FailureRecordingCallback; + +import static org.apache.cassandra.exceptions.RequestFailureReason.TIMEOUT; +import static org.apache.cassandra.exceptions.RequestFailureReason.UNKNOWN; +import static org.apache.cassandra.utils.FBUtilities.getBroadcastAddressAndPort; + +public abstract class PaxosRequestCallback extends FailureRecordingCallback +{ + private static final Logger logger = LoggerFactory.getLogger(PaxosRequestCallback.class); + private static final boolean USE_SELF_EXECUTION = CassandraRelevantProperties.PAXOS_EXECUTE_ON_SELF.getBoolean(); + + protected abstract void onResponse(T response, InetAddressAndPort from); + + @Override + public void onResponse(Message message) + { + onResponse(message.payload, message.from()); + } + + protected void executeOnSelf(I parameter, BiFunction execute) + { + T response; + try + { + response = execute.apply(parameter, getBroadcastAddressAndPort()); + if (response == null) + return; + } + catch (Exception ex) + { + RequestFailureReason reason = UNKNOWN; + if (ex instanceof WriteTimeoutException) reason = TIMEOUT; + else logger.error("Failed to apply {} locally", parameter, ex); + + onFailure(getBroadcastAddressAndPort(), reason); + return; + } + + onResponse(response, getBroadcastAddressAndPort()); + } + + static boolean shouldExecuteOnSelf(InetAddressAndPort replica) + { + return USE_SELF_EXECUTION && replica.equals(getBroadcastAddressAndPort()); + } +} diff --git a/src/java/org/apache/cassandra/service/paxos/PaxosState.java b/src/java/org/apache/cassandra/service/paxos/PaxosState.java index 4c5665c253..e802cd070f 100644 --- a/src/java/org/apache/cassandra/service/paxos/PaxosState.java +++ b/src/java/org/apache/cassandra/service/paxos/PaxosState.java @@ -1,5 +1,5 @@ /* - * + * * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -7,179 +7,816 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. - * + * */ package org.apache.cassandra.service.paxos; +import java.io.IOException; +import java.util.Map; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; +import java.util.function.BiConsumer; +import java.util.function.Function; -import org.apache.cassandra.schema.TableMetadata; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.primitives.Ints; + +import com.github.benmanes.caffeine.cache.Caffeine; +import org.apache.cassandra.concurrent.ImmediateExecutor; +import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.*; +import org.apache.cassandra.metrics.PaxosMetrics; +import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.exceptions.ReadTimeoutException; +import org.apache.cassandra.exceptions.RequestTimeoutException; +import org.apache.cassandra.exceptions.WriteTimeoutException; +import org.apache.cassandra.service.paxos.uncommitted.PaxosBallotTracker; +import org.apache.cassandra.service.paxos.uncommitted.PaxosStateTracker; +import org.apache.cassandra.service.paxos.uncommitted.PaxosUncommittedTracker; import org.apache.cassandra.tracing.Tracing; +import org.apache.cassandra.utils.Nemesis; import static java.util.concurrent.TimeUnit.SECONDS; import static org.apache.cassandra.utils.Clock.Global.nanoTime; +import static org.apache.cassandra.config.Config.PaxosStatePurging.gc_grace; +import static org.apache.cassandra.config.Config.PaxosStatePurging.legacy; +import static org.apache.cassandra.config.DatabaseDescriptor.paxosStatePurging; +import static org.apache.cassandra.service.paxos.Commit.*; +import static org.apache.cassandra.service.paxos.PaxosState.MaybePromise.Outcome.*; +import static org.apache.cassandra.service.paxos.Commit.Accepted.latestAccepted; +import static org.apache.cassandra.service.paxos.Commit.Committed.latestCommitted; +import static org.apache.cassandra.service.paxos.Commit.isAfter; -public class PaxosState +/** + * We save to memory the result of each operation before persisting to disk, however each operation that performs + * the update does not return a result to the coordinator until the result is fully persisted. + */ +public class PaxosState implements PaxosOperationLock { - private static class Lock - { - private static final ConcurrentMap LOCKS = new ConcurrentHashMap<>(); - int count = 1; + private static volatile boolean DISABLE_COORDINATOR_LOCKING = Boolean.getBoolean("cassandra.paxos.disable_coordinator_locking"); + public static final ConcurrentHashMap ACTIVE = new ConcurrentHashMap<>(); + public static final Map RECENT = Caffeine.newBuilder() + .maximumWeight(DatabaseDescriptor.getPaxosCacheSizeInMiB() << 20) + .weigher((k, v) -> Ints.saturatedCast((v.accepted != null ? v.accepted.update.unsharedHeapSize() : 0L) + v.committed.update.unsharedHeapSize())) + .executor(ImmediateExecutor.INSTANCE) + .build().asMap(); - static Lock get(DecoratedKey key) + private static class TrackerHandle + { + static final PaxosStateTracker tracker; + + static { - return LOCKS.compute(key, (ignore, cur) -> { - if (cur == null) - return new Lock(); - ++cur.count; + try + { + tracker = PaxosStateTracker.create(Directories.dataDirectories); + } + catch (IOException e) + { + throw new RuntimeException(e); + } + } + } + + public static void setDisableCoordinatorLocking(boolean disable) + { + DISABLE_COORDINATOR_LOCKING = disable; + } + + public static boolean getDisableCoordinatorLocking() + { + return DISABLE_COORDINATOR_LOCKING; + } + + public static PaxosUncommittedTracker uncommittedTracker() + { + return TrackerHandle.tracker.uncommitted(); + } + + public static PaxosBallotTracker ballotTracker() + { + return TrackerHandle.tracker.ballots(); + } + + public static void initializeTrackers() + { + Preconditions.checkState(TrackerHandle.tracker != null); + PaxosMetrics.initialize(); + } + + public static void maybeRebuildUncommittedState() throws IOException + { + TrackerHandle.tracker.maybeRebuild(); + } + + public static void startAutoRepairs() + { + TrackerHandle.tracker.uncommitted().startAutoRepairs(); + } + + public static class Key + { + final DecoratedKey partitionKey; + final TableMetadata metadata; + + public Key(DecoratedKey partitionKey, TableMetadata metadata) + { + this.partitionKey = partitionKey; + this.metadata = metadata; + } + + public int hashCode() + { + return partitionKey.hashCode() * 31 + metadata.id.hashCode(); + } + + public boolean equals(Object that) + { + return that instanceof Key && equals((Key) that); + } + + public boolean equals(Key that) + { + return this.partitionKey.equals(that.partitionKey) + && this.metadata.id.equals(that.metadata.id); + } + } + + public static class Snapshot + { + public final @Nonnull + Ballot promised; + public final @Nonnull + Ballot promisedWrite; // <= promised + public final @Nullable Accepted accepted; // if already committed, this will be null + public final @Nonnull Committed committed; + + public Snapshot(@Nonnull Ballot promised, @Nonnull Ballot promisedWrite, @Nullable Accepted accepted, @Nonnull Committed committed) + { + assert isAfter(promised, promisedWrite) || promised == promisedWrite; + assert accepted == null || accepted.update.partitionKey().equals(committed.update.partitionKey()); + assert accepted == null || accepted.update.metadata().id.equals(committed.update.metadata().id); + assert accepted == null || committed.isBefore(accepted.ballot); + + this.promised = promised; + this.promisedWrite = promisedWrite; + this.accepted = accepted; + this.committed = committed; + } + + public @Nonnull + Ballot latestWitnessedOrLowBound(Ballot latestWriteOrLowBound) + { + return promised == promisedWrite ? latestWriteOrLowBound : latest(promised, latestWriteOrLowBound); + } + + public @Nonnull + Ballot latestWitnessedOrLowBound() + { + // warn: if proposal has same timestamp as promised, we should prefer accepted + // since (if different) it reached a quorum of promises; this means providing it as first argument + Ballot latest; + latest = latest(accepted, committed).ballot; + latest = latest(latest, promised); + latest = latest(latest, ballotTracker().getLowBound()); + return latest; + } + + public @Nonnull + Ballot latestWriteOrLowBound() + { + // warn: if proposal has same timestamp as promised, we should prefer accepted + // since (if different) it reached a quorum of promises; this means providing it as first argument + Ballot latest = accepted != null && !accepted.update.isEmpty() ? accepted.ballot : null; + latest = latest(latest, committed.ballot); + latest = latest(latest, promisedWrite); + latest = latest(latest, ballotTracker().getLowBound()); + return latest; + } + + public static Snapshot merge(Snapshot a, Snapshot b) + { + if (a == null || b == null) + return a == null ? b : a; + + Committed committed = latestCommitted(a.committed, b.committed); + if (a instanceof UnsafeSnapshot && b instanceof UnsafeSnapshot) + return new UnsafeSnapshot(committed); + + Accepted accepted; + Ballot promised, promisedWrite; + if (a instanceof UnsafeSnapshot || b instanceof UnsafeSnapshot) + { + if (a instanceof UnsafeSnapshot) + a = b; // we already have the winning Committed saved above, so just want the full snapshot (if either) + + if (committed == a.committed) + return a; + + promised = a.promised; + promisedWrite = a.promisedWrite; + accepted = isAfter(a.accepted, committed) ? a.accepted : null; + } + else + { + accepted = latestAccepted(a.accepted, b.accepted); + accepted = isAfter(accepted, committed) ? accepted : null; + promised = latest(a.promised, b.promised); + promisedWrite = latest(a.promisedWrite, b.promisedWrite); + } + + return new Snapshot(promised, promisedWrite, accepted, committed); + } + + Snapshot removeExpired(int nowInSec) + { + boolean isAcceptedExpired = accepted != null && accepted.isExpired(nowInSec); + boolean isCommittedExpired = committed.isExpired(nowInSec); + + if (paxosStatePurging() == gc_grace) + { + long expireOlderThan = SECONDS.toMicros(nowInSec - committed.update.metadata().params.gcGraceSeconds); + isAcceptedExpired |= accepted != null && accepted.ballot.unixMicros() < expireOlderThan; + isCommittedExpired |= committed.ballot.unixMicros() < expireOlderThan; + } + + if (!isAcceptedExpired && !isCommittedExpired) + return this; + + return new Snapshot(promised, promisedWrite, + isAcceptedExpired ? null : accepted, + isCommittedExpired + ? Committed.none(committed.update.partitionKey(), committed.update.metadata()) + : committed); + } + } + + // used to permit recording Committed outcomes without waiting for initial read + public static class UnsafeSnapshot extends Snapshot + { + public UnsafeSnapshot(@Nonnull Committed committed) + { + super(Ballot.none(), Ballot.none(), null, committed); + } + + public UnsafeSnapshot(@Nonnull Commit committed) + { + this(new Committed(committed.ballot, committed.update)); + } + } + + @VisibleForTesting + public static class MaybePromise + { + public enum Outcome { REJECT, PERMIT_READ, PROMISE } + + final Snapshot before; + final Snapshot after; + final Ballot supersededBy; + final Outcome outcome; + + MaybePromise(Snapshot before, Snapshot after, Ballot supersededBy, Outcome outcome) + { + this.before = before; + this.after = after; + this.supersededBy = supersededBy; + this.outcome = outcome; + } + + static MaybePromise promise(Snapshot before, Snapshot after) + { + return new MaybePromise(before, after, null, PROMISE); + } + + static MaybePromise permitRead(Snapshot before, Ballot supersededBy) + { + return new MaybePromise(before, before, supersededBy, PERMIT_READ); + } + + static MaybePromise reject(Snapshot snapshot, Ballot supersededBy) + { + return new MaybePromise(snapshot, snapshot, supersededBy, REJECT); + } + + public Outcome outcome() + { + return outcome; + } + + public Ballot supersededBy() + { + return supersededBy; + } + } + + @Nemesis private static final AtomicReferenceFieldUpdater currentUpdater = AtomicReferenceFieldUpdater.newUpdater(PaxosState.class, Snapshot.class, "current"); + + final Key key; + private int active; // current number of active referents (once drops to zero, we remove the global entry) + @Nemesis private volatile Snapshot current; + @Nemesis private volatile Thread lockedBy; + @Nemesis private volatile int waiting; + + private static final AtomicReferenceFieldUpdater lockedByUpdater = AtomicReferenceFieldUpdater.newUpdater(PaxosState.class, Thread.class, "lockedBy"); + + private PaxosState(Key key, Snapshot current) + { + this.key = key; + this.current = current; + } + + @VisibleForTesting + public static PaxosState get(Commit commit) + { + return get(commit.update.partitionKey(), commit.update.metadata()); + } + + public static PaxosState get(DecoratedKey partitionKey, TableMetadata table) + { + // TODO would be nice to refactor verb handlers to support re-submitting to executor if waiting for another thread to read state + return getUnsafe(partitionKey, table).maybeLoad(); + } + + // does not increment total number of accessors, since we would accept null (so only access if others are, not for own benefit) + private static PaxosState tryGetUnsafe(DecoratedKey partitionKey, TableMetadata metadata) + { + return ACTIVE.compute(new Key(partitionKey, metadata), (key, cur) -> { + if (cur == null) + { + Snapshot saved = RECENT.remove(key); + if (saved != null) + //noinspection resource + cur = new PaxosState(key, saved); + } + if (cur != null) + ++cur.active; + return cur; + }); + } + + private static PaxosState getUnsafe(DecoratedKey partitionKey, TableMetadata metadata) + { + return ACTIVE.compute(new Key(partitionKey, metadata), (key, cur) -> { + if (cur == null) + { + //noinspection resource + cur = new PaxosState(key, RECENT.remove(key)); + } + ++cur.active; + return cur; + }); + } + + private static PaxosState getUnsafe(Commit commit) + { + return getUnsafe(commit.update.partitionKey(), commit.update.metadata()); + } + + // don't increment the total count, as we are only using this for locking purposes when coordinating + @VisibleForTesting + public static PaxosOperationLock lock(DecoratedKey partitionKey, TableMetadata metadata, long deadline, ConsistencyLevel consistencyForConsensus, boolean isWrite) throws RequestTimeoutException + { + if (DISABLE_COORDINATOR_LOCKING) + return PaxosOperationLock.noOp(); + + PaxosState lock = ACTIVE.compute(new Key(partitionKey, metadata), (key, cur) -> { + if (cur == null) + cur = new PaxosState(key, RECENT.remove(key)); + ++cur.active; + return cur; + }); + + try + { + if (!lock.lock(deadline)) + throw throwTimeout(metadata, consistencyForConsensus, isWrite); + return lock; + } + catch (Throwable t) + { + lock.close(); + throw t; + } + } + + private static RequestTimeoutException throwTimeout(TableMetadata metadata, ConsistencyLevel consistencyForConsensus, boolean isWrite) + { + int blockFor = consistencyForConsensus.blockFor(Keyspace.open(metadata.keyspace).getReplicationStrategy()); + throw isWrite + ? new WriteTimeoutException(WriteType.CAS, consistencyForConsensus, 0, blockFor) + : new ReadTimeoutException(consistencyForConsensus, 0, blockFor, false); + } + + private PaxosState maybeLoad() + { + try + { + Snapshot current = this.current; + if (current == null || current instanceof UnsafeSnapshot) + { + synchronized (this) + { + current = this.current; + if (current == null || current instanceof UnsafeSnapshot) + { + Snapshot snapshot = SystemKeyspace.loadPaxosState(key.partitionKey, key.metadata, 0); + currentUpdater.accumulateAndGet(this, snapshot, Snapshot::merge); + } + } + } + } + catch (Throwable t) + { + try { close(); } catch (Throwable t2) { t.addSuppressed(t2); } + throw t; + } + + return this; + } + + private boolean lock(long deadline) + { + try + { + Thread thread = Thread.currentThread(); + if (lockedByUpdater.compareAndSet(this, null, thread)) + return true; + + synchronized (this) + { + waiting++; + + try + { + while (true) + { + if (lockedByUpdater.compareAndSet(this, null, thread)) + return true; + + while (lockedBy != null) + { + long now = nanoTime(); + if (now >= deadline) + return false; + + wait(1 + ((deadline - now) - 1) / 1000000); + } + } + } + finally + { + waiting--; + } + } + } + catch (InterruptedException e) + { + Thread.currentThread().interrupt(); + return false; + } + } + + private void maybeUnlock() + { + // no visibility requirements, as if we hold the lock it was last updated by us + if (lockedBy == null) + return; + + Thread thread = Thread.currentThread(); + + if (lockedBy == thread) + { + lockedBy = null; + if (waiting > 0) + { + synchronized (this) + { + notify(); + } + } + } + } + + public void close() + { + maybeUnlock(); + ACTIVE.compute(key, (key, cur) -> + { + assert cur != null; + if (--cur.active > 0) return cur; - }); - } - void put(DecoratedKey key) + Snapshot stash = cur.current; + if (stash != null && stash.getClass() == Snapshot.class) + RECENT.put(key, stash); + return null; + }); + } + + Snapshot current(Ballot ballot) + { + return current((int)ballot.unix(SECONDS)); + } + + Snapshot current(int nowInSec) + { + // CASSANDRA-12043 is not an issue for v2, as we perform Commit+Prepare and PrepareRefresh + // which are able to make progress whether or not the old commit is shadowed by the TTL (since they + // depend only on the write being successful, not the data being read again later). + // However, we still use nowInSec to guard reads to ensure we do not log any linearizability violations + // due to discrepancies in gc grace handling + + Snapshot current = this.current; + if (current == null || current.getClass() != Snapshot.class) + throw new IllegalStateException(); + return current.removeExpired(nowInSec); + } + + @VisibleForTesting + public Snapshot currentSnapshot() + { + return current; + } + + @VisibleForTesting + public void updateStateUnsafe(Function f) + { + current = f.apply(current); + } + + /** + * Record the requested ballot as promised if it is newer than our current promise; otherwise do nothing. + * @return a PromiseResult containing the before and after state for this operation + */ + public MaybePromise promiseIfNewer(Ballot ballot, boolean isWrite) + { + Snapshot before, after; + while (true) { - LOCKS.compute(key, (ignore, cur) -> --cur.count == 0 ? null : cur); - } - } - - public final Commit promised; - public final Commit accepted; - public final Commit mostRecentCommit; - - public PaxosState(DecoratedKey key, TableMetadata metadata) - { - this(Commit.emptyCommit(key, metadata), Commit.emptyCommit(key, metadata), Commit.emptyCommit(key, metadata)); - } - - public PaxosState(Commit promised, Commit accepted, Commit mostRecentCommit) - { - assert promised.update.partitionKey().equals(accepted.update.partitionKey()) && accepted.update.partitionKey().equals(mostRecentCommit.update.partitionKey()); - assert promised.update.metadata().id.equals(accepted.update.metadata().id) && accepted.update.metadata().id.equals(mostRecentCommit.update.metadata().id); - - this.promised = promised; - this.accepted = accepted; - this.mostRecentCommit = mostRecentCommit; - } - - public static PrepareResponse prepare(Commit toPrepare) - { - long start = nanoTime(); - try - { - Lock lock = Lock.get(toPrepare.update.partitionKey()); - try + Snapshot realBefore = current; + before = realBefore.removeExpired((int)ballot.unix(SECONDS)); + Ballot latestWriteOrLowBound = before.latestWriteOrLowBound(); + Ballot latest = before.latestWitnessedOrLowBound(latestWriteOrLowBound); + if (isAfter(ballot, latest)) { - synchronized (lock) + after = new Snapshot(ballot, isWrite ? ballot : before.promisedWrite, before.accepted, before.committed); + if (currentUpdater.compareAndSet(this, before, after)) { - // When preparing, we need to use the same time as "now" (that's the time we use to decide if something - // is expired or not) accross nodes otherwise we may have a window where a Most Recent Commit shows up - // on some replica and not others during a new proposal (in StorageProxy.beginAndRepairPaxos()), and no - // amount of re-submit will fix this (because the node on which the commit has expired will have a - // tombstone that hides any re-submit). See CASSANDRA-12043 for details. - int nowInSec = (int) toPrepare.ballot.unix(SECONDS); - PaxosState state = SystemKeyspace.loadPaxosState(toPrepare.update.partitionKey(), toPrepare.update.metadata(), nowInSec); - if (toPrepare.isAfter(state.promised)) + // It doesn't matter if a later operation witnesses this before it's persisted, + // as it can only lead to rejecting a promise which leaves no persistent state + // (and it's anyway safe to arbitrarily reject promises) + if (isWrite) { - Tracing.trace("Promising ballot {}", toPrepare.ballot); - SystemKeyspace.savePaxosPromise(toPrepare); - return new PrepareResponse(true, state.accepted, state.mostRecentCommit); + Tracing.trace("Promising read/write ballot {}", ballot); + SystemKeyspace.savePaxosWritePromise(key.partitionKey, key.metadata, ballot); } else { - Tracing.trace("Promise rejected; {} is not sufficiently newer than {}", toPrepare, state.promised); - // return the currently promised ballot (not the last accepted one) so the coordinator can make sure it uses newer ballot next time (#5667) - return new PrepareResponse(false, state.promised, state.mostRecentCommit); + Tracing.trace("Promising read ballot {}", ballot); + SystemKeyspace.savePaxosReadPromise(key.partitionKey, key.metadata, ballot); } + return MaybePromise.promise(before, after); } } - finally + else if (isAfter(ballot, latestWriteOrLowBound)) { - lock.put(toPrepare.update.partitionKey()); + Tracing.trace("Permitting only read by ballot {}", ballot); + return MaybePromise.permitRead(before, latest); } - } - finally - { - Keyspace.open(toPrepare.update.metadata().keyspace).getColumnFamilyStore(toPrepare.update.metadata().id).metric.casPrepare.addNano(nanoTime() - start); + else + { + Tracing.trace("Promise rejected; {} older than {}", ballot, latest); + return MaybePromise.reject(before, latest); + } + + Snapshot realAfter = new Snapshot(ballot, isWrite ? ballot : realBefore.promisedWrite, realBefore.accepted, realBefore.committed); + after = new Snapshot(ballot, realAfter.promisedWrite, before.accepted, before.committed); + if (currentUpdater.compareAndSet(this, realBefore, realAfter)) + break; } + // It doesn't matter if a later operation witnesses this before it's persisted, + // as it can only lead to rejecting a promise which leaves no persistent state + // (and it's anyway safe to arbitrarily reject promises) + Tracing.trace("Promising ballot {}", ballot); + if (isWrite) SystemKeyspace.savePaxosWritePromise(key.partitionKey, key.metadata, ballot); + else SystemKeyspace.savePaxosReadPromise(key.partitionKey, key.metadata, ballot); + return MaybePromise.promise(before, after); } - public static Boolean propose(Commit proposal) + /** + * Record an acceptance of the proposal if there is no newer promise; otherwise inform the caller of the newer ballot + */ + public Ballot acceptIfLatest(Proposal proposal) { + if (paxosStatePurging() == legacy && !(proposal instanceof AcceptedWithTTL)) + proposal = AcceptedWithTTL.withDefaultTTL(proposal); + + // state.promised can be null, because it is invalidated by committed; + // we may also have accepted a newer proposal than we promised, so we confirm that we are the absolute newest + // (or that we have the exact same ballot as our promise, which is the typical case) + Snapshot before, after; + while (true) + { + Snapshot realBefore = current; + before = realBefore.removeExpired((int)proposal.ballot.unix(SECONDS)); + Ballot latest = before.latestWitnessedOrLowBound(); + if (!proposal.isSameOrAfter(latest)) + { + Tracing.trace("Rejecting proposal {}; latest is now {}", proposal.ballot, latest); + return latest; + } + + if (proposal.hasSameBallot(before.committed)) // TODO: consider not answering + return null; // no need to save anything, or indeed answer at all + + after = new Snapshot(realBefore.promised, realBefore.promisedWrite, proposal.accepted(), realBefore.committed); + if (currentUpdater.compareAndSet(this, realBefore, after)) + break; + } + + // It is more worrisome to permit witnessing an accepted proposal before we have persisted it + // because this has more tangible effects on the recipient, but again it is safe: either it is + // - witnessed to reject (which is always safe, as it prevents rather than creates an outcome); or + // - witnessed as an in progress proposal + // in the latter case, for there to be any effect on the state the proposal must be re-proposed, or not, + // on its own terms, and must + // be persisted by the re-proposer, and so it remains a non-issue + // though this + Tracing.trace("Accepting proposal {}", proposal); + SystemKeyspace.savePaxosProposal(proposal); + return null; + } + + public void commit(Agreed commit) + { + applyCommit(commit, this, (apply, to) -> + currentUpdater.accumulateAndGet(to, new UnsafeSnapshot(apply), Snapshot::merge) + ); + } + + public static void commitDirect(Commit commit) + { + applyCommit(commit, null, (apply, ignore) -> { + try (PaxosState state = tryGetUnsafe(apply.update.partitionKey(), apply.update.metadata())) + { + if (state != null) + currentUpdater.accumulateAndGet(state, new UnsafeSnapshot(apply), Snapshot::merge); + } + }); + } + + private static void applyCommit(Commit commit, PaxosState state, BiConsumer postCommit) + { + if (paxosStatePurging() == legacy && !(commit instanceof CommittedWithTTL)) + commit = CommittedWithTTL.withDefaultTTL(commit); + long start = nanoTime(); try { - Lock lock = Lock.get(proposal.update.partitionKey()); - try - { - synchronized (lock) - { - - int nowInSec = (int) proposal.ballot.unix(SECONDS); - PaxosState state = SystemKeyspace.loadPaxosState(proposal.update.partitionKey(), proposal.update.metadata(), nowInSec); - if (proposal.hasBallot(state.promised.ballot) || proposal.isAfter(state.promised)) - { - Tracing.trace("Accepting proposal {}", proposal); - SystemKeyspace.savePaxosProposal(proposal); - return true; - } - else - { - Tracing.trace("Rejecting proposal for {} because inProgress is now {}", proposal, state.promised); - return false; - } - } - } - finally - { - lock.put(proposal.update.partitionKey()); - } - } - finally - { - Keyspace.open(proposal.update.metadata().keyspace).getColumnFamilyStore(proposal.update.metadata().id).metric.casPropose.addNano(nanoTime() - start); - } - } - - public static void commit(Commit proposal) - { - long start = nanoTime(); - try - { - // There is no guarantee we will see commits in the right order, because messages - // can get delayed, so a proposal can be older than our current most recent ballot/commit. - // Committing it is however always safe due to column timestamps, so always do it. However, - // if our current in-progress ballot is strictly greater than the proposal one, we shouldn't - // erase the in-progress update. + // TODO: run Paxos Repair before truncate so we can excise this // The table may have been truncated since the proposal was initiated. In that case, we // don't want to perform the mutation and potentially resurrect truncated data - if (proposal.ballot.unixMicros() >= SystemKeyspace.getTruncatedAt(proposal.update.metadata().id)) + if (commit.ballot.unixMicros() >= SystemKeyspace.getTruncatedAt(commit.update.metadata().id)) { - Tracing.trace("Committing proposal {}", proposal); - Mutation mutation = proposal.makeMutation(); + Tracing.trace("Committing proposal {}", commit); + Mutation mutation = commit.makeMutation(); Keyspace.open(mutation.getKeyspaceName()).apply(mutation, true); } else { - Tracing.trace("Not committing proposal {} as ballot timestamp predates last truncation time", proposal); + Tracing.trace("Not committing proposal {} as ballot timestamp predates last truncation time", commit); } - // We don't need to lock, we're just blindly updating - SystemKeyspace.savePaxosCommit(proposal); + + // for commits we save to disk first, because we can; even here though it is safe to permit later events to + // witness the state before it is persisted. The only tricky situation is that we use the witnessing of + // a quorum of nodes having witnessed the latest commit to decide if we need to disseminate a commit + // again before proceeding with any new operation, but in this case we have already persisted the relevant + // information, namely the base table mutation. So this fact is persistent, even if knowldge of this fact + // is not (and if this is lost, it may only lead to a future operation unnecessarily committing again) + SystemKeyspace.savePaxosCommit(commit); + postCommit.accept(commit, state); } finally { - Keyspace.open(proposal.update.metadata().keyspace).getColumnFamilyStore(proposal.update.metadata().id).metric.casCommit.addNano(nanoTime() - start); + Keyspace.openAndGetStore(commit.update.metadata()).metric.casCommit.addNano(nanoTime() - start); } } + + public static PrepareResponse legacyPrepare(Commit toPrepare) + { + long start = nanoTime(); + try (PaxosState unsafeState = getUnsafe(toPrepare)) + { + synchronized (unsafeState.key) + { + unsafeState.maybeLoad(); + assert unsafeState.current != null; + + while (true) + { + // ignore nowInSec when merging as this can only be an issue during the transition period, so the unbounded + // problem of CASSANDRA-12043 is not an issue + Snapshot realBefore = unsafeState.current; + Snapshot before = realBefore.removeExpired((int)toPrepare.ballot.unix(SECONDS)); + Ballot latest = before.latestWitnessedOrLowBound(); + if (toPrepare.isAfter(latest)) + { + Snapshot after = new Snapshot(toPrepare.ballot, toPrepare.ballot, realBefore.accepted, realBefore.committed); + if (currentUpdater.compareAndSet(unsafeState, realBefore, after)) + { + Tracing.trace("Promising ballot {}", toPrepare.ballot); + DecoratedKey partitionKey = toPrepare.update.partitionKey(); + TableMetadata metadata = toPrepare.update.metadata(); + SystemKeyspace.savePaxosWritePromise(partitionKey, metadata, toPrepare.ballot); + return new PrepareResponse(true, before.accepted == null ? Accepted.none(partitionKey, metadata) : before.accepted, before.committed); + } + } + else + { + Tracing.trace("Promise rejected; {} is not sufficiently newer than {}", toPrepare, before.promised); + // return the currently promised ballot (not the last accepted one) so the coordinator can make sure it uses newer ballot next time (#5667) + return new PrepareResponse(false, new Commit(before.promised, toPrepare.update), before.committed); + } + } + } + } + finally + { + Keyspace.openAndGetStore(toPrepare.update.metadata()).metric.casPrepare.addNano(nanoTime() - start); + } + } + + public static Boolean legacyPropose(Commit proposal) + { + if (paxosStatePurging() == legacy && !(proposal instanceof AcceptedWithTTL)) + proposal = AcceptedWithTTL.withDefaultTTL(proposal); + + long start = nanoTime(); + try (PaxosState unsafeState = getUnsafe(proposal)) + { + synchronized (unsafeState.key) + { + unsafeState.maybeLoad(); + assert unsafeState.current != null; + + while (true) + { + Snapshot realBefore = unsafeState.current; + Snapshot before = realBefore.removeExpired((int)proposal.ballot.unix(SECONDS)); + boolean accept = proposal.isSameOrAfter(before.latestWitnessedOrLowBound()); + if (accept) + { + if (proposal.hasSameBallot(before.committed) || + currentUpdater.compareAndSet(unsafeState, realBefore, + new Snapshot(realBefore.promised, realBefore.promisedWrite, + new Accepted(proposal), realBefore.committed))) + { + Tracing.trace("Accepting proposal {}", proposal); + SystemKeyspace.savePaxosProposal(proposal); + return true; + } + } + else + { + Tracing.trace("Rejecting proposal for {} because inProgress is now {}", proposal, before.promised); + return false; + } + } + } + } + finally + { + Keyspace.openAndGetStore(proposal.update.metadata()).metric.casPropose.addNano(nanoTime() - start); + } + } + + public static void unsafeReset() + { + ACTIVE.clear(); + RECENT.clear(); + ballotTracker().truncate(); + } + + @SuppressWarnings("resource") + public static Snapshot unsafeGetIfPresent(DecoratedKey partitionKey, TableMetadata metadata) + { + Key key = new Key(partitionKey, metadata); + PaxosState cur = ACTIVE.get(key); + if (cur != null) return cur.current; + return RECENT.get(key); + } } diff --git a/src/java/org/apache/cassandra/service/paxos/TablePaxosRepairHistory.java b/src/java/org/apache/cassandra/service/paxos/TablePaxosRepairHistory.java new file mode 100644 index 0000000000..ad4836676a --- /dev/null +++ b/src/java/org/apache/cassandra/service/paxos/TablePaxosRepairHistory.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.paxos; + +import java.util.Collection; + +import org.apache.cassandra.db.SystemKeyspace; +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.Token; +import org.apache.cassandra.service.paxos.Ballot; + +public class TablePaxosRepairHistory +{ + private final String keyspace; + private final String table; + private volatile PaxosRepairHistory history; + + private TablePaxosRepairHistory(String keyspace, String table, PaxosRepairHistory history) + { + this.keyspace = keyspace; + this.table = table; + this.history = history; + } + + public static TablePaxosRepairHistory load(String keyspace, String table) + { + return new TablePaxosRepairHistory(keyspace, table, SystemKeyspace.loadPaxosRepairHistory(keyspace, table)); + } + + public Ballot getBallotForToken(Token token) + { + return history.ballotForToken(token); + } + + private void updatePaxosRepairTable(PaxosRepairHistory update, boolean flush) + { + SystemKeyspace.savePaxosRepairHistory(keyspace, table, update, flush); + } + + public synchronized void add(Collection> ranges, Ballot ballot, boolean flush) + { + PaxosRepairHistory update = PaxosRepairHistory.add(history, ranges, ballot); + updatePaxosRepairTable(update, flush); + history = update; + } + + public synchronized void merge(PaxosRepairHistory toMerge, boolean flush) + { + PaxosRepairHistory update = PaxosRepairHistory.merge(history, toMerge); + if (!update.equals(history)) + updatePaxosRepairTable(update, flush); + history = update; + } + + public PaxosRepairHistory getHistory() + { + return history; + } + + public PaxosRepairHistory getHistoryForRanges(Collection> ranges) + { + return PaxosRepairHistory.trim(history, ranges); + } +} diff --git a/src/java/org/apache/cassandra/service/paxos/cleanup/PaxosCleanup.java b/src/java/org/apache/cassandra/service/paxos/cleanup/PaxosCleanup.java new file mode 100644 index 0000000000..6eb1ebd574 --- /dev/null +++ b/src/java/org/apache/cassandra/service/paxos/cleanup/PaxosCleanup.java @@ -0,0 +1,156 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.paxos.cleanup; + +import java.util.Collection; +import java.util.List; +import java.util.concurrent.Executor; +import java.util.function.Consumer; + +import com.google.common.base.Preconditions; +import com.google.common.collect.Iterables; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.concurrent.ScheduledExecutors; +import org.apache.cassandra.db.Keyspace; +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.Token; +import org.apache.cassandra.gms.EndpointState; +import org.apache.cassandra.gms.Gossiper; +import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.locator.RangesAtEndpoint; +import org.apache.cassandra.schema.Schema; +import org.apache.cassandra.schema.TableId; +import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.service.StorageService; +import org.apache.cassandra.service.paxos.Ballot; +import org.apache.cassandra.utils.FBUtilities; +import org.apache.cassandra.utils.concurrent.AsyncFuture; +import org.apache.cassandra.utils.concurrent.Future; + +import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static org.apache.cassandra.config.DatabaseDescriptor.getCasContentionTimeout; +import static org.apache.cassandra.config.DatabaseDescriptor.getWriteRpcTimeout; +import static org.apache.cassandra.utils.FBUtilities.getBroadcastAddressAndPort; + +public class PaxosCleanup extends AsyncFuture implements Runnable +{ + private static final Logger logger = LoggerFactory.getLogger(PaxosCleanup.class); + + private final Collection endpoints; + private final TableMetadata table; + private final Collection> ranges; + private final boolean skippedReplicas; + private final Executor executor; + + // references kept for debugging + private PaxosStartPrepareCleanup startPrepare; + private PaxosFinishPrepareCleanup finishPrepare; + private PaxosCleanupSession session; + private PaxosCleanupComplete complete; + + public PaxosCleanup(Collection endpoints, TableMetadata table, Collection> ranges, boolean skippedReplicas, Executor executor) + { + this.endpoints = endpoints; + this.table = table; + this.ranges = ranges; + this.skippedReplicas = skippedReplicas; + this.executor = executor; + } + + private void addCallback(Future future, Consumer onComplete) + { + future.addCallback(onComplete, this::tryFailure); + } + + public static PaxosCleanup cleanup(Collection endpoints, TableMetadata table, Collection> ranges, boolean skippedReplicas, Executor executor) + { + PaxosCleanup cleanup = new PaxosCleanup(endpoints, table, ranges, skippedReplicas, executor); + executor.execute(cleanup); + return cleanup; + } + + public void run() + { + EndpointState localEpState = Gossiper.instance.getEndpointStateForEndpoint(getBroadcastAddressAndPort()); + startPrepare = PaxosStartPrepareCleanup.prepare(table.id, endpoints, localEpState, ranges); + addCallback(startPrepare, this::finishPrepare); + } + + private void finishPrepare(PaxosCleanupHistory result) + { + ScheduledExecutors.nonPeriodicTasks.schedule(() -> { + finishPrepare = PaxosFinishPrepareCleanup.finish(endpoints, result); + addCallback(finishPrepare, (v) -> startSession(result.highBound)); + }, Math.min(getCasContentionTimeout(MILLISECONDS), getWriteRpcTimeout(MILLISECONDS)), MILLISECONDS); + } + + private void startSession(Ballot lowBound) + { + session = new PaxosCleanupSession(endpoints, table.id, ranges); + addCallback(session, (v) -> finish(lowBound)); + executor.execute(session); + } + + private void finish(Ballot lowBound) + { + complete = new PaxosCleanupComplete(endpoints, table.id, ranges, lowBound, skippedReplicas); + addCallback(complete, this::trySuccess); + executor.execute(complete); + } + + private static boolean isOutOfRange(String ksName, Collection> repairRanges) + { + Keyspace keyspace = Keyspace.open(ksName); + List> localRanges = Range.normalize(keyspace.getReplicationStrategy() + .getAddressReplicas() + .get(FBUtilities.getBroadcastAddressAndPort()) + .ranges()); + + RangesAtEndpoint pendingRanges = StorageService.instance.getTokenMetadata().getPendingRanges(ksName, FBUtilities.getBroadcastAddressAndPort()); + if (!pendingRanges.isEmpty()) + { + localRanges.addAll(pendingRanges.ranges()); + localRanges = Range.normalize(localRanges); + } + + for (Range repairRange : Range.normalize(repairRanges)) + { + if (!Iterables.any(localRanges, localRange -> localRange.contains(repairRange))) + return true; + } + return false; + } + + static boolean isInRangeAndShouldProcess(Collection> ranges, TableId tableId) + { + TableMetadata metadata = Schema.instance.getTableMetadata(tableId); + + Keyspace keyspace = Keyspace.open(metadata.keyspace); + Preconditions.checkNotNull(keyspace); + + if (!isOutOfRange(metadata.keyspace, ranges)) + return true; + + logger.warn("Out of range PaxosCleanup request for {}: {}", metadata, ranges); + return false; + } +} diff --git a/src/java/org/apache/cassandra/service/paxos/cleanup/PaxosCleanupComplete.java b/src/java/org/apache/cassandra/service/paxos/cleanup/PaxosCleanupComplete.java new file mode 100644 index 0000000000..0196e9cce0 --- /dev/null +++ b/src/java/org/apache/cassandra/service/paxos/cleanup/PaxosCleanupComplete.java @@ -0,0 +1,144 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.paxos.cleanup; + +import java.io.IOException; +import java.util.*; + +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.TypeSizes; +import org.apache.cassandra.dht.AbstractBounds; +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.Token; +import org.apache.cassandra.exceptions.RequestFailureReason; +import org.apache.cassandra.io.IVersionedSerializer; +import org.apache.cassandra.io.util.DataInputPlus; +import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.net.*; +import org.apache.cassandra.schema.Schema; +import org.apache.cassandra.schema.TableId; +import org.apache.cassandra.service.paxos.Ballot; +import org.apache.cassandra.utils.concurrent.AsyncFuture; + +import static org.apache.cassandra.config.DatabaseDescriptor.getPartitioner; +import static org.apache.cassandra.net.NoPayload.noPayload; +import static org.apache.cassandra.net.Verb.PAXOS2_CLEANUP_COMPLETE_REQ; + +public class PaxosCleanupComplete extends AsyncFuture implements RequestCallbackWithFailure, Runnable +{ + private final Set waitingResponse; + final TableId tableId; + final Collection> ranges; + final Ballot lowBound; + final boolean skippedReplicas; + + PaxosCleanupComplete(Collection endpoints, TableId tableId, Collection> ranges, Ballot lowBound, boolean skippedReplicas) + { + this.waitingResponse = new HashSet<>(endpoints); + this.tableId = tableId; + this.ranges = ranges; + this.lowBound = lowBound; + this.skippedReplicas = skippedReplicas; + } + + public synchronized void run() + { + Request request = !skippedReplicas ? new Request(tableId, lowBound, ranges) + : new Request(tableId, Ballot.none(), Collections.emptyList()); + Message message = Message.out(PAXOS2_CLEANUP_COMPLETE_REQ, request); + for (InetAddressAndPort endpoint : waitingResponse) + MessagingService.instance().sendWithCallback(message, endpoint, this); + } + + @Override + public void onFailure(InetAddressAndPort from, RequestFailureReason reason) + { + tryFailure(new PaxosCleanupException("Timed out waiting on response from " + from)); + } + + @Override + public synchronized void onResponse(Message msg) + { + if (isDone()) + return; + + if (!waitingResponse.remove(msg.from())) + throw new IllegalArgumentException("Received unexpected response from " + msg.from()); + + if (waitingResponse.isEmpty()) + trySuccess(null); + } + + static class Request + { + final TableId tableId; + final Ballot lowBound; + final Collection> ranges; + + Request(TableId tableId, Ballot lowBound, Collection> ranges) + { + this.tableId = tableId; + this.ranges = ranges; + this.lowBound = lowBound; + } + } + + public static final IVersionedSerializer serializer = new IVersionedSerializer() + { + public void serialize(Request request, DataOutputPlus out, int version) throws IOException + { + request.tableId.serialize(out); + request.lowBound.serialize(out); + out.writeInt(request.ranges.size()); + for (Range rt : request.ranges) + AbstractBounds.tokenSerializer.serialize(rt, out, version); + } + + public Request deserialize(DataInputPlus in, int version) throws IOException + { + TableId tableId = TableId.deserialize(in); + Ballot lowBound = Ballot.deserialize(in); + int numRanges = in.readInt(); + List> ranges = new ArrayList<>(); + for (int i = 0; i < numRanges; i++) + { + Range range = (Range) AbstractBounds.tokenSerializer.deserialize(in, getPartitioner(), version); + ranges.add(range); + } + return new Request(tableId, lowBound, ranges); + } + + public long serializedSize(Request request, int version) + { + long size = request.tableId.serializedSize(); + size += Ballot.sizeInBytes(); + size += TypeSizes.sizeof(request.ranges.size()); + for (Range range : request.ranges) + size += AbstractBounds.tokenSerializer.serializedSize(range, version); + return size; + } + }; + + public static final IVerbHandler verbHandler = (in) -> { + ColumnFamilyStore cfs = Schema.instance.getColumnFamilyStoreInstance(in.payload.tableId); + cfs.onPaxosRepairComplete(in.payload.ranges, in.payload.lowBound); + MessagingService.instance().respond(noPayload, in); + }; +} diff --git a/src/java/org/apache/cassandra/service/paxos/cleanup/PaxosCleanupException.java b/src/java/org/apache/cassandra/service/paxos/cleanup/PaxosCleanupException.java new file mode 100644 index 0000000000..9eb8fbe5b4 --- /dev/null +++ b/src/java/org/apache/cassandra/service/paxos/cleanup/PaxosCleanupException.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.paxos.cleanup; + +class PaxosCleanupException extends RuntimeException +{ + PaxosCleanupException(String message) + { + super(message); + } +} diff --git a/src/java/org/apache/cassandra/service/paxos/cleanup/PaxosCleanupHistory.java b/src/java/org/apache/cassandra/service/paxos/cleanup/PaxosCleanupHistory.java new file mode 100644 index 0000000000..70b4099eb7 --- /dev/null +++ b/src/java/org/apache/cassandra/service/paxos/cleanup/PaxosCleanupHistory.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.paxos.cleanup; + +import java.io.IOException; + +import org.apache.cassandra.io.IVersionedSerializer; +import org.apache.cassandra.io.util.DataInputPlus; +import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.schema.TableId; +import org.apache.cassandra.service.paxos.Ballot; +import org.apache.cassandra.service.paxos.PaxosRepairHistory; + +public class PaxosCleanupHistory +{ + final TableId tableId; + final Ballot highBound; + final PaxosRepairHistory history; + + public PaxosCleanupHistory(TableId tableId, Ballot highBound, PaxosRepairHistory history) + { + this.tableId = tableId; + this.highBound = highBound; + this.history = history; + } + + public static final IVersionedSerializer serializer = new IVersionedSerializer() + { + public void serialize(PaxosCleanupHistory message, DataOutputPlus out, int version) throws IOException + { + message.tableId.serialize(out); + message.highBound.serialize(out); + PaxosRepairHistory.serializer.serialize(message.history, out, version); + } + + public PaxosCleanupHistory deserialize(DataInputPlus in, int version) throws IOException + { + TableId tableId = TableId.deserialize(in); + Ballot lowBound = Ballot.deserialize(in); + PaxosRepairHistory history = PaxosRepairHistory.serializer.deserialize(in, version); + return new PaxosCleanupHistory(tableId, lowBound, history); + } + + public long serializedSize(PaxosCleanupHistory message, int version) + { + long size = message.tableId.serializedSize(); + size += Ballot.sizeInBytes(); + size += PaxosRepairHistory.serializer.serializedSize(message.history, version); + return size; + } + }; +} diff --git a/src/java/org/apache/cassandra/service/paxos/cleanup/PaxosCleanupLocalCoordinator.java b/src/java/org/apache/cassandra/service/paxos/cleanup/PaxosCleanupLocalCoordinator.java new file mode 100644 index 0000000000..0cd95c85dd --- /dev/null +++ b/src/java/org/apache/cassandra/service/paxos/cleanup/PaxosCleanupLocalCoordinator.java @@ -0,0 +1,189 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.paxos.cleanup; + +import java.util.Collection; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; + +import com.google.common.base.Preconditions; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.db.ConsistencyLevel; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.Token; +import org.apache.cassandra.schema.Schema; +import org.apache.cassandra.schema.TableId; +import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.service.paxos.AbstractPaxosRepair; +import org.apache.cassandra.service.paxos.PaxosRepair; +import org.apache.cassandra.service.paxos.PaxosState; +import org.apache.cassandra.service.paxos.uncommitted.UncommittedPaxosKey; +import org.apache.cassandra.utils.CloseableIterator; +import org.apache.cassandra.utils.FBUtilities; +import org.apache.cassandra.utils.concurrent.AsyncFuture; + +import static org.apache.cassandra.service.paxos.cleanup.PaxosCleanupSession.TIMEOUT_NANOS; +import static org.apache.cassandra.utils.Clock.Global.nanoTime; + +public class PaxosCleanupLocalCoordinator extends AsyncFuture +{ + private static final Logger logger = LoggerFactory.getLogger(PaxosCleanupLocalCoordinator.class); + private static final UUID INTERNAL_SESSION = new UUID(0, 0); + + private final UUID session; + private final TableId tableId; + private final TableMetadata table; + private final Collection> ranges; + private final CloseableIterator uncommittedIter; + private int count = 0; + private final long deadline; + + private final Map inflight = new ConcurrentHashMap<>(); + private final PaxosTableRepairs tableRepairs; + + private PaxosCleanupLocalCoordinator(UUID session, TableId tableId, Collection> ranges, CloseableIterator uncommittedIter) + { + this.session = session; + this.tableId = tableId; + this.table = Schema.instance.getTableMetadata(tableId); + this.ranges = ranges; + this.uncommittedIter = uncommittedIter; + this.tableRepairs = PaxosTableRepairs.getForTable(tableId); + this.deadline = TIMEOUT_NANOS + nanoTime(); + } + + public synchronized void start() + { + if (table == null) + { + fail("Unknown tableId: " + tableId); + return; + } + + if (!PaxosRepair.validatePeerCompatibility(table, ranges)) + { + fail("Unsupported peer versions for " + tableId + ' ' + ranges.toString()); + return; + } + + logger.info("Completing uncommitted paxos instances for {} on ranges {} for session {}", table, ranges, session); + + scheduleKeyRepairsOrFinish(); + } + + @SuppressWarnings("resource") + public static PaxosCleanupLocalCoordinator create(PaxosCleanupRequest request) + { + CloseableIterator iterator = PaxosState.uncommittedTracker().uncommittedKeyIterator(request.tableId, request.ranges); + return new PaxosCleanupLocalCoordinator(request.session, request.tableId, request.ranges, iterator); + } + + @SuppressWarnings("resource") + public static PaxosCleanupLocalCoordinator createForAutoRepair(TableId tableId, Collection> ranges) + { + CloseableIterator iterator = PaxosState.uncommittedTracker().uncommittedKeyIterator(tableId, ranges); + return new PaxosCleanupLocalCoordinator(INTERNAL_SESSION, tableId, ranges, iterator); + } + + /** + * Schedule as many key repairs as we can, up to the paralellism limit. If no repairs are scheduled and + * none are in flight when the iterator is exhausted, the session will be finished + */ + private void scheduleKeyRepairsOrFinish() + { + int parallelism = DatabaseDescriptor.getPaxosRepairParallelism(); + Preconditions.checkArgument(parallelism > 0); + if (inflight.size() < parallelism) + { + if (nanoTime() - deadline >= 0) + { + fail("timeout"); + return; + } + + while (inflight.size() < parallelism && uncommittedIter.hasNext()) + repairKey(uncommittedIter.next()); + + } + + if (inflight.isEmpty()) + finish(); + } + + private boolean repairKey(UncommittedPaxosKey uncommitted) + { + logger.trace("repairing {}", uncommitted); + Preconditions.checkState(!inflight.containsKey(uncommitted.getKey())); + ConsistencyLevel consistency = uncommitted.getConsistencyLevel(); + + // we don't know the consistency of this operation, presumably because it originated + // before we started tracking paxos cl, so we don't attempt to repair it + if (consistency == null) + return false; + + inflight.put(uncommitted.getKey(), tableRepairs.startOrGetOrQueue(uncommitted.getKey(), uncommitted.ballot(), uncommitted.getConsistencyLevel(), table, result -> { + if (result.wasSuccessful()) + onKeyFinish(uncommitted.getKey()); + else + onKeyFailure(result.toString()); + })); + return true; + } + + private synchronized void onKeyFinish(DecoratedKey key) + { + if (!inflight.containsKey(key)) + return; + logger.trace("finished repairing {}", key); + inflight.remove(key); + count++; + + scheduleKeyRepairsOrFinish(); + } + + private void complete(PaxosCleanupResponse response) + { + uncommittedIter.close(); + trySuccess(response); + } + + private void onKeyFailure(String reason) + { + // not synchronized to avoid deadlock with callback we register on start + inflight.values().forEach(AbstractPaxosRepair::cancel); + fail(reason); + } + + private synchronized void fail(String reason) + { + logger.info("Failing paxos cleanup session {} for {} on ranges {}. Reason: {}", session, table, ranges, reason); + complete(PaxosCleanupResponse.failed(session, reason)); + } + + private void finish() + { + logger.info("Completed {} uncommitted paxos instances for {} on ranges {} for session {}", count, table, ranges, session); + complete(PaxosCleanupResponse.success(session)); + } +} diff --git a/src/java/org/apache/cassandra/service/paxos/cleanup/PaxosCleanupRequest.java b/src/java/org/apache/cassandra/service/paxos/cleanup/PaxosCleanupRequest.java new file mode 100644 index 0000000000..3e081c3196 --- /dev/null +++ b/src/java/org/apache/cassandra/service/paxos/cleanup/PaxosCleanupRequest.java @@ -0,0 +1,142 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.paxos.cleanup; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.UUID; +import javax.annotation.Nullable; + +import com.google.common.util.concurrent.FutureCallback; + +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.db.TypeSizes; +import org.apache.cassandra.dht.AbstractBounds; +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.Token; +import org.apache.cassandra.io.IVersionedSerializer; +import org.apache.cassandra.io.util.DataInputPlus; +import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.net.IVerbHandler; +import org.apache.cassandra.net.Message; +import org.apache.cassandra.schema.TableId; +import org.apache.cassandra.utils.UUIDSerializer; + +import static org.apache.cassandra.net.MessagingService.instance; +import static org.apache.cassandra.net.NoPayload.noPayload; +import static org.apache.cassandra.net.Verb.PAXOS2_CLEANUP_RSP; +import static org.apache.cassandra.net.Verb.PAXOS2_CLEANUP_RSP2; + +// TODO: send the high bound as a minimum commit point, so later repairs can terminate early if a later commit has been witnessed +public class PaxosCleanupRequest +{ + public final UUID session; + public final TableId tableId; + public final Collection> ranges; + + static Collection> rangesOrMin(Collection> ranges) + { + if (ranges != null && !ranges.isEmpty()) + return ranges; + + Token min = DatabaseDescriptor.getPartitioner().getMinimumToken(); + return Collections.singleton(new Range<>(min, min)); + } + + public PaxosCleanupRequest(UUID session, TableId tableId, Collection> ranges) + { + this.session = session; + this.tableId = tableId; + this.ranges = rangesOrMin(ranges); + } + + public static final IVerbHandler verbHandler = in -> { + PaxosCleanupRequest request = in.payload; + + if (!PaxosCleanup.isInRangeAndShouldProcess(request.ranges, request.tableId)) + { + String msg = String.format("Rejecting cleanup request %s from %s. Some ranges are not replicated (%s)", + request.session, in.from(), request.ranges); + Message response = Message.out(PAXOS2_CLEANUP_RSP2, PaxosCleanupResponse.failed(request.session, msg)); + instance().send(response, in.respondTo()); + return; + } + + PaxosCleanupLocalCoordinator coordinator = PaxosCleanupLocalCoordinator.create(request); + + coordinator.addCallback(new FutureCallback() + { + public void onSuccess(@Nullable PaxosCleanupResponse finished) + { + Message response = Message.out(PAXOS2_CLEANUP_RSP2, coordinator.getNow()); + instance().send(response, in.respondTo()); + } + + public void onFailure(Throwable throwable) + { + Message response = Message.out(PAXOS2_CLEANUP_RSP2, PaxosCleanupResponse.failed(request.session, throwable.getMessage())); + instance().send(response, in.respondTo()); + } + }); + + // ack the request so the coordinator knows we've started + instance().respond(noPayload, in); + + coordinator.start(); + }; + + public static final IVersionedSerializer serializer = new IVersionedSerializer() + { + public void serialize(PaxosCleanupRequest completer, DataOutputPlus out, int version) throws IOException + { + UUIDSerializer.serializer.serialize(completer.session, out, version); + completer.tableId.serialize(out); + out.writeInt(completer.ranges.size()); + for (Range range: completer.ranges) + AbstractBounds.tokenSerializer.serialize(range, out, version); + } + + public PaxosCleanupRequest deserialize(DataInputPlus in, int version) throws IOException + { + UUID session = UUIDSerializer.serializer.deserialize(in, version); + TableId tableId = TableId.deserialize(in); + + int numRanges = in.readInt(); + List> ranges = new ArrayList<>(numRanges); + for (int i=0; i) AbstractBounds.tokenSerializer.deserialize(in, DatabaseDescriptor.getPartitioner(), version)); + } + return new PaxosCleanupRequest(session, tableId, ranges); + } + + public long serializedSize(PaxosCleanupRequest completer, int version) + { + long size = UUIDSerializer.serializer.serializedSize(completer.session, version); + size += completer.tableId.serializedSize(); + size += TypeSizes.sizeof(completer.ranges.size()); + for (Range range: completer.ranges) + size += AbstractBounds.tokenSerializer.serializedSize(range, version); + return size; + } + }; +} diff --git a/src/java/org/apache/cassandra/service/paxos/cleanup/PaxosCleanupResponse.java b/src/java/org/apache/cassandra/service/paxos/cleanup/PaxosCleanupResponse.java new file mode 100644 index 0000000000..1c90162001 --- /dev/null +++ b/src/java/org/apache/cassandra/service/paxos/cleanup/PaxosCleanupResponse.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.paxos.cleanup; + +import java.io.IOException; +import java.util.UUID; + +import javax.annotation.Nullable; + +import org.apache.cassandra.db.TypeSizes; +import org.apache.cassandra.io.IVersionedSerializer; +import org.apache.cassandra.io.util.DataInputPlus; +import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.net.IVerbHandler; +import org.apache.cassandra.utils.UUIDSerializer; + +public class PaxosCleanupResponse +{ + public final UUID session; + public final boolean wasSuccessful; + public final String message; + + public PaxosCleanupResponse(UUID session, boolean wasSuccessful, @Nullable String message) + { + this.session = session; + this.wasSuccessful = wasSuccessful; + this.message = message; + } + + public static PaxosCleanupResponse success(UUID session) + { + return new PaxosCleanupResponse(session, true, null); + } + + public static PaxosCleanupResponse failed(UUID session, String message) + { + return new PaxosCleanupResponse(session, false, message); + } + + public static final IVerbHandler verbHandler = (message) -> PaxosCleanupSession.finishSession(message.from(), message.payload); + + public static final IVersionedSerializer serializer = new IVersionedSerializer() + { + public void serialize(PaxosCleanupResponse finished, DataOutputPlus out, int version) throws IOException + { + UUIDSerializer.serializer.serialize(finished.session, out, version); + out.writeBoolean(finished.wasSuccessful); + out.writeBoolean(finished.message != null); + if (finished.message != null) + out.writeUTF(finished.message); + } + + public PaxosCleanupResponse deserialize(DataInputPlus in, int version) throws IOException + { + UUID session = UUIDSerializer.serializer.deserialize(in, version); + boolean success = in.readBoolean(); + String message = in.readBoolean() ? in.readUTF() : null; + return new PaxosCleanupResponse(session, success, message); + } + + public long serializedSize(PaxosCleanupResponse finished, int version) + { + long size = UUIDSerializer.serializer.serializedSize(finished.session, version); + size += TypeSizes.sizeof(finished.wasSuccessful); + size += TypeSizes.sizeof(finished.message != null); + if (finished.message != null) + size += TypeSizes.sizeof(finished.message); + return size; + } + }; +} diff --git a/src/java/org/apache/cassandra/service/paxos/cleanup/PaxosCleanupSession.java b/src/java/org/apache/cassandra/service/paxos/cleanup/PaxosCleanupSession.java new file mode 100644 index 0000000000..3d765eaada --- /dev/null +++ b/src/java/org/apache/cassandra/service/paxos/cleanup/PaxosCleanupSession.java @@ -0,0 +1,267 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.paxos.cleanup; + +import java.lang.ref.WeakReference; +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; + +import com.google.common.base.Preconditions; + +import org.apache.cassandra.concurrent.ScheduledExecutors; +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.Token; +import org.apache.cassandra.exceptions.RequestFailureReason; +import org.apache.cassandra.gms.ApplicationState; +import org.apache.cassandra.gms.EndpointState; +import org.apache.cassandra.gms.IEndpointStateChangeSubscriber; +import org.apache.cassandra.gms.IFailureDetectionEventListener; +import org.apache.cassandra.gms.VersionedValue; +import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.net.Message; +import org.apache.cassandra.net.MessagingService; +import org.apache.cassandra.net.RequestCallbackWithFailure; +import org.apache.cassandra.schema.TableId; +import org.apache.cassandra.utils.concurrent.AsyncFuture; + +import static org.apache.cassandra.net.Verb.PAXOS2_CLEANUP_REQ; +import static org.apache.cassandra.utils.Clock.Global.nanoTime; + +public class PaxosCleanupSession extends AsyncFuture implements Runnable, + IEndpointStateChangeSubscriber, + IFailureDetectionEventListener, + RequestCallbackWithFailure +{ + private static final Map sessions = new ConcurrentHashMap<>(); + + static final long TIMEOUT_NANOS; + static + { + long timeoutSeconds = Integer.getInteger("cassandra.paxos_cleanup_session_timeout_seconds", (int) TimeUnit.HOURS.toSeconds(2)); + TIMEOUT_NANOS = TimeUnit.SECONDS.toNanos(timeoutSeconds); + } + + private static class TimeoutTask implements Runnable + { + private final WeakReference ref; + + TimeoutTask(PaxosCleanupSession session) + { + this.ref = new WeakReference<>(session); + } + + @Override + public void run() + { + PaxosCleanupSession session = ref.get(); + if (session == null || session.isDone()) + return; + + long remaining = session.lastMessageSentNanos + TIMEOUT_NANOS - nanoTime(); + if (remaining > 0) + schedule(remaining); + else + session.fail(String.format("Paxos cleanup session %s timed out", session.session)); + } + + ScheduledFuture schedule(long delayNanos) + { + return ScheduledExecutors.scheduledTasks.scheduleTimeoutWithDelay(this, delayNanos, TimeUnit.NANOSECONDS); + } + + private static ScheduledFuture schedule(PaxosCleanupSession session) + { + return new TimeoutTask(session).schedule(TIMEOUT_NANOS); + } + } + + private final UUID session = UUID.randomUUID(); + private final TableId tableId; + private final Collection> ranges; + private final Queue pendingCleanups = new ConcurrentLinkedQueue<>(); + private InetAddressAndPort inProgress = null; + private volatile long lastMessageSentNanos = nanoTime(); + private ScheduledFuture timeout; + + PaxosCleanupSession(Collection endpoints, TableId tableId, Collection> ranges) + { + this.tableId = tableId; + this.ranges = ranges; + + pendingCleanups.addAll(endpoints); + } + + private static void setSession(PaxosCleanupSession session) + { + Preconditions.checkState(!sessions.containsKey(session.session)); + sessions.put(session.session, session); + } + + private static void removeSession(PaxosCleanupSession session) + { + Preconditions.checkState(sessions.containsKey(session.session)); + sessions.remove(session.session); + } + + @Override + public void run() + { + setSession(this); + startNextOrFinish(); + if (!isDone()) + timeout = TimeoutTask.schedule(this); + } + + private void startCleanup(InetAddressAndPort endpoint) + { + lastMessageSentNanos = nanoTime(); + PaxosCleanupRequest completer = new PaxosCleanupRequest(session, tableId, ranges); + Message msg = Message.out(PAXOS2_CLEANUP_REQ, completer); + MessagingService.instance().sendWithCallback(msg, endpoint, this); + } + + private synchronized void startNextOrFinish() + { + InetAddressAndPort endpoint = pendingCleanups.poll(); + + if (endpoint == null) + Preconditions.checkState(inProgress == null, "Unable to complete paxos cleanup session %s, still waiting on %s", session, inProgress); + else + Preconditions.checkState(inProgress == null, "Unable to start paxos cleanup on %s for %s, still waiting on response from %s", endpoint, session, inProgress); + + inProgress = endpoint; + + if (endpoint != null) + { + startCleanup(endpoint); + } + else + { + removeSession(this); + trySuccess(null); + if (timeout != null) + timeout.cancel(true); + } + } + + private synchronized void fail(String message) + { + if (isDone()) + return; + removeSession(this); + tryFailure(new PaxosCleanupException(message)); + if (timeout != null) + timeout.cancel(true); + } + + private synchronized void finish(InetAddressAndPort from, PaxosCleanupResponse finished) + { + Preconditions.checkArgument(from.equals(inProgress), "Received unexpected cleanup complete response from %s for session %s. Expected %s", from, session, inProgress); + inProgress = null; + + if (finished.wasSuccessful) + { + startNextOrFinish(); + } + else + { + fail(String.format("Paxos cleanup session %s failed on %s with message: %s", session, from, finished.message)); + } + } + + public static void finishSession(InetAddressAndPort from, PaxosCleanupResponse response) + { + PaxosCleanupSession session = sessions.get(response.session); + if (session != null) + session.finish(from, response); + } + + private synchronized void maybeKillSession(InetAddressAndPort unavailable, String reason) + { + // don't fail if we've already completed the cleanup for the unavailable endpoint, + // if it's something that affects availability, the ongoing sessions will fail themselves + if (!pendingCleanups.contains(unavailable)) + return; + + fail(String.format("Paxos cleanup session %s failed after %s %s", session, unavailable, reason)); + } + + @Override + public void onJoin(InetAddressAndPort endpoint, EndpointState epState) + { + + } + + @Override + public void beforeChange(InetAddressAndPort endpoint, EndpointState currentState, ApplicationState newStateKey, VersionedValue newValue) + { + + } + + @Override + public void onChange(InetAddressAndPort endpoint, ApplicationState state, VersionedValue value) + { + + } + + @Override + public void onAlive(InetAddressAndPort endpoint, EndpointState state) + { + + } + + @Override + public void onDead(InetAddressAndPort endpoint, EndpointState state) + { + maybeKillSession(endpoint, "marked dead"); + } + + @Override + public void onRemove(InetAddressAndPort endpoint) + { + maybeKillSession(endpoint, "removed from ring"); + } + + @Override + public void onRestart(InetAddressAndPort endpoint, EndpointState state) + { + maybeKillSession(endpoint, "restarted"); + } + + @Override + public void convict(InetAddressAndPort ep, double phi) + { + maybeKillSession(ep, "convicted by failure detector"); + } + + @Override + public void onFailure(InetAddressAndPort from, RequestFailureReason reason) + { + fail(from.toString() + ' ' + reason + " for cleanup request for paxos cleanup session " + session); + } + + @Override + public void onResponse(Message msg) + { + // noop, we're only interested in failures + } +} diff --git a/src/java/org/apache/cassandra/service/paxos/cleanup/PaxosFinishPrepareCleanup.java b/src/java/org/apache/cassandra/service/paxos/cleanup/PaxosFinishPrepareCleanup.java new file mode 100644 index 0000000000..92d8d35028 --- /dev/null +++ b/src/java/org/apache/cassandra/service/paxos/cleanup/PaxosFinishPrepareCleanup.java @@ -0,0 +1,169 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.paxos.cleanup; + +import java.io.IOException; +import java.util.*; +import java.util.concurrent.atomic.AtomicReference; + +import org.apache.cassandra.concurrent.Stage; +import org.apache.cassandra.db.SystemKeyspace; +import org.apache.cassandra.exceptions.RequestFailureReason; +import org.apache.cassandra.io.FSWriteError; +import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.net.*; +import org.apache.cassandra.schema.Schema; +import org.apache.cassandra.service.paxos.Ballot; +import org.apache.cassandra.service.paxos.PaxosState; +import org.apache.cassandra.utils.Throwables; +import org.apache.cassandra.utils.concurrent.AsyncFuture; +import org.apache.cassandra.utils.concurrent.IntrusiveStack; + +import static org.apache.cassandra.exceptions.RequestFailureReason.UNKNOWN; +import static org.apache.cassandra.net.NoPayload.noPayload; + +public class PaxosFinishPrepareCleanup extends AsyncFuture implements RequestCallbackWithFailure +{ + private final Set waitingResponse; + + PaxosFinishPrepareCleanup(Collection endpoints) + { + this.waitingResponse = new HashSet<>(endpoints); + } + + public static PaxosFinishPrepareCleanup finish(Collection endpoints, PaxosCleanupHistory result) + { + PaxosFinishPrepareCleanup callback = new PaxosFinishPrepareCleanup(endpoints); + synchronized (callback) + { + Message message = Message.out(Verb.PAXOS2_CLEANUP_FINISH_PREPARE_REQ, result); + for (InetAddressAndPort endpoint : endpoints) + MessagingService.instance().sendWithCallback(message, endpoint, callback); + } + return callback; + } + + @Override + public void onFailure(InetAddressAndPort from, RequestFailureReason reason) + { + tryFailure(new PaxosCleanupException(reason + " failure response from " + from)); + } + + public synchronized void onResponse(Message msg) + { + if (isDone()) + return; + + if (!waitingResponse.remove(msg.from())) + throw new IllegalArgumentException("Received unexpected response from " + msg.from()); + + if (waitingResponse.isEmpty()) + trySuccess(null); + } + + static class PendingCleanup extends IntrusiveStack + { + private static final AtomicReference pendingCleanup = new AtomicReference(); + private static final Runnable CLEANUP = () -> { + PendingCleanup list = pendingCleanup.getAndSet(null); + if (list == null) + return; + + Ballot highBound = Ballot.none(); + for (PendingCleanup pending : IntrusiveStack.iterable(list)) + { + PaxosCleanupHistory cleanupHistory = pending.message.payload; + if (cleanupHistory.highBound.compareTo(highBound) > 0) + highBound = cleanupHistory.highBound; + } + try + { + try + { + PaxosState.ballotTracker().updateLowBound(highBound); + } + catch (IOException e) + { + throw new FSWriteError(e); + } + } + catch (Throwable t) + { + for (PendingCleanup pending : IntrusiveStack.iterable(list)) + MessagingService.instance().respondWithFailure(UNKNOWN, pending.message); + throw t; + } + + Set failed = null; + Throwable fail = null; + for (PendingCleanup pending : IntrusiveStack.iterable(list)) + { + try + { + Schema.instance.getColumnFamilyStoreInstance(pending.message.payload.tableId) + .syncPaxosRepairHistory(pending.message.payload.history, false); + } + catch (Throwable t) + { + fail = Throwables.merge(fail, t); + if (failed == null) + failed = Collections.newSetFromMap(new IdentityHashMap<>()); + failed.add(pending); + MessagingService.instance().respondWithFailure(UNKNOWN, pending.message); + } + } + + try + { + SystemKeyspace.flushPaxosRepairHistory(); + for (PendingCleanup pending : IntrusiveStack.iterable(list)) + { + if (failed == null || !failed.contains(pending)) + MessagingService.instance().respond(noPayload, pending.message); + } + } + catch (Throwable t) + { + fail = Throwables.merge(fail, t); + for (PendingCleanup pending : IntrusiveStack.iterable(list)) + { + if (failed == null || !failed.contains(pending)) + MessagingService.instance().respondWithFailure(UNKNOWN, pending.message); + } + } + Throwables.maybeFail(fail); + }; + + final Message message; + PendingCleanup(Message message) + { + this.message = message; + } + + public static void add(Message message) + { + PendingCleanup next = new PendingCleanup(message); + PendingCleanup prev = IntrusiveStack.push(AtomicReference::get, AtomicReference::compareAndSet, pendingCleanup, next); + if (prev == null) + Stage.MISC.execute(CLEANUP); + } + } + + public static final IVerbHandler verbHandler = PendingCleanup::add; +} diff --git a/src/java/org/apache/cassandra/service/paxos/cleanup/PaxosStartPrepareCleanup.java b/src/java/org/apache/cassandra/service/paxos/cleanup/PaxosStartPrepareCleanup.java new file mode 100644 index 0000000000..9f30692ad4 --- /dev/null +++ b/src/java/org/apache/cassandra/service/paxos/cleanup/PaxosStartPrepareCleanup.java @@ -0,0 +1,192 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.paxos.cleanup; + +import java.io.IOException; +import java.util.*; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.db.*; +import org.apache.cassandra.dht.AbstractBounds; +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.Token; +import org.apache.cassandra.exceptions.RequestFailureReason; +import org.apache.cassandra.gms.EndpointState; +import org.apache.cassandra.gms.Gossiper; +import org.apache.cassandra.io.IVersionedSerializer; +import org.apache.cassandra.io.util.DataInputPlus; +import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.net.*; +import org.apache.cassandra.schema.Schema; +import org.apache.cassandra.schema.TableId; +import org.apache.cassandra.service.PendingRangeCalculatorService; +import org.apache.cassandra.service.paxos.Ballot; +import org.apache.cassandra.service.paxos.Commit; +import org.apache.cassandra.service.paxos.PaxosRepairHistory; +import org.apache.cassandra.utils.concurrent.AsyncFuture; + +import static org.apache.cassandra.net.Verb.PAXOS2_CLEANUP_START_PREPARE_REQ; +import static org.apache.cassandra.service.paxos.Paxos.newBallot; +import static org.apache.cassandra.service.paxos.PaxosState.ballotTracker; + +/** + * Determines the highest ballot we should attempt to repair + */ +public class PaxosStartPrepareCleanup extends AsyncFuture implements RequestCallbackWithFailure +{ + private static final Logger logger = LoggerFactory.getLogger(PaxosStartPrepareCleanup.class); + + public static final RequestSerializer serializer = new RequestSerializer(); + + private final TableId table; + + private final Set waitingResponse; + private Ballot maxBallot = null; + private PaxosRepairHistory history = null; + + PaxosStartPrepareCleanup(TableId table, Collection endpoints) + { + this.table = table; + this.waitingResponse = new HashSet<>(endpoints); + } + + /** + * We run paxos repair as part of topology changes, so we include the local endpoint state in the paxos repair + * prepare message to prevent racing with gossip dissemination and guarantee that every repair participant is aware + * of the pending ring change during repair. + */ + public static PaxosStartPrepareCleanup prepare(TableId tableId, Collection endpoints, EndpointState localEpState, Collection> ranges) + { + PaxosStartPrepareCleanup callback = new PaxosStartPrepareCleanup(tableId, endpoints); + synchronized (callback) + { + Message message = Message.out(PAXOS2_CLEANUP_START_PREPARE_REQ, new Request(tableId, localEpState, ranges)); + for (InetAddressAndPort endpoint : endpoints) + MessagingService.instance().sendWithCallback(message, endpoint, callback); + } + return callback; + } + + @Override + public void onFailure(InetAddressAndPort from, RequestFailureReason reason) + { + tryFailure(new PaxosCleanupException("Received " + reason + " failure response from " + from)); + } + + public synchronized void onResponse(Message msg) + { + if (isDone()) + return; + + if (!waitingResponse.remove(msg.from())) + throw new IllegalArgumentException("Received unexpected response from " + msg.from()); + + if (Commit.isAfter(msg.payload.highBound, maxBallot)) + maxBallot = msg.payload.highBound; + + history = PaxosRepairHistory.merge(history, msg.payload.history); + + if (waitingResponse.isEmpty()) + trySuccess(new PaxosCleanupHistory(table, maxBallot, history)); + } + + private static void maybeUpdateTopology(InetAddressAndPort endpoint, EndpointState remote) + { + EndpointState local = Gossiper.instance.getEndpointStateForEndpoint(endpoint); + if (local == null || local.isSupersededBy(remote)) + { + logger.trace("updating endpoint info for {} with {}", endpoint, remote); + Map states = Collections.singletonMap(endpoint, remote); + + Gossiper.runInGossipStageBlocking(() -> { + Gossiper.instance.notifyFailureDetector(states); + Gossiper.instance.applyStateLocally(states); + }); + // TODO: We should also wait for schema pulls/pushes, however this would be quite an involved change to MigrationManager + // (which currently drops some migration tasks on the floor). + // Note it would be fine for us to fail to complete the migration task and simply treat this response as a failure/timeout. + } + // even if we have th latest gossip info, wait until pending range calculations are complete + PendingRangeCalculatorService.instance.blockUntilFinished(); + } + + public static class Request + { + final TableId tableId; + final EndpointState epState; + final Collection> ranges; + + public Request(TableId tableId, EndpointState epState, Collection> ranges) + { + this.tableId = tableId; + this.epState = epState; + this.ranges = ranges; + } + } + + public static class RequestSerializer implements IVersionedSerializer + { + public void serialize(Request request, DataOutputPlus out, int version) throws IOException + { + request.tableId.serialize(out); + EndpointState.serializer.serialize(request.epState, out, version); + out.writeInt(request.ranges.size()); + for (Range rt : request.ranges) + AbstractBounds.tokenSerializer.serialize(rt, out, version); + } + + public Request deserialize(DataInputPlus in, int version) throws IOException + { + TableId tableId = TableId.deserialize(in); + EndpointState epState = EndpointState.serializer.deserialize(in, version); + + int numRanges = in.readInt(); + List> ranges = new ArrayList<>(); + for (int i = 0; i < numRanges; i++) + { + Range range = (Range) AbstractBounds.tokenSerializer.deserialize(in, DatabaseDescriptor.getPartitioner(), version); + ranges.add(range); + } + return new Request(tableId, epState, ranges); + } + + public long serializedSize(Request request, int version) + { + long size = request.tableId.serializedSize(); + size += EndpointState.serializer.serializedSize(request.epState, version); + size += TypeSizes.sizeof(request.ranges.size()); + for (Range range : request.ranges) + size += AbstractBounds.tokenSerializer.serializedSize(range, version); + return size; + } + } + + public static final IVerbHandler verbHandler = in -> { + ColumnFamilyStore table = Schema.instance.getColumnFamilyStoreInstance(in.payload.tableId); + maybeUpdateTopology(in.from(), in.payload.epState); + Ballot highBound = newBallot(ballotTracker().getHighBound(), ConsistencyLevel.SERIAL); + PaxosRepairHistory history = table.getPaxosRepairHistoryForRanges(in.payload.ranges); + Message out = in.responseWith(new PaxosCleanupHistory(table.metadata.id, highBound, history)); + MessagingService.instance().send(out, in.respondTo()); + }; +} diff --git a/src/java/org/apache/cassandra/service/paxos/cleanup/PaxosTableRepairs.java b/src/java/org/apache/cassandra/service/paxos/cleanup/PaxosTableRepairs.java new file mode 100644 index 0000000000..6da4e0bce1 --- /dev/null +++ b/src/java/org/apache/cassandra/service/paxos/cleanup/PaxosTableRepairs.java @@ -0,0 +1,235 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.paxos.cleanup; + +import java.util.ArrayDeque; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; +import java.util.function.Predicate; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.db.ConsistencyLevel; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.schema.TableId; +import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.service.paxos.AbstractPaxosRepair; +import org.apache.cassandra.service.paxos.Ballot; +import org.apache.cassandra.service.paxos.PaxosRepair; +import org.apache.cassandra.utils.NoSpamLogger; + +import static org.apache.cassandra.service.paxos.Commit.isAfter; +import static org.apache.cassandra.utils.Clock.Global.nanoTime; + +/** + * Coordinates repairs on a given key to prevent multiple repairs being scheduled for a single key + */ +public class PaxosTableRepairs implements AbstractPaxosRepair.Listener +{ + private static final Logger logger = LoggerFactory.getLogger(PaxosTableRepairs.class); + + static class KeyRepair + { + private final DecoratedKey key; + + private final ArrayDeque queued = new ArrayDeque<>(); + + private KeyRepair(DecoratedKey key) + { + this.key = key; + } + + void onFirst(Predicate predicate, Consumer consumer, boolean removeBeforeAction) + { + while (!queued.isEmpty()) + { + AbstractPaxosRepair repair = queued.peek(); + if (repair.isComplete()) + { + queued.remove(); + continue; + } + + if (predicate.test(repair)) + { + if (removeBeforeAction) + queued.remove(); + consumer.accept(repair); + } + return; + } + } + + void clear() + { + while (!queued.isEmpty()) + queued.remove().cancelUnexceptionally(); + } + + AbstractPaxosRepair startOrGetOrQueue(PaxosTableRepairs tableRepairs, DecoratedKey key, Ballot incompleteBallot, ConsistencyLevel consistency, TableMetadata table, Consumer onComplete) + { + Preconditions.checkArgument(this.key.equals(key)); + + if (!queued.isEmpty() && !isAfter(incompleteBallot, queued.peekLast().incompleteBallot())) + { + queued.peekLast().addListener(onComplete); + return queued.peekLast(); + } + + AbstractPaxosRepair repair = tableRepairs.createRepair(key, incompleteBallot, consistency, table); + + repair.addListener(tableRepairs); + repair.addListener(onComplete); + + queued.add(repair); + maybeScheduleNext(); + return repair; + } + + @VisibleForTesting + AbstractPaxosRepair activeRepair() + { + return queued.peek(); + } + + @VisibleForTesting + boolean queueContains(AbstractPaxosRepair repair) + { + return queued.contains(repair); + } + + void maybeScheduleNext() + { + onFirst(repair -> !repair.isStarted(), AbstractPaxosRepair::start, false); + } + + void complete(AbstractPaxosRepair repair) + { + queued.remove(repair); + maybeScheduleNext(); + } + + int pending() + { + return queued.size(); + } + + boolean isEmpty() + { + return queued.isEmpty(); + } + } + + private final Map keyRepairs = new ConcurrentHashMap<>(); + + @VisibleForTesting + KeyRepair getKeyRepairUnsafe(DecoratedKey key) + { + return keyRepairs.get(key); + } + + synchronized AbstractPaxosRepair startOrGetOrQueue(DecoratedKey key, Ballot incompleteBallot, ConsistencyLevel consistency, TableMetadata table, Consumer onComplete) + { + KeyRepair keyRepair = keyRepairs.computeIfAbsent(key, KeyRepair::new); + return keyRepair.startOrGetOrQueue(this, key, incompleteBallot, consistency, table, onComplete); + } + + public synchronized void onComplete(AbstractPaxosRepair repair, AbstractPaxosRepair.Result result) + { + KeyRepair keyRepair = keyRepairs.get(repair.partitionKey()); + if (keyRepair == null) + { + NoSpamLogger.log(logger, NoSpamLogger.Level.WARN, 1, TimeUnit.MINUTES, + "onComplete callback fired for nonexistant KeyRepair"); + return; + } + + keyRepair.complete(repair); + if (keyRepair.queueContains(repair)) + NoSpamLogger.log(logger, NoSpamLogger.Level.WARN, 1, TimeUnit.MINUTES, + "repair not removed after call to onComplete"); + + if (keyRepair.isEmpty()) + keyRepairs.remove(repair.partitionKey()); + } + + synchronized void evictHungRepairs(long activeSinceNanos) + { + Predicate timeoutPredicate = repair -> repair.startedNanos() - activeSinceNanos < 0; + for (KeyRepair repair : keyRepairs.values()) + { + if (repair.isEmpty()) + NoSpamLogger.log(logger, NoSpamLogger.Level.WARN, 1, TimeUnit.MINUTES, + "inactive KeyRepair found, this means post-repair cleanup/schedule isn't working properly"); + repair.onFirst(timeoutPredicate, r -> { + logger.warn("cancelling timed out paxos repair: {}", r); + r.cancelUnexceptionally(); + }, true); + repair.maybeScheduleNext(); + if (repair.isEmpty()) + keyRepairs.remove(repair.key); + } + } + + synchronized void clear() + { + for (KeyRepair repair : keyRepairs.values()) + repair.clear(); + keyRepairs.clear(); + } + + @VisibleForTesting + synchronized boolean hasActiveRepairs(DecoratedKey key) + { + return keyRepairs.containsKey(key); + } + + AbstractPaxosRepair createRepair(DecoratedKey key, Ballot incompleteBallot, ConsistencyLevel consistency, TableMetadata table) + { + return PaxosRepair.create(consistency, key, incompleteBallot, table); + } + + private static final ConcurrentMap tableRepairsMap = new ConcurrentHashMap<>(); + + static PaxosTableRepairs getForTable(TableId tableId) + { + return tableRepairsMap.computeIfAbsent(tableId, k -> new PaxosTableRepairs()); + } + + public static void evictHungRepairs() + { + long deadline = nanoTime() - TimeUnit.MINUTES.toNanos(5); + for (PaxosTableRepairs repairs : tableRepairsMap.values()) + repairs.evictHungRepairs(deadline); + } + + public static void clearRepairs() + { + for (PaxosTableRepairs repairs : tableRepairsMap.values()) + repairs.clear(); + } + +} diff --git a/src/java/org/apache/cassandra/service/paxos/uncommitted/PaxosBallotTracker.java b/src/java/org/apache/cassandra/service/paxos/uncommitted/PaxosBallotTracker.java new file mode 100644 index 0000000000..7404bffae4 --- /dev/null +++ b/src/java/org/apache/cassandra/service/paxos/uncommitted/PaxosBallotTracker.java @@ -0,0 +1,207 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.paxos.uncommitted; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.concurrent.atomic.AtomicReference; +import java.util.zip.CRC32; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import org.apache.cassandra.service.ClientState; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.db.rows.Row; +import org.apache.cassandra.io.util.File; +import org.apache.cassandra.io.util.RandomAccessReader; +import org.apache.cassandra.io.util.SequentialWriter; +import org.apache.cassandra.service.paxos.Ballot; +import org.apache.cassandra.service.paxos.Commit; + +import static org.apache.cassandra.io.util.SequentialWriterOption.FINISH_ON_CLOSE; + +/** + * Tracks the highest paxos ballot we've seen, and the lowest ballot we can accept. + * + * During paxos repair, the coordinator gets the highest ballot seen by each participant. At the end of repair, that + * high ballot is set as the new low bound. Combined with paxos repair during topology changes, this eliminates the + * possibility of new nodes accepting ballots that are before the most recently accepted ballot for a key. + */ +public class PaxosBallotTracker +{ + private static final Logger logger = LoggerFactory.getLogger(PaxosBallotTracker.class); + + private static final int FILE_VERSION = 0; + static final String FNAME = "ballot.meta"; + private static final String TMP_FNAME = FNAME + ".tmp"; + + private final File directory; + private final AtomicReference highBound; + private volatile Ballot lowBound; + + private PaxosBallotTracker(File directory, Ballot highBound, Ballot lowBound) + { + Preconditions.checkNotNull(lowBound); + Preconditions.checkNotNull(highBound); + this.directory = directory; + this.highBound = new AtomicReference<>(highBound); + this.lowBound = lowBound; + } + + /** + * creates a new crc32 instance seeded with a non-zero value + */ + private static void serializeBallot(SequentialWriter writer, CRC32 crc, Ballot ballot) throws IOException + { + ByteBuffer bytes = ballot.toBytes(); + writer.write(bytes); + crc.update(bytes); + } + + private static Ballot deserializeBallot(RandomAccessReader reader, CRC32 crc, byte[] bytes) throws IOException + { + reader.readFully(bytes); + crc.update(bytes); + return Ballot.deserialize(ByteBuffer.wrap(bytes)); + } + + public static void truncate(File directory) throws IOException + { + logger.info("truncating paxos ballot metadata in {}", directory); + deleteIfExists(new File(directory, TMP_FNAME)); + deleteIfExists(new File(directory, FNAME)); + } + + public static PaxosBallotTracker load(File directory) throws IOException + { + deleteIfExists(new File(directory, TMP_FNAME)); + + File file = new File(directory, FNAME); + if (!file.exists()) + return new PaxosBallotTracker(directory, Ballot.none(), Ballot.none()); + + try (RandomAccessReader reader = RandomAccessReader.open(file)) + { + int version = reader.readInt(); + if (version != FILE_VERSION) + throw new IOException("Unsupported ballot file version: " + version); + + byte[] bytes = new byte[16]; + CRC32 crc = new CRC32(); + Ballot highBallot = deserializeBallot(reader, crc, bytes); + Ballot lowBallot = deserializeBallot(reader, crc, bytes); + int checksum = Integer.reverseBytes(reader.readInt()); + if (!reader.isEOF() || (int) crc.getValue() != checksum) + throw new IOException("Ballot file corrupted"); + + return new PaxosBallotTracker(directory, highBallot, lowBallot); + } + } + + private static void deleteIfExists(File file) + { + if (file.exists()) + file.delete(); + } + + public synchronized void flush() throws IOException + { + File file = new File(directory, TMP_FNAME); + deleteIfExists(file); + + try(SequentialWriter writer = new SequentialWriter(file, FINISH_ON_CLOSE)) + { + CRC32 crc = new CRC32(); + writer.writeInt(FILE_VERSION); + serializeBallot(writer, crc, getHighBound()); + serializeBallot(writer, crc, getLowBound()); + writer.writeInt(Integer.reverseBytes((int) crc.getValue())); + } + file.move(new File(directory, FNAME)); + } + + public synchronized void truncate() + { + deleteIfExists(new File(directory, TMP_FNAME)); + deleteIfExists(new File(directory, FNAME)); + highBound.set(Ballot.none()); + lowBound = Ballot.none(); + } + + private void updateHighBound(Ballot current, Ballot next) + { + while (Commit.isAfter(next, current) && !highBound.compareAndSet(current, next)) + current = highBound.get(); + } + + void updateHighBound(Ballot next) + { + updateHighBound(highBound.get(), next); + } + + public void onUpdate(Row row) + { + Ballot current = highBound.get(); + Ballot next = PaxosRows.getHighBallot(row, current); + if (current == next) + return; + + updateHighBound(current, next); + } + + @VisibleForTesting + void updateHighBoundUnsafe(Ballot expected, Ballot update) + { + highBound.compareAndSet(expected, update); + } + + public File getDirectory() + { + return directory; + } + + public synchronized void updateLowBound(Ballot update) throws IOException + { + if (!Commit.isAfter(update, lowBound)) + { + logger.debug("Not updating lower bound with earlier or equal ballot from {} to {}", lowBound, update); + return; + } + + logger.debug("Updating lower bound from {} to {}", lowBound, update); + ClientState.getTimestampForPaxos(lowBound.unixMicros()); + lowBound = update; + flush(); + } + + public Ballot getHighBound() + { + return highBound.get(); + } + + /** + * @return a unique ballot that has never been proposed, below which we will reject all proposals + */ + public Ballot getLowBound() + { + return lowBound; + } +} diff --git a/src/java/org/apache/cassandra/service/paxos/uncommitted/PaxosKeyState.java b/src/java/org/apache/cassandra/service/paxos/uncommitted/PaxosKeyState.java new file mode 100644 index 0000000000..c55269afa1 --- /dev/null +++ b/src/java/org/apache/cassandra/service/paxos/uncommitted/PaxosKeyState.java @@ -0,0 +1,168 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.paxos.uncommitted; + +import java.util.Comparator; +import java.util.Iterator; +import java.util.Objects; + +import com.google.common.base.Preconditions; +import com.google.common.collect.Iterators; +import com.google.common.collect.Lists; +import com.google.common.primitives.Longs; + +import org.apache.cassandra.db.ConsistencyLevel; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.schema.TableId; +import org.apache.cassandra.service.paxos.Ballot; +import org.apache.cassandra.utils.CloseableIterator; +import org.apache.cassandra.utils.MergeIterator; + +public class PaxosKeyState implements UncommittedPaxosKey +{ + static final Comparator KEY_COMPARATOR = Comparator.comparing(o -> o.key); + static final Comparator BALLOT_COMPARATOR = (o1, o2) -> Longs.compare(o1.ballot.uuidTimestamp(), o2.ballot.uuidTimestamp()); + + final TableId tableId; + final DecoratedKey key; + final Ballot ballot; + final boolean committed; + + public PaxosKeyState(TableId tableId, DecoratedKey key, Ballot ballot, boolean committed) + { + Preconditions.checkNotNull(tableId); + Preconditions.checkNotNull(ballot); + this.tableId = tableId; + this.key = key; + this.ballot = ballot; + this.committed = committed; + } + + public DecoratedKey getKey() + { + return key; + } + + public ConsistencyLevel getConsistencyLevel() + { + switch (ballot.flag()) + { + default: throw new IllegalStateException(); + case GLOBAL: return ConsistencyLevel.SERIAL; + case LOCAL: return ConsistencyLevel.LOCAL_SERIAL; + case NONE: return null; + } + } + + @Override + public Ballot ballot() + { + return ballot; + } + + public boolean equals(Object o) + { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + PaxosKeyState that = (PaxosKeyState) o; + return committed == that.committed && + Objects.equals(key, that.key) && + Objects.equals(ballot, that.ballot); + } + + public int hashCode() + { + return Objects.hash(key, ballot, committed); + } + + public String toString() + { + return "BallotState{" + + "tableId=" + tableId + + ", key=" + key + + ", ballot=" + ballot + + ", committed=" + committed + + '}'; + } + + static PaxosKeyState merge(PaxosKeyState left, PaxosKeyState right) + { + if (left == null) + return right; + + if (right == null) + return left; + + int cmp = BALLOT_COMPARATOR.compare(left, right); + + // prefer committed operations if the ballots are the same so they can be filtered out later + if (cmp == 0) + return left.committed ? left : right; + else + return cmp > 0 ? left : right; + } + + static class Reducer extends MergeIterator.Reducer + { + private PaxosKeyState mostRecent = null; + + public void reduce(int idx, PaxosKeyState current) + { + mostRecent = merge(mostRecent, current); + } + + protected PaxosKeyState getReduced() + { + return mostRecent; + } + + protected void onKeyChange() + { + super.onKeyChange(); + mostRecent = null; + } + } + + public static CloseableIterator mergeUncommitted(CloseableIterator... iterators) + { + return MergeIterator.get(Lists.newArrayList(iterators), PaxosKeyState.KEY_COMPARATOR, new Reducer()); + } + + public static CloseableIterator toUncommittedInfo(CloseableIterator iter) + { + Iterator filtered = Iterators.filter(iter, k -> !k.committed); + return new CloseableIterator() + { + public void close() + { + iter.close(); + } + + public boolean hasNext() + { + return filtered.hasNext(); + } + + public UncommittedPaxosKey next() + { + return filtered.next(); + } + }; + } +} diff --git a/src/java/org/apache/cassandra/service/paxos/uncommitted/PaxosRows.java b/src/java/org/apache/cassandra/service/paxos/uncommitted/PaxosRows.java new file mode 100644 index 0000000000..8bdbdf7524 --- /dev/null +++ b/src/java/org/apache/cassandra/service/paxos/uncommitted/PaxosRows.java @@ -0,0 +1,359 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.paxos.uncommitted; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.UUID; + +import javax.annotation.Nullable; + +import com.google.common.collect.Lists; + +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.SystemKeyspace; +import org.apache.cassandra.db.marshal.AbstractType; +import org.apache.cassandra.db.marshal.BytesType; +import org.apache.cassandra.db.marshal.Int32Type; +import org.apache.cassandra.db.marshal.TimeUUIDType; +import org.apache.cassandra.db.marshal.UUIDType; +import org.apache.cassandra.db.marshal.ValueAccessor; +import org.apache.cassandra.db.partitions.PartitionUpdate; +import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator; +import org.apache.cassandra.db.rows.Cell; +import org.apache.cassandra.db.rows.DeserializationHelper; +import org.apache.cassandra.db.rows.Row; +import org.apache.cassandra.db.rows.UnfilteredRowIterator; +import org.apache.cassandra.exceptions.UnknownTableException; +import org.apache.cassandra.net.MessagingService; +import org.apache.cassandra.schema.ColumnMetadata; +import org.apache.cassandra.schema.SchemaConstants; +import org.apache.cassandra.schema.TableId; +import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.service.paxos.Ballot; +import org.apache.cassandra.service.paxos.Commit; +import org.apache.cassandra.service.paxos.Commit.Accepted; +import org.apache.cassandra.service.paxos.Commit.AcceptedWithTTL; +import org.apache.cassandra.service.paxos.Commit.Committed; +import org.apache.cassandra.service.paxos.Commit.CommittedWithTTL; +import org.apache.cassandra.utils.AbstractIterator; +import org.apache.cassandra.utils.CloseableIterator; +import org.apache.cassandra.utils.JVMStabilityInspector; + +import static org.apache.cassandra.db.partitions.PartitionUpdate.PartitionUpdateSerializer.*; +import static org.apache.cassandra.service.paxos.Commit.isAfter; +import static org.apache.cassandra.service.paxos.Commit.latest; + +@SuppressWarnings({ "unchecked", "rawtypes" }) +public class PaxosRows +{ + private static final ColumnMetadata WRITE_PROMISE = paxosColumn("in_progress_ballot", TimeUUIDType.instance); + private static final ColumnMetadata READ_PROMISE = paxosColumn("in_progress_read_ballot", TimeUUIDType.instance); + private static final ColumnMetadata PROPOSAL = paxosColumn("proposal_ballot", TimeUUIDType.instance); + private static final ColumnMetadata PROPOSAL_UPDATE = paxosColumn("proposal", BytesType.instance); + private static final ColumnMetadata PROPOSAL_VERSION = paxosColumn("proposal_version", Int32Type.instance); + private static final ColumnMetadata COMMIT = paxosColumn("most_recent_commit_at", TimeUUIDType.instance); + private static final ColumnMetadata COMMIT_UPDATE = paxosColumn("most_recent_commit", BytesType.instance); + private static final ColumnMetadata COMMIT_VERSION = paxosColumn("most_recent_commit_version", Int32Type.instance); + + private PaxosRows() {} + + private static ColumnMetadata paxosColumn(String name, AbstractType type) + { + return ColumnMetadata.regularColumn(SchemaConstants.SYSTEM_KEYSPACE_NAME, SystemKeyspace.PAXOS, name, type); + } + + public static Ballot getPromise(Row row) + { + return getBallot(row, READ_PROMISE, Ballot.none()); + } + + public static Ballot getWritePromise(Row row) + { + return getBallot(row, WRITE_PROMISE, Ballot.none()); + } + + public static Accepted getAccepted(Row row) + { + Cell ballotCell = row.getCell(PROPOSAL); + if (ballotCell == null) + return null; + + Ballot ballot = ballotCell.accessor().toBallot(ballotCell.value()); + int version = getInt(row, PROPOSAL_VERSION, MessagingService.VERSION_30); + PartitionUpdate update = getUpdate(row, PROPOSAL_UPDATE, version); + return ballotCell.isExpiring() + ? new AcceptedWithTTL(ballot, update, ballotCell.localDeletionTime()) + : new Accepted(ballot, update); + } + + public static Committed getCommitted(TableMetadata metadata, DecoratedKey partitionKey, Row row) + { + Cell ballotCell = row.getCell(COMMIT); + if (ballotCell == null) + return Committed.none(partitionKey, metadata); + + Ballot ballot = ballotCell.accessor().toBallot(ballotCell.value()); + int version = getInt(row, COMMIT_VERSION, MessagingService.VERSION_30); + PartitionUpdate update = getUpdate(row, COMMIT_UPDATE, version); + return ballotCell.isExpiring() + ? new CommittedWithTTL(ballot, update, ballotCell.localDeletionTime()) + : new Committed(ballot, update); + } + + public static TableId getTableId(Row row) + { + return TableId.fromUUID(UUIDType.instance.compose(row.clustering().get(0), (ValueAccessor)row.clustering().accessor())); + } + + public static UUID getTableUuid(Row row) + { + return UUIDType.instance.compose(row.clustering().get(0), (ValueAccessor)row.clustering().accessor()); + } + + private static int getInt(Row row, ColumnMetadata cmeta, @SuppressWarnings("SameParameterValue") int ifNull) + { + Cell cell = row.getCell(cmeta); + if (cell == null) + return ifNull; + return Int32Type.instance.compose(cell.value(), cell.accessor()); + } + + private static PartitionUpdate getUpdate(Row row, ColumnMetadata cmeta, int version) + { + Cell cell = row.getCell(cmeta); + if (cell == null) + throw new IllegalStateException(); + try + { + return PartitionUpdate.fromBytes(cell.buffer(), version); + } + catch (RuntimeException e) + { + // the legacy behaviors of not deleting proposal_version along with proposal and proposal_ballot on commit, + // and accepting proposals younger than the most recent commit combined with the right sequence of tombstone + // purging and retention over a few compactions can result in 3.x format proposals without a proposal version + // value, causing deserialization to fail when looking up the table. So here we detect that and attempt to + // deserialize with the current version + if (e.getCause() instanceof UnknownTableException && version == MessagingService.VERSION_30) + return PartitionUpdate.fromBytes(cell.buffer(), MessagingService.current_version); + + throw e; + } + } + + private static Ballot getBallot(Row row, ColumnMetadata cmeta) + { + return getBallot(row, cmeta, null); + } + + private static Ballot getBallot(Row row, ColumnMetadata cmeta, Ballot ifNull) + { + Cell cell = row.getCell(cmeta); + if (cell == null) + return ifNull; + return cell.accessor().toBallot(cell.value()); + } + + private static boolean proposalIsEmpty(Row row, DecoratedKey key) + { + try + { + Cell proposalVersionCell = row.getCell(PROPOSAL_VERSION); + if (proposalVersionCell == null) + return true; + Integer proposalVersion = Int32Type.instance.compose(proposalVersionCell.value(), proposalVersionCell.accessor()); + if (proposalVersion == null) + return true; + + Cell proposal = row.getCell(PROPOSAL_UPDATE); + if (proposal == null) + return true; + + ByteBuffer proposalValue = proposal.buffer(); + if (!proposalValue.hasRemaining()) + return true; + + return isEmpty(proposalValue, DeserializationHelper.Flag.LOCAL, key); + } + catch (IOException e) + { + JVMStabilityInspector.inspectThrowable(e); + throw new RuntimeException(e); + } + } + + private static long getTimestamp(Row row, ColumnMetadata cmeta) + { + Cell cell = row.getCell(cmeta); + if (cell == null || cell.valueSize() == 0) + return Long.MIN_VALUE; + return cell.timestamp(); + } + + static PaxosKeyState getCommitState(DecoratedKey key, Row row, TableId targetTableId) + { + if (row == null) + return null; + + UUID tableUuid = getTableUuid(row); + if (targetTableId != null && !targetTableId.asUUID().equals(tableUuid)) + return null; + + Ballot promise = latest(getBallot(row, WRITE_PROMISE), getBallot(row, READ_PROMISE)); + Ballot proposal = getBallot(row, PROPOSAL); + Ballot commit = getBallot(row, COMMIT); + + Ballot inProgress = null; + Ballot committed = null; + if (isAfter(promise, proposal)) + { + if (isAfter(promise, commit)) + inProgress = promise; + else + committed = commit; + } + else if (isAfter(proposal, commit)) + { + if (proposalIsEmpty(row, key)) + committed = proposal; + else + inProgress = proposal; + } + else + { + committed = commit; + } + + TableId tableId = TableId.fromUUID(tableUuid); + return inProgress != null ? + new PaxosKeyState(tableId, key, inProgress, false) : + new PaxosKeyState(tableId, key, committed, true); + } + + private static class PaxosMemtableToKeyStateIterator extends AbstractIterator implements CloseableIterator + { + private final UnfilteredPartitionIterator partitions; + private UnfilteredRowIterator partition; + private final @Nullable TableId filterByTableId; // if unset, return records for all tables + + private PaxosMemtableToKeyStateIterator(UnfilteredPartitionIterator partitions, TableId filterByTableId) + { + this.partitions = partitions; + this.filterByTableId = filterByTableId; + } + + protected PaxosKeyState computeNext() + { + while (true) + { + if (partition != null && partition.hasNext()) + { + PaxosKeyState commitState = PaxosRows.getCommitState(partition.partitionKey(), + (Row) partition.next(), + filterByTableId); + if (commitState == null) + continue; + + return commitState; + } + else if (partition != null) + { + partition.close(); + partition = null; + } + + if (partitions.hasNext()) + { + partition = partitions.next(); + } + else + { + partitions.close(); + return endOfData(); + } + } + } + + public void close() + { + if (partition != null) + partition.close(); + partitions.close(); + } + } + + static CloseableIterator toIterator(UnfilteredPartitionIterator partitions, TableId filterBytableId, boolean materializeLazily) + { + CloseableIterator iter = new PaxosMemtableToKeyStateIterator(partitions, filterBytableId); + if (materializeLazily) + return iter; + + try + { + // eagerly materialize key states for repairs so we're not referencing memtables for the entire repair + return CloseableIterator.wrap(Lists.newArrayList(iter).iterator()); + } + finally + { + iter.close(); + } + } + + public static Ballot getHighBallot(Row row, Ballot current) + { + long maxUnixMicros = current != null ? current.unixMicros() : Long.MIN_VALUE; + ColumnMetadata maxCol = null; + + long inProgressRead = getTimestamp(row, READ_PROMISE); + if (inProgressRead > maxUnixMicros) + { + maxUnixMicros = inProgressRead; + maxCol = READ_PROMISE; + } + + long inProgressWrite = getTimestamp(row, WRITE_PROMISE); + if (inProgressWrite > maxUnixMicros) + { + maxUnixMicros = inProgressWrite; + maxCol = WRITE_PROMISE; + } + + long proposal = getTimestamp(row, PROPOSAL); + if (proposal > maxUnixMicros) + { + maxUnixMicros = proposal; + maxCol = PROPOSAL; + } + + long commit = getTimestamp(row, COMMIT); + if (commit > maxUnixMicros) + maxCol = COMMIT; + + return maxCol == null ? current : getBallot(row, maxCol); + } + + public static boolean hasBallotBeforeOrEqualTo(Row row, Ballot ballot) + { + return !Commit.isAfter(ballot, getBallot(row, WRITE_PROMISE)) + && !Commit.isAfter(ballot, getBallot(row, READ_PROMISE)) + && !Commit.isAfter(ballot, getBallot(row, PROPOSAL)) + && !Commit.isAfter(ballot, getBallot(row, COMMIT)); + } +} diff --git a/src/java/org/apache/cassandra/service/paxos/uncommitted/PaxosStateTracker.java b/src/java/org/apache/cassandra/service/paxos/uncommitted/PaxosStateTracker.java new file mode 100644 index 0000000000..d3594b3979 --- /dev/null +++ b/src/java/org/apache/cassandra/service/paxos/uncommitted/PaxosStateTracker.java @@ -0,0 +1,322 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.paxos.uncommitted; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.List; +import java.util.NoSuchElementException; + +import com.google.common.base.Preconditions; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.cql3.QueryOptions; +import org.apache.cassandra.cql3.QueryProcessor; +import org.apache.cassandra.cql3.statements.SelectStatement; +import org.apache.cassandra.db.Clustering; +import org.apache.cassandra.db.Directories; +import org.apache.cassandra.db.Keyspace; +import org.apache.cassandra.db.ReadExecutionController; +import org.apache.cassandra.db.ReadQuery; +import org.apache.cassandra.db.SystemKeyspace; +import org.apache.cassandra.db.marshal.BytesType; +import org.apache.cassandra.db.marshal.ListType; +import org.apache.cassandra.db.marshal.UTF8Type; +import org.apache.cassandra.db.partitions.PartitionIterator; +import org.apache.cassandra.db.rows.Cell; +import org.apache.cassandra.db.rows.Row; +import org.apache.cassandra.db.rows.RowIterator; +import org.apache.cassandra.io.util.File; +import org.apache.cassandra.schema.ColumnMetadata; +import org.apache.cassandra.schema.Schema; +import org.apache.cassandra.service.ClientState; +import org.apache.cassandra.service.paxos.Ballot; +import org.apache.cassandra.service.paxos.Commit; +import org.apache.cassandra.service.paxos.PaxosRepairHistory; +import org.apache.cassandra.utils.CloseableIterator; +import org.apache.cassandra.utils.FBUtilities; + +import static org.apache.cassandra.db.SystemKeyspace.PAXOS_REPAIR_HISTORY; +import static org.apache.cassandra.schema.SchemaConstants.SYSTEM_KEYSPACE_NAME; + +/** + * Tracks uncommitted and ballot high/low bounds + */ +public class PaxosStateTracker +{ + private static final Logger logger = LoggerFactory.getLogger(PaxosStateTracker.class); + + // when starting with no data, skip rebuilding uncommitted data from the paxos table + static final String SKIP_REBUILD_PROP = "cassandra.skip_paxos_state_rebuild"; + static final String FORCE_REBUILD_PROP = "cassandra.force_paxos_state_rebuild"; + static final String TRUNCATE_BALLOT_METADATA_PROP = "cassandra.truncate_ballot_metadata"; + + private static boolean skipRebuild() + { + return Boolean.getBoolean(SKIP_REBUILD_PROP); + } + + private static boolean forceRebuild() + { + return Boolean.getBoolean(FORCE_REBUILD_PROP); + } + + private static boolean truncateBallotMetadata() + { + return Boolean.getBoolean(TRUNCATE_BALLOT_METADATA_PROP); + } + + private static final String DIRECTORY = "system/_paxos_repair_state"; + + private final PaxosUncommittedTracker uncommitted; + private final PaxosBallotTracker ballots; + private boolean rebuildNeeded; + + public PaxosStateTracker(PaxosUncommittedTracker uncommitted, PaxosBallotTracker ballots, boolean rebuildNeeded) + { + this.uncommitted = uncommitted; + this.ballots = ballots; + this.rebuildNeeded = rebuildNeeded; + } + + public boolean isRebuildNeeded() + { + return rebuildNeeded; + } + + static File stateDirectory(File dataDirectory) + { + return new File(dataDirectory, DIRECTORY); + } + + public static PaxosStateTracker create(File[] directories) throws IOException + { + File stateDirectory = null; + boolean hasExistingData = false; + + for (File directory : directories) + { + File candidate = stateDirectory(directory); + if (candidate.exists() && new File(candidate, PaxosBallotTracker.FNAME).exists()) + { + Preconditions.checkState(!hasExistingData, + "Multiple paxos repair metadata directories found (%s, %s), remove the older directory and restart.", + stateDirectory, candidate); + hasExistingData = true; + stateDirectory = candidate; + } + } + + if (stateDirectory == null) + stateDirectory = stateDirectory(directories[0]); + + boolean rebuildNeeded = !hasExistingData || forceRebuild(); + + if (truncateBallotMetadata() && !rebuildNeeded) + logger.warn("{} was set, but {} was not and no rebuild is required. Ballot data will not be truncated", + TRUNCATE_BALLOT_METADATA_PROP, FORCE_REBUILD_PROP); + + if (rebuildNeeded) + { + if (stateDirectory.exists()) + { + PaxosUncommittedTracker.truncate(stateDirectory); + if (truncateBallotMetadata()) + PaxosBallotTracker.truncate(stateDirectory); + } + else + { + stateDirectory.createDirectoriesIfNotExists(); + } + } + + PaxosUncommittedTracker uncommitted = PaxosUncommittedTracker.load(stateDirectory); + PaxosBallotTracker ballots = PaxosBallotTracker.load(stateDirectory); + + if (!rebuildNeeded) + uncommitted.start(); + + return new PaxosStateTracker(uncommitted, ballots, rebuildNeeded); + } + + public static PaxosStateTracker create(Directories.DataDirectories dataDirectories) throws IOException + { + return create(dataDirectories.getAllDirectories().stream().map(d -> d.location).toArray(File[]::new)); + } + + @SuppressWarnings("resource") + private void rebuildUncommittedData() throws IOException + { + logger.info("Beginning uncommitted paxos data rebuild. Set -Dcassandra.skip_paxos_state_rebuild=true and restart to skip"); + + String queryStr = "SELECT * FROM " + SYSTEM_KEYSPACE_NAME + '.' + SystemKeyspace.PAXOS; + SelectStatement stmt = (SelectStatement) QueryProcessor.parseStatement(queryStr).prepare(ClientState.forInternalCalls()); + ReadQuery query = stmt.getQuery(QueryOptions.DEFAULT, FBUtilities.nowInSeconds()); + try (ReadExecutionController controller = query.executionController(); + PartitionIterator partitions = query.executeInternal(controller); + PaxosKeyStateRowsIterator rows = new PaxosKeyStateRowsIterator(partitions)) + { + uncommitted.rebuild(rows); + } + } + + class PaxosKeyStateRowsIterator implements CloseableIterator + { + // note: this is not closed by this iterator + final PartitionIterator partitions; + + RowIterator partition = null; + PaxosKeyState next = null; + + PaxosKeyStateRowsIterator(PartitionIterator partitions) + { + this.partitions = partitions; + } + + @Override + public boolean hasNext() + { + if (next != null) + return true; + + while (true) + { + if (partition != null && partition.hasNext()) + { + PaxosKeyState commitState = PaxosRows.getCommitState(partition.partitionKey(), partition.next(), null); + if (commitState == null) + continue; + ballots.updateHighBound(commitState.ballot); + if (!commitState.committed) + { + next = commitState; + return true; + } + } + else + { + if (partition != null) + { + partition.close(); + partition = null; + } + + if (!partitions.hasNext()) + return false; + + partition = partitions.next(); + } + } + } + + @Override + public PaxosKeyState next() + { + if (next == null && !hasNext()) + throw new NoSuchElementException(); + PaxosKeyState next = this.next; + this.next = null; + return next; + } + + @Override + public void close() + { + if (partition != null) + { + partition.close(); + partition = null; + } + } + } + + private void updateLowBoundFromRepairHistory() throws IOException + { + String queryStr = "SELECT * FROM " + SYSTEM_KEYSPACE_NAME + '.' + PAXOS_REPAIR_HISTORY; + SelectStatement stmt = (SelectStatement) QueryProcessor.parseStatement(queryStr).prepare(ClientState.forInternalCalls()); + ReadQuery query = stmt.getQuery(QueryOptions.DEFAULT, FBUtilities.nowInSeconds()); + + Ballot lowBound = null; + ListType listType = ListType.getInstance(BytesType.instance, false); + ColumnMetadata pointsColumn = ColumnMetadata.regularColumn(SYSTEM_KEYSPACE_NAME, PAXOS_REPAIR_HISTORY, "points", listType); + try (ReadExecutionController controller = query.executionController(); PartitionIterator partitions = query.executeInternal(controller)) + { + while (partitions.hasNext()) + { + try (RowIterator partition = partitions.next()) + { + String keyspaceName = UTF8Type.instance.compose(partition.partitionKey().getKey()); + if (Schema.instance.getKeyspaceMetadata(keyspaceName) == null) + continue; + + Keyspace.open(keyspaceName); + while (partition.hasNext()) + { + Row row = partition.next(); + Clustering clustering = row.clustering(); + String tableName = UTF8Type.instance.compose(clustering.get(0), clustering.accessor()); + if (Schema.instance.getTableMetadata(keyspaceName, tableName) == null) + continue; + + Cell pointsCell = row.getCell(pointsColumn); + List points = listType.compose(pointsCell.value(), pointsCell.accessor()); + PaxosRepairHistory history = PaxosRepairHistory.fromTupleBufferList(points); + lowBound = Commit.latest(lowBound, history.maxLowBound()); + } + } + } + } + ballots.updateLowBound(lowBound); + } + + public void maybeRebuild() throws IOException + { + if (!rebuildNeeded) + return; + + if (truncateBallotMetadata()) + { + logger.info("Truncating {}.{}", SYSTEM_KEYSPACE_NAME, PAXOS_REPAIR_HISTORY); + Keyspace.open(SYSTEM_KEYSPACE_NAME).getColumnFamilyStore(PAXOS_REPAIR_HISTORY).truncateBlocking(); + } + + if (!skipRebuild()) + { + rebuildUncommittedData(); + + if (!truncateBallotMetadata()) // no point doing this if we just truncated the repair history table + updateLowBoundFromRepairHistory(); + logger.info("Uncommitted paxos data rebuild completed"); + } + uncommitted.start(); + ballots.flush(); // explicitly flush since a missing ballot file on startup indicates a rebuild is needed + rebuildNeeded = false; + } + + public PaxosUncommittedTracker uncommitted() + { + return uncommitted; + } + + public PaxosBallotTracker ballots() + { + return ballots; + } +} diff --git a/src/java/org/apache/cassandra/service/paxos/uncommitted/PaxosUncommittedIndex.java b/src/java/org/apache/cassandra/service/paxos/uncommitted/PaxosUncommittedIndex.java new file mode 100644 index 0000000000..904d2f55cd --- /dev/null +++ b/src/java/org/apache/cassandra/service/paxos/uncommitted/PaxosUncommittedIndex.java @@ -0,0 +1,264 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.paxos.uncommitted; + +import java.util.*; +import java.util.concurrent.Callable; +import java.util.function.BiFunction; +import java.util.stream.Collectors; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.util.concurrent.Callables; + +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.cql3.Operator; +import org.apache.cassandra.db.*; +import org.apache.cassandra.db.filter.ColumnFilter; +import org.apache.cassandra.db.filter.RowFilter; +import org.apache.cassandra.db.lifecycle.View; +import org.apache.cassandra.db.marshal.AbstractType; +import org.apache.cassandra.db.partitions.*; +import org.apache.cassandra.db.rows.Row; +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.Token; +import org.apache.cassandra.exceptions.InvalidRequestException; +import org.apache.cassandra.index.Index; +import org.apache.cassandra.index.IndexRegistry; +import org.apache.cassandra.index.transactions.IndexTransaction; +import org.apache.cassandra.schema.ColumnMetadata; +import org.apache.cassandra.schema.IndexMetadata; +import org.apache.cassandra.schema.Indexes; +import org.apache.cassandra.schema.TableId; +import org.apache.cassandra.utils.CloseableIterator; + +import static java.util.Collections.*; +import static org.apache.cassandra.schema.SchemaConstants.SYSTEM_KEYSPACE_NAME; +import static org.apache.cassandra.service.paxos.PaxosState.ballotTracker; +import static org.apache.cassandra.service.paxos.PaxosState.uncommittedTracker; + +/** + * A 2i implementation made specifically for system.paxos that listens for changes to paxos state by interpreting + * mutations against system.paxos and updates the uncommitted tracker accordingly. + * + * No read expressions are supported by the index. + * + * This is implemented as a 2i so it can piggy back off the commit log and paxos table flushes, and avoid worrying + * about implementing a parallel log/flush system for the tracker and potential bugs there. It also means we don't + * have to worry about cases where the tracker can become out of sync with the paxos table due to failure/edge cases + * outside of the PaxosTableState class itself. + */ +public class PaxosUncommittedIndex implements Index, PaxosUncommittedTracker.UpdateSupplier +{ + public final ColumnFamilyStore baseCfs; + protected IndexMetadata metadata; + + private static final DataRange FULL_RANGE = DataRange.allData(DatabaseDescriptor.getPartitioner()); + private final ColumnFilter memtableColumnFilter; + + public PaxosUncommittedIndex(ColumnFamilyStore baseTable, IndexMetadata metadata) + { + Preconditions.checkState(baseTable.metadata.keyspace.equals(SYSTEM_KEYSPACE_NAME)); + Preconditions.checkState(baseTable.metadata.name.equals(SystemKeyspace.PAXOS)); + + this.baseCfs = baseTable; + this.metadata = metadata; + + this.memtableColumnFilter = ColumnFilter.all(baseTable.metadata.get()); + uncommittedTracker().unsafSetUpdateSupplier(this); + } + + public static IndexMetadata indexMetadata() + { + Map options = new HashMap<>(); + options.put("class_name", PaxosUncommittedIndex.class.getName()); + options.put("target", ""); + return IndexMetadata.fromSchemaMetadata("PaxosUncommittedIndex", IndexMetadata.Kind.CUSTOM, options); + } + + public static Indexes indexes() + { + return Indexes.builder().add(indexMetadata()).build(); + } + + public Callable getInitializationTask() + { + return Callables.returning(null); + } + + public IndexMetadata getIndexMetadata() + { + return metadata; + } + + public Callable getMetadataReloadTask(IndexMetadata indexMetadata) + { + return Callables.returning(null); + } + + public void register(IndexRegistry registry) + { + registry.registerIndex(this); + } + + public Optional getBackingTable() + { + return Optional.empty(); + } + + private CloseableIterator getPaxosUpdates(List iterators, TableId filterByTableId, boolean materializeLazily) + { + Preconditions.checkArgument((filterByTableId == null) == materializeLazily); + + return PaxosRows.toIterator(UnfilteredPartitionIterators.merge(iterators, UnfilteredPartitionIterators.MergeListener.NOOP), filterByTableId, materializeLazily); + } + + public CloseableIterator repairIterator(TableId tableId, Collection> ranges) + { + Preconditions.checkNotNull(tableId); + + View view = baseCfs.getTracker().getView(); + List memtables = view.flushingMemtables.isEmpty() + ? view.liveMemtables + : ImmutableList.builder().addAll(view.flushingMemtables).addAll(view.liveMemtables).build(); + + List dataRanges = ranges.stream().map(DataRange::forTokenRange).collect(Collectors.toList()); + List iters = new ArrayList<>(memtables.size() * ranges.size()); + + for (int j=0, jsize=dataRanges.size(); j flushIterator(Memtable flushing) + { + List iters = singletonList(flushing.makePartitionIterator(memtableColumnFilter, FULL_RANGE)); + return getPaxosUpdates(iters, null, true); + } + + public Callable getBlockingFlushTask() + { + return (Callable) () -> { + ballotTracker().flush(); + return null; + }; + } + + public Callable getBlockingFlushTask(Memtable paxos) + { + return (Callable) () -> { + uncommittedTracker().flushUpdates(paxos); + ballotTracker().flush(); + return null; + }; + } + + public Callable getInvalidateTask() + { + return (Callable) () -> { + uncommittedTracker().truncate(); + ballotTracker().truncate(); + return null; + }; + } + + public Callable getTruncateTask(long truncatedAt) + { + return (Callable) () -> { + uncommittedTracker().truncate(); + ballotTracker().truncate(); + return null; + }; + } + + public boolean shouldBuildBlocking() + { + return false; + } + + public boolean dependsOn(ColumnMetadata column) + { + return false; + } + + public boolean supportsExpression(ColumnMetadata column, Operator operator) + { + // should prevent this from ever being used + return false; + } + + public AbstractType customExpressionValueType() + { + return null; + } + + public RowFilter getPostIndexQueryFilter(RowFilter filter) + { + return null; + } + + public long getEstimatedResultRows() + { + return 0; + } + + public void validate(PartitionUpdate update) throws InvalidRequestException + { + + } + + public Indexer indexerFor(DecoratedKey key, RegularAndStaticColumns columns, int nowInSec, WriteContext ctx, IndexTransaction.Type transactionType) + { + return indexer; + } + + public BiFunction postProcessorFor(ReadCommand command) + { + return null; + } + + public Searcher searcherFor(ReadCommand command) + { + throw new UnsupportedOperationException(); + } + + private final Indexer indexer = new Indexer() + { + public void begin() {} + public void partitionDelete(DeletionTime deletionTime) {} + public void rangeTombstone(RangeTombstone tombstone) {} + + public void insertRow(Row row) + { + ballotTracker().onUpdate(row); + } + + public void updateRow(Row oldRowData, Row newRowData) + { + ballotTracker().onUpdate(newRowData); + } + + public void removeRow(Row row) {} + public void finish() {} + }; +} diff --git a/src/java/org/apache/cassandra/service/paxos/uncommitted/PaxosUncommittedTracker.java b/src/java/org/apache/cassandra/service/paxos/uncommitted/PaxosUncommittedTracker.java new file mode 100644 index 0000000000..50b0363314 --- /dev/null +++ b/src/java/org/apache/cassandra/service/paxos/uncommitted/PaxosUncommittedTracker.java @@ -0,0 +1,376 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.paxos.uncommitted; + +import java.io.IOException; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.TimeUnit; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Sets; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.concurrent.ScheduledExecutors; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.db.Memtable; +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.Token; +import org.apache.cassandra.io.util.File; +import org.apache.cassandra.schema.Schema; +import org.apache.cassandra.schema.SchemaConstants; +import org.apache.cassandra.schema.TableId; +import org.apache.cassandra.io.util.FileUtils; +import org.apache.cassandra.service.StorageService; +import org.apache.cassandra.service.paxos.cleanup.PaxosTableRepairs; +import org.apache.cassandra.utils.CloseableIterator; + +import static org.apache.cassandra.config.DatabaseDescriptor.paxosRepairEnabled; +import static org.apache.cassandra.service.paxos.uncommitted.PaxosKeyState.mergeUncommitted; + +/** + * Tracks uncommitted paxos operations to enable operation completion as part of repair by returning an iterator of + * partition keys with uncommitted paxos operations (and their consistency levels) for a given table and token range(s) + * + * There are 2 parts to the uncommitted states it tracks: operations flushed to disk, and updates still in memory. This + * class handles merging these two sources for queries and for merging states as part of flush. In practice, in memory + * updates are the contents of the system.paxos memtables, although this has been generalized into an "UpdateSupplier" + * interface to accomodate testing. + */ +public class PaxosUncommittedTracker +{ + private static final Logger logger = LoggerFactory.getLogger(PaxosUncommittedTracker.class); + private static final Range FULL_RANGE = new Range<>(DatabaseDescriptor.getPartitioner().getMinimumToken(), + DatabaseDescriptor.getPartitioner().getMinimumToken()); + + private volatile boolean autoRepairsEnabled = !Boolean.getBoolean("cassandra.disable_paxos_auto_repairs"); + private volatile boolean stateFlushEnabled = !Boolean.getBoolean("cassandra.disable_paxos_state_flush"); + + private boolean started = false; + private boolean autoRepairStarted = false; + + private final Set autoRepairTableIds = Sets.newConcurrentHashSet(); + + public interface UpdateSupplier + { + CloseableIterator repairIterator(TableId tableId, Collection> ranges); + CloseableIterator flushIterator(Memtable paxos); + } + + private final File dataDirectory; + private volatile ImmutableMap tableStates; + private volatile UpdateSupplier updateSupplier; + + public PaxosUncommittedTracker(File dataDirectory, ImmutableMap tableStates) + { + this.dataDirectory = dataDirectory; + this.tableStates = tableStates; + } + + public PaxosUncommittedTracker(File dataDirectory) + { + this(dataDirectory, ImmutableMap.of()); + } + + public File getDirectory() + { + return dataDirectory; + } + + public static void truncate(File dataDirectory) + { + logger.info("truncating paxos uncommitted metadata in {}", dataDirectory); + for (File file : dataDirectory.tryList()) + { + if (file.name().equals(PaxosBallotTracker.FNAME)) + continue; + + if (file.isDirectory()) + FileUtils.deleteRecursive(file); + else + FileUtils.deleteWithConfirm(file); + } + } + + public static PaxosUncommittedTracker load(File dataDirectory) + { + ImmutableMap.Builder builder = ImmutableMap.builder(); + for (TableId tableId : UncommittedTableData.listTableIds(dataDirectory)) + { + builder.put(tableId, UncommittedTableData.load(dataDirectory, tableId)); + } + + return new PaxosUncommittedTracker(dataDirectory, builder.build()); + } + + @VisibleForTesting + UncommittedTableData getOrCreateTableState(TableId tableId) + { + UncommittedTableData state = tableStates.get(tableId); + if (state == null) + { + synchronized (this) + { + state = tableStates.get(tableId); + if (state != null) + return state; + + state = UncommittedTableData.load(dataDirectory, tableId); + tableStates = ImmutableMap.builder() + .putAll(tableStates).put(tableId, state) + .build(); + } + } + return state; + } + + synchronized void flushUpdates(Memtable paxos) throws IOException + { + if (!stateFlushEnabled || !started) + return; + + Map flushWriters = new HashMap<>(); + try (CloseableIterator iterator = updateSupplier.flushIterator(paxos)) + { + while (iterator.hasNext()) + { + PaxosKeyState next = iterator.next(); + UncommittedTableData.FlushWriter writer = flushWriters.get(next.tableId); + if (writer == null) + { + writer = getOrCreateTableState(next.tableId).flushWriter(); + flushWriters.put(next.tableId, writer); + } + writer.append(next); + } + } + catch (Throwable t) + { + for (UncommittedTableData.FlushWriter writer : flushWriters.values()) + t = writer.abort(t); + throw new IOException(t); + } + + for (UncommittedTableData.FlushWriter writer : flushWriters.values()) + writer.finish(); + } + + @VisibleForTesting + UncommittedTableData getTableState(TableId tableId) + { + return tableStates.get(tableId); + } + + @SuppressWarnings("resource") + public CloseableIterator uncommittedKeyIterator(TableId tableId, Collection> ranges) + { + ranges = (ranges == null || ranges.isEmpty()) ? Collections.singleton(FULL_RANGE) : Range.normalize(ranges); + CloseableIterator updates = updateSupplier.repairIterator(tableId, ranges); + + try + { + UncommittedTableData state = tableStates.get(tableId); + if (state == null) + return PaxosKeyState.toUncommittedInfo(updates); + + CloseableIterator fileIter = state.iterator(ranges); + try + { + @SuppressWarnings("unchecked") CloseableIterator merged = mergeUncommitted(updates, fileIter); + + return PaxosKeyState.toUncommittedInfo(merged); + } + catch (Throwable t) + { + fileIter.close(); + throw t; + } + } + catch (Throwable t) + { + updates.close(); + throw t; + } + } + + synchronized void truncate() + { + logger.info("truncating paxos uncommitted info"); + tableStates.values().forEach(UncommittedTableData::truncate); + tableStates = ImmutableMap.of(); + } + + public synchronized void start() + { + if (started) + return; + + logger.info("enabling PaxosUncommittedTracker"); + started = true; + } + + public synchronized void rebuild(Iterator iterator) throws IOException + { + Preconditions.checkState(!started); + truncate(); + + Map flushWriters = new HashMap<>(); + try + { + while (iterator.hasNext()) + { + PaxosKeyState next = iterator.next(); + UncommittedTableData.FlushWriter writer = flushWriters.get(next.tableId); + if (writer == null) + { + writer = getOrCreateTableState(next.tableId).rebuildWriter(); + flushWriters.put(next.tableId, writer); + } + writer.append(next); + } + for (UncommittedTableData.FlushWriter writer : flushWriters.values()) + writer.finish(); + } + catch (Throwable t) + { + for (UncommittedTableData.FlushWriter writer : flushWriters.values()) + t = writer.abort(t); + throw new IOException(t); + } + + start(); + } + + synchronized void consolidateFiles() + { + tableStates.values().forEach(UncommittedTableData::maybeScheduleMerge); + } + + synchronized void schedulePaxosAutoRepairs() + { + if (!paxosRepairEnabled() || !autoRepairsEnabled) + return; + + for (UncommittedTableData tableData : tableStates.values()) + { + if (tableData.numFiles() == 0) + continue; + + if (SchemaConstants.REPLICATED_SYSTEM_KEYSPACE_NAMES.contains(tableData.keyspace())) + continue; + + TableId tableId = tableData.tableId(); + if (Schema.instance.getTableMetadata(tableId) == null) + continue; + + logger.debug("Starting paxos auto repair for {}.{}", tableData.keyspace(), tableData.table()); + + if (!autoRepairTableIds.add(tableId)) + { + logger.debug("Skipping paxos auto repair for {}.{}, another auto repair is already in progress", tableData.keyspace(), tableData.table()); + continue; + } + + StorageService.instance.autoRepairPaxos(tableId).addCallback((success, failure) -> { + if (failure != null) logger.error("Paxos auto repair for {}.{} failed", tableData.keyspace(), tableData.table(), failure); + else logger.debug("Paxos auto repair for {}.{} completed", tableData.keyspace(), tableData.table()); + autoRepairTableIds.remove(tableId); + }); + } + } + + private static void runAndLogException(String desc, Runnable runnable) + { + try + { + runnable.run(); + } + catch (Throwable t) + { + logger.error("Unhandled exception running " + desc, t); + } + } + + void maintenance() + { + runAndLogException("file consolidation", this::consolidateFiles); + runAndLogException("schedule auto repairs", this::schedulePaxosAutoRepairs); + runAndLogException("evict hung repairs", PaxosTableRepairs::evictHungRepairs); + } + + public synchronized void startAutoRepairs() + { + if (autoRepairStarted) + return; + int seconds = Integer.getInteger("cassandra.auto_repair_frequency_seconds", (int) TimeUnit.MINUTES.toSeconds(5)); + ScheduledExecutors.scheduledTasks.scheduleAtFixedRate(this::maintenance, seconds, seconds, TimeUnit.SECONDS); + autoRepairStarted = true; + } + + @VisibleForTesting + public boolean hasInflightAutoRepairs() + { + return !autoRepairTableIds.isEmpty(); + } + + public boolean isAutoRepairsEnabled() + { + return autoRepairsEnabled; + } + + public void setAutoRepairsEnabled(boolean autoRepairsEnabled) + { + this.autoRepairsEnabled = autoRepairsEnabled; + } + + public boolean isStateFlushEnabled() + { + return stateFlushEnabled; + } + + public void setStateFlushEnabled(boolean enabled) + { + this.stateFlushEnabled = enabled; + } + + public Set tableIds() + { + return tableStates.keySet(); + } + + public UpdateSupplier unsafGetUpdateSupplier() + { + return updateSupplier; + } + + public void unsafSetUpdateSupplier(UpdateSupplier updateSupplier) + { + Preconditions.checkArgument(updateSupplier != null); + this.updateSupplier = updateSupplier; + } + +} diff --git a/src/java/org/apache/cassandra/service/paxos/uncommitted/UncommittedDataFile.java b/src/java/org/apache/cassandra/service/paxos/uncommitted/UncommittedDataFile.java new file mode 100644 index 0000000000..931497c730 --- /dev/null +++ b/src/java/org/apache/cassandra/service/paxos/uncommitted/UncommittedDataFile.java @@ -0,0 +1,384 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.paxos.uncommitted; + +import java.io.IOException; +import java.nio.file.Files; +import java.util.Collection; +import java.util.HashSet; +import java.util.Iterator; +import java.util.NoSuchElementException; +import java.util.Set; +import java.util.UUID; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.collect.Iterables; +import com.google.common.collect.PeekingIterator; + +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.PartitionPosition; +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.Token; +import org.apache.cassandra.io.FSReadError; +import org.apache.cassandra.io.util.ChecksummedRandomAccessReader; +import org.apache.cassandra.io.util.ChecksummedSequentialWriter; +import org.apache.cassandra.io.util.File; +import org.apache.cassandra.io.util.RandomAccessReader; +import org.apache.cassandra.io.util.SequentialWriter; +import org.apache.cassandra.io.util.SequentialWriterOption; +import org.apache.cassandra.net.MessagingService; +import org.apache.cassandra.schema.TableId; +import org.apache.cassandra.service.paxos.Ballot; +import org.apache.cassandra.utils.AbstractIterator; +import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.utils.CloseableIterator; +import org.apache.cassandra.utils.Throwables; + +public class UncommittedDataFile +{ + static final String EXTENSION = "paxos"; + static final String TMP_SUFFIX = ".tmp"; + private static final int VERSION = 0; + final TableId tableId; + private final File file; + private final File crcFile; + private final long generation; + private int activeReaders = 0; + private boolean markedDeleted = false; + + private UncommittedDataFile(TableId tableId, File file, File crcFile, long generation) + { + this.tableId = tableId; + this.file = file; + this.crcFile = crcFile; + this.generation = generation; + } + + public static UncommittedDataFile create(TableId tableId, File file, File crcFile, long generation) + { + return new UncommittedDataFile(tableId, file, crcFile, generation); + } + + static Writer writer(File directory, String keyspace, String table, TableId tableId, long generation) throws IOException + { + return new Writer(directory, keyspace, table, tableId, generation); + } + + static Set listTableIds(File directory) + { + Pattern pattern = Pattern.compile(".*-([a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12})-(\\d+)\\." + EXTENSION + '$'); + Set tableIds = new HashSet<>(); + for (String fname : directory.listNamesUnchecked()) + { + Matcher matcher = pattern.matcher(fname); + if (matcher.matches()) + tableIds.add(TableId.fromUUID(UUID.fromString(matcher.group(1)))); + } + return tableIds; + } + + static Pattern fileRegexFor(TableId tableId) + { + return Pattern.compile(".*-" + tableId.toString() + "-(\\d+)\\." + EXTENSION + ".*"); + } + + static boolean isTmpFile(String fname) + { + return fname.endsWith(TMP_SUFFIX); + } + + static boolean isCrcFile(String fname) + { + return fname.endsWith(".crc"); + } + + static String fileName(String keyspace, String table, TableId tableId, long generation) + { + return String.format("%s-%s-%s-%s.%s", keyspace, table, tableId, generation, EXTENSION); + } + + static String crcName(String fname) + { + return fname + ".crc"; + } + + synchronized void markDeleted() + { + markedDeleted = true; + maybeDelete(); + } + + private void maybeDelete() + { + if (markedDeleted && activeReaders == 0) + { + file.delete(); + crcFile.delete(); + } + } + + synchronized private void onIteratorClose() + { + activeReaders--; + maybeDelete(); + } + + @VisibleForTesting + File file() + { + return file; + } + + @VisibleForTesting + int getActiveReaders() + { + return activeReaders; + } + + @VisibleForTesting + boolean isMarkedDeleted() + { + return markedDeleted; + } + + long generation() + { + return generation; + } + + /** + * Return an iterator of the file contents for the given token ranges. Token ranges + * must be normalized + */ + synchronized CloseableIterator iterator(Collection> ranges) + { + Preconditions.checkArgument(Iterables.elementsEqual(Range.normalize(ranges), ranges)); + if (markedDeleted) + return null; + activeReaders++; + return new KeyCommitStateIterator(ranges); + } + + private interface PeekingKeyCommitIterator extends CloseableIterator, PeekingIterator + { + static final PeekingKeyCommitIterator EMPTY = new PeekingKeyCommitIterator() + { + public PaxosKeyState peek() { throw new NoSuchElementException(); } + public void remove() { throw new NoSuchElementException(); } + public void close() { } + public boolean hasNext() { return false; } + public PaxosKeyState next() { throw new NoSuchElementException(); } + }; + } + + static class Writer + { + final File directory; + final String keyspace; + final String table; + final TableId tableId; + long generation; + + private final File file; + private final File crcFile; + final SequentialWriter writer; + DecoratedKey lastKey = null; + + private String fileName(long generation) + { + return UncommittedDataFile.fileName(keyspace, table, tableId, generation); + } + + private String crcName(long generation) + { + return UncommittedDataFile.crcName(fileName(generation)); + } + + Writer(File directory, String keyspace, String table, TableId tableId, long generation) throws IOException + { + this.directory = directory; + this.keyspace = keyspace; + this.table = table; + this.tableId = tableId; + this.generation = generation; + + directory.createDirectoriesIfNotExists(); + + this.file = new File(this.directory, fileName(generation) + TMP_SUFFIX); + this.crcFile = new File(this.directory, crcName(generation) + TMP_SUFFIX); + this.writer = new ChecksummedSequentialWriter(file, crcFile, null, SequentialWriterOption.DEFAULT); + this.writer.writeInt(VERSION); + } + + void append(PaxosKeyState state) throws IOException + { + if (lastKey != null) + Preconditions.checkArgument(state.key.compareTo(lastKey) > 0); + lastKey = state.key; + ByteBufferUtil.writeWithShortLength(state.key.getKey(), writer); + state.ballot.serialize(writer); + writer.writeBoolean(state.committed); + } + + Throwable abort(Throwable accumulate) + { + return writer.abort(accumulate); + } + + UncommittedDataFile finish() + { + writer.finish(); + File finalCrc = new File(directory, crcName(generation)); + File finalData = new File(directory, fileName(generation)); + try + { + crcFile.move(finalCrc); + file.move(finalData); + return new UncommittedDataFile(tableId, finalData, finalCrc, generation); + } + catch (Throwable e) + { + Throwable merged = e; + for (File f : new File[]{crcFile, finalCrc, file, finalData}) + { + try + { + if (f.exists()) + Files.delete(f.toPath()); + } + catch (Throwable t) + { + merged = Throwables.merge(merged, t); + } + } + + if (merged != e) + throw new RuntimeException(merged); + throw e; + } + } + } + + class KeyCommitStateIterator extends AbstractIterator implements PeekingKeyCommitIterator + { + private final Iterator> rangeIterator; + private final RandomAccessReader reader; + + private Range currentRange; + + KeyCommitStateIterator(Collection> ranges) + { + this.rangeIterator = ranges.iterator(); + try + { + this.reader = ChecksummedRandomAccessReader.open(file, crcFile); + } + catch (IOException e) + { + throw new FSReadError(e, file); + } + validateVersion(this.reader); + + Preconditions.checkArgument(rangeIterator.hasNext()); + currentRange = convertRange(rangeIterator.next()); + } + + private Range convertRange(Range tokenRange) + { + return new Range<>(tokenRange.left.maxKeyBound(), tokenRange.right.maxKeyBound()); + } + + private void validateVersion(RandomAccessReader reader) + { + try + { + int version = reader.readInt(); + Preconditions.checkArgument(version == VERSION, "unsupported file version: %s", version); + } + catch (IOException e) + { + throw new FSReadError(e, file); + } + } + + PaxosKeyState createKeyState(DecoratedKey key, RandomAccessReader reader) throws IOException + { + return new PaxosKeyState(tableId, key, + Ballot.deserialize(reader), + reader.readBoolean()); + } + + /** + * skip any bytes after the key + */ + void skipEntryRemainder(RandomAccessReader reader) throws IOException + { + reader.skipBytes((int) Ballot.sizeInBytes()); + reader.readBoolean(); + } + + protected synchronized PaxosKeyState computeNext() + { + try + { + nextKey: + while (!reader.isEOF()) + { + DecoratedKey key = DatabaseDescriptor.getPartitioner().decorateKey(ByteBufferUtil.readWithShortLength(reader)); + + while (!currentRange.contains(key)) + { + // if this falls before our current target range, just keep going + if (currentRange.left.compareTo(key) >= 0) + { + skipEntryRemainder(reader); + continue nextKey; + } + + // otherwise check against subsequent ranges and end iteration if there are none + if (!rangeIterator.hasNext()) + return endOfData(); + + currentRange = convertRange(rangeIterator.next()); + } + + return createKeyState(key, reader); + } + return endOfData(); + } + catch (IOException e) + { + throw new FSReadError(e, file); + } + } + + public void close() + { + synchronized (this) + { + reader.close(); + } + onIteratorClose(); + } + } +} diff --git a/src/java/org/apache/cassandra/service/paxos/uncommitted/UncommittedPaxosKey.java b/src/java/org/apache/cassandra/service/paxos/uncommitted/UncommittedPaxosKey.java new file mode 100644 index 0000000000..c78f94db85 --- /dev/null +++ b/src/java/org/apache/cassandra/service/paxos/uncommitted/UncommittedPaxosKey.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.paxos.uncommitted; + +import org.apache.cassandra.db.ConsistencyLevel; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.service.paxos.Ballot; + +public interface UncommittedPaxosKey +{ + DecoratedKey getKey(); + ConsistencyLevel getConsistencyLevel(); + Ballot ballot(); +} diff --git a/src/java/org/apache/cassandra/service/paxos/uncommitted/UncommittedTableData.java b/src/java/org/apache/cassandra/service/paxos/uncommitted/UncommittedTableData.java new file mode 100644 index 0000000000..92cdb85799 --- /dev/null +++ b/src/java/org/apache/cassandra/service/paxos/uncommitted/UncommittedTableData.java @@ -0,0 +1,617 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.paxos.uncommitted; + +import java.io.IOError; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.NavigableSet; +import java.util.Set; +import java.util.concurrent.ConcurrentSkipListSet; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import com.google.common.collect.Iterators; +import com.google.common.collect.PeekingIterator; +import com.google.common.collect.Sets; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.concurrent.ExecutorPlus; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.cql3.SchemaElement; +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.Token; +import org.apache.cassandra.io.FSReadError; +import org.apache.cassandra.io.util.File; +import org.apache.cassandra.schema.Schema; +import org.apache.cassandra.schema.TableId; +import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.service.StorageService; +import org.apache.cassandra.service.paxos.Ballot; +import org.apache.cassandra.service.paxos.Commit; +import org.apache.cassandra.service.paxos.PaxosRepairHistory; +import org.apache.cassandra.utils.AbstractIterator; +import org.apache.cassandra.utils.CloseableIterator; +import org.apache.cassandra.utils.ExecutorUtils; +import org.apache.cassandra.utils.FBUtilities; +import org.apache.cassandra.utils.MergeIterator; +import org.apache.cassandra.utils.Throwables; + +import static com.google.common.collect.Iterables.elementsEqual; +import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory; +import static org.apache.cassandra.service.paxos.uncommitted.UncommittedDataFile.isCrcFile; +import static org.apache.cassandra.service.paxos.uncommitted.UncommittedDataFile.isTmpFile; +import static org.apache.cassandra.service.paxos.uncommitted.UncommittedDataFile.writer; + +/** + * On memtable flush + */ +public class UncommittedTableData +{ + private static final Logger logger = LoggerFactory.getLogger(UncommittedTableData.class); + private static final Collection> FULL_RANGE; + + static + { + Token min = DatabaseDescriptor.getPartitioner().getMinimumToken(); + FULL_RANGE = Collections.singleton(new Range<>(min, min)); + } + + private static final SchemaElement UNKNOWN_TABLE = TableMetadata.minimal("UNKNOWN", "UNKNOWN"); + private static final ExecutorPlus executor = executorFactory().sequential("PaxosUncommittedMerge"); + + public interface FlushWriter + { + void append(PaxosKeyState commitState) throws IOException; + + void finish(); + + Throwable abort(Throwable accumulate); + + default void appendAll(Iterable states) throws IOException + { + for (PaxosKeyState state : states) + append(state); + } + } + + private static class FilteringIterator extends AbstractIterator implements CloseableIterator + { + private final CloseableIterator wrapped; + private final PeekingIterator peeking; + private final PeekingIterator> rangeIterator; + private final PaxosRepairHistory.Searcher historySearcher; + + FilteringIterator(CloseableIterator wrapped, List> ranges, PaxosRepairHistory history) + { + this.wrapped = wrapped; + this.peeking = Iterators.peekingIterator(wrapped); + this.rangeIterator = Iterators.peekingIterator(Range.normalize(ranges).iterator()); + this.historySearcher = history.searcher(); + } + + protected PaxosKeyState computeNext() + { + while (true) + { + if (!peeking.hasNext() || !rangeIterator.hasNext()) + return endOfData(); + + Range range = rangeIterator.peek(); + + Token token = peeking.peek().key.getToken(); + if (!range.contains(token)) + { + if (range.right.compareTo(token) < 0) + rangeIterator.next(); + else + peeking.next(); + continue; + } + + PaxosKeyState next = peeking.next(); + + Ballot lowBound = historySearcher.ballotForToken(token); + if (Commit.isAfter(lowBound, next.ballot)) + continue; + + return next; + } + } + + public void close() + { + wrapped.close(); + } + } + + static abstract class FilterFactory + { + abstract List> getReplicatedRanges(); + abstract PaxosRepairHistory getPaxosRepairHistory(); + + CloseableIterator filter(CloseableIterator iterator) + { + return new FilteringIterator(iterator, getReplicatedRanges(), getPaxosRepairHistory()); + } + } + + private static class CFSFilterFactory extends FilterFactory + { + private final TableId tableId; + + /** + * @param tableId must refer to a known CFS + */ + CFSFilterFactory(TableId tableId) + { + this.tableId = tableId; + } + + List> getReplicatedRanges() + { + if (tableId == null) + return Range.normalize(FULL_RANGE); + + ColumnFamilyStore table = Schema.instance.getColumnFamilyStoreInstance(tableId); + if (table == null) + return Range.normalize(FULL_RANGE); + + String ksName = table.keyspace.getName(); + List> ranges = StorageService.instance.getLocalAndPendingRanges(ksName); + + // don't filter anything if we're not aware of any locally replicated ranges + if (ranges.isEmpty()) + return Range.normalize(FULL_RANGE); + + return Range.normalize(ranges); + } + + PaxosRepairHistory getPaxosRepairHistory() + { + ColumnFamilyStore cfs = Schema.instance.getColumnFamilyStoreInstance(tableId); + if (cfs == null) + return PaxosRepairHistory.EMPTY; + + return cfs.getPaxosRepairHistory(); + } + } + + static class Data + { + final ImmutableSet files; + + Data(ImmutableSet files) + { + this.files = files; + } + + Data withFile(UncommittedDataFile file) + { + return new Data(ImmutableSet.builder().addAll(files).add(file).build()); + } + + void truncate() + { + for (UncommittedDataFile file : files) + file.markDeleted(); + } + } + + private static class Reducer extends MergeIterator.Reducer + { + PaxosKeyState merged = null; + + public void reduce(int idx, PaxosKeyState current) + { + merged = PaxosKeyState.merge(merged, current); + } + + protected PaxosKeyState getReduced() + { + return merged; + } + + protected void onKeyChange() + { + merged = null; + } + } + + @SuppressWarnings("resource") + private static CloseableIterator merge(Collection files, Collection> ranges) + { + List> iterators = new ArrayList<>(files.size()); + try + { + for (UncommittedDataFile file : files) + { + CloseableIterator iterator = file.iterator(ranges); + if (iterator == null) continue; + iterators.add(iterator); + } + return MergeIterator.get(iterators, PaxosKeyState.KEY_COMPARATOR, new Reducer()); + } + catch (Throwable t) + { + Throwables.close(t, iterators); + throw t; + } + } + + class Merge implements Runnable + { + final int generation; + boolean isScheduled = false; + + Merge(int generation) + { + this.generation = generation; + } + + public void run() + { + try + { + Preconditions.checkState(!dependsOnActiveFlushes()); + Data current = data; + SchemaElement name = tableName(tableId); + UncommittedDataFile.Writer writer = writer(directory, name.elementKeyspace(), name.elementName(), tableId, generation); + Set files = Sets.newHashSet(Iterables.filter(current.files, u -> u.generation() < generation)); + logger.info("merging {} paxos uncommitted files into a new generation {} file for {}.{}", files.size(), generation, keyspace(), table()); + try (CloseableIterator iterator = filterFactory.filter(merge(files, FULL_RANGE))) + { + while (iterator.hasNext()) + { + PaxosKeyState next = iterator.next(); + + if (next.committed) + continue; + + writer.append(next); + } + mergeComplete(this, writer.finish()); + } + } + catch (IOException e) + { + throw new IOError(e); + } + } + + void maybeSchedule() + { + if (isScheduled) + return; + + if (dependsOnActiveFlushes()) + return; + + executor.submit(merge); + merge.isScheduled = true; + } + + boolean dependsOnActiveFlushes() + { + return !activeFlushes.headSet(generation).isEmpty(); + } + } + + private final File directory; + private final TableId tableId; + private final FilterFactory filterFactory; + + private volatile Data data; + private volatile Merge merge; + private volatile boolean rebuilding = false; + + private int nextGeneration; + private final NavigableSet activeFlushes = new ConcurrentSkipListSet<>(); + + private UncommittedTableData(File directory, TableId tableId, FilterFactory filterFactory, Data data) + { + this.directory = directory; + this.tableId = tableId; + this.filterFactory = filterFactory; + this.data = data; + this.nextGeneration = 1 + (int) data.files.stream().mapToLong(UncommittedDataFile::generation).max().orElse(-1); + } + + static UncommittedTableData load(File directory, TableId tableId, FilterFactory flushFilterFactory) + { + Preconditions.checkArgument(directory.exists()); + Preconditions.checkArgument(directory.isDirectory()); + Preconditions.checkNotNull(tableId); + + String[] fnames = directory.tryListNames(); + Preconditions.checkArgument(fnames != null); + + Pattern pattern = UncommittedDataFile.fileRegexFor(tableId); + Set generations = new HashSet<>(); + List files = new ArrayList<>(); + for (String fname : fnames) + { + Matcher matcher = pattern.matcher(fname); + if (!matcher.matches()) + continue; + + File file = new File(directory, fname); + if (isTmpFile(fname)) + { + logger.info("deleting left over uncommitted paxos temp file {} for tableId {}", file, tableId); + file.delete(); + continue; + } + + if (isCrcFile(fname)) + continue; + + File crcFile = new File(directory, UncommittedDataFile.crcName(fname)); + if (!crcFile.exists()) + throw new FSReadError(new IOException(String.format("%s does not have a corresponding crc file", file)), crcFile); + long generation = Long.parseLong(matcher.group(1)); + files.add(UncommittedDataFile.create(tableId, file, crcFile, generation)); + generations.add(generation); + } + + // cleanup orphaned crc files + for (String fname : fnames) + { + if (!isCrcFile(fname)) + continue; + + Matcher matcher = pattern.matcher(fname); + if (!matcher.matches()) + continue; + + long generation = Long.parseLong(matcher.group(1)); + if (!generations.contains(generation)) + { + File file = new File(directory, fname); + logger.info("deleting left over uncommitted paxos crc file {} for tableId {}", file, tableId); + file.delete(); + } + } + + return new UncommittedTableData(directory, tableId, flushFilterFactory, new Data(ImmutableSet.copyOf(files))); + } + + static UncommittedTableData load(File directory, TableId tableId) + { + return load(directory, tableId, new CFSFilterFactory(tableId)); + } + + static Set listTableIds(File directory) + { + Preconditions.checkArgument(directory.isDirectory()); + return UncommittedDataFile.listTableIds(directory); + } + + private static SchemaElement tableName(TableId tableId) + { + TableMetadata name = Schema.instance.getTableMetadata(tableId); + return name != null ? name : UNKNOWN_TABLE; + } + + int numFiles() + { + return data.files.size(); + } + + TableId tableId() + { + return tableId; + } + + public String keyspace() + { + return tableName(tableId).elementKeyspace(); + } + + public String table() + { + return tableName(tableId).elementName(); + } + + /** + * Return an iterator of the file contents for the given token ranges. Token ranges + * must be normalized + */ + synchronized CloseableIterator iterator(Collection> ranges) + { + // we don't wait for pending flushes because flushing memtable data is added in PaxosUncommittedIndex + Preconditions.checkArgument(elementsEqual(Range.normalize(ranges), ranges)); + return filterFactory.filter(merge(data.files, ranges)); + } + + private void flushTerminated(int generation) + { + activeFlushes.remove(generation); + if (merge != null) + merge.maybeSchedule(); + } + + private synchronized void flushSuccess(int generation, UncommittedDataFile newFile) + { + assert newFile == null || generation == newFile.generation(); + if (newFile != null) + data = data.withFile(newFile); + flushTerminated(generation); + } + + private synchronized void flushAborted(int generation) + { + flushTerminated(generation); + } + + private synchronized void mergeComplete(Merge merge, UncommittedDataFile newFile) + { + Preconditions.checkArgument(this.merge == merge); + ImmutableSet.Builder files = ImmutableSet.builder(); + files.add(newFile); + for (UncommittedDataFile file : data.files) + { + if (file.generation() > merge.generation) + files.add(file); + else + file.markDeleted(); + } + + data = new Data(files.build()); + this.merge = null; + logger.info("paxos uncommitted merge completed for {}.{}, new generation {} file added", keyspace(), table(), newFile.generation()); + } + + synchronized FlushWriter flushWriter() throws IOException + { + int generation = nextGeneration++; + UncommittedDataFile.Writer writer = writer(directory, keyspace(), table(), tableId, generation); + activeFlushes.add(generation); + logger.info("flushing generation {} uncommitted paxos file for {}.{}", generation, keyspace(), table()); + + return new FlushWriter() + { + public void append(PaxosKeyState commitState) throws IOException + { + writer.append(commitState); + } + + public void finish() + { + flushSuccess(generation, writer.finish()); + } + + public Throwable abort(Throwable accumulate) + { + accumulate = writer.abort(accumulate); + flushAborted(generation); + return accumulate; + } + }; + } + + private synchronized void rebuildComplete(UncommittedDataFile file) + { + Preconditions.checkState(rebuilding); + Preconditions.checkState(!hasInProgressIO()); + Preconditions.checkState(data.files.isEmpty()); + + data = new Data(ImmutableSet.of(file)); + logger.info("paxos rebuild completed for {}.{}", keyspace(), table()); + rebuilding = false; + } + + synchronized FlushWriter rebuildWriter() throws IOException + { + Preconditions.checkState(!rebuilding); + Preconditions.checkState(nextGeneration == 0); + Preconditions.checkState(!hasInProgressIO()); + rebuilding = true; + int generation = nextGeneration++; + UncommittedDataFile.Writer writer = writer(directory, keyspace(), table(), tableId, generation); + + return new FlushWriter() + { + public void append(PaxosKeyState commitState) throws IOException + { + if (commitState.committed) + return; + + writer.append(commitState); + } + + public void finish() + { + rebuildComplete(writer.finish()); + } + + public Throwable abort(Throwable accumulate) + { + accumulate = writer.abort(accumulate); + logger.info("paxos rebuild aborted for {}.{}", keyspace(), table()); + rebuilding = false; + return accumulate; + } + }; + } + + synchronized void maybeScheduleMerge() + { + logger.info("Scheduling uncommitted paxos data merge task for {}.{}", keyspace(), table()); + if (data.files.size() < 2 || merge != null) + return; + + createMergeTask().maybeSchedule(); + } + + @VisibleForTesting + synchronized Merge createMergeTask() + { + Preconditions.checkState(merge == null); + merge = new Merge(nextGeneration++); + return merge; + } + + synchronized boolean hasInProgressIO() + { + return merge != null || !activeFlushes.isEmpty(); + } + + void truncate() + { + logger.info("truncating uncommitting paxos date for {}.{}", keyspace(), table()); + data.truncate(); + data = new Data(ImmutableSet.of()); + } + + @VisibleForTesting + Data data() + { + return data; + } + + @VisibleForTesting + long nextGeneration() + { + return nextGeneration; + } + + @VisibleForTesting + Merge currentMerge() + { + return merge; + } + + public static void shutdownAndWait(long timeout, TimeUnit units) throws InterruptedException, TimeoutException + { + ExecutorUtils.shutdownAndWait(timeout, units, executor); + } +} diff --git a/src/java/org/apache/cassandra/service/paxos/AbstractPaxosCallback.java b/src/java/org/apache/cassandra/service/paxos/v1/AbstractPaxosCallback.java similarity index 81% rename from src/java/org/apache/cassandra/service/paxos/AbstractPaxosCallback.java rename to src/java/org/apache/cassandra/service/paxos/v1/AbstractPaxosCallback.java index 54cdb1facb..e6ef1dac00 100644 --- a/src/java/org/apache/cassandra/service/paxos/AbstractPaxosCallback.java +++ b/src/java/org/apache/cassandra/service/paxos/v1/AbstractPaxosCallback.java @@ -1,5 +1,4 @@ /* - * * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -7,18 +6,16 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ -package org.apache.cassandra.service.paxos; +package org.apache.cassandra.service.paxos.v1; import org.apache.cassandra.utils.concurrent.CountDownLatch; @@ -50,7 +47,7 @@ public abstract class AbstractPaxosCallback implements RequestCallback public int getResponseCount() { - return (int) (targets - latch.count()); + return targets - latch.count(); } public void await() throws WriteTimeoutException diff --git a/src/java/org/apache/cassandra/service/paxos/PrepareCallback.java b/src/java/org/apache/cassandra/service/paxos/v1/PrepareCallback.java similarity index 56% rename from src/java/org/apache/cassandra/service/paxos/PrepareCallback.java rename to src/java/org/apache/cassandra/service/paxos/v1/PrepareCallback.java index 79be30e617..a6afc3a6f9 100644 --- a/src/java/org/apache/cassandra/service/paxos/PrepareCallback.java +++ b/src/java/org/apache/cassandra/service/paxos/v1/PrepareCallback.java @@ -1,6 +1,4 @@ -package org.apache.cassandra.service.paxos; /* - * * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -8,25 +6,22 @@ package org.apache.cassandra.service.paxos; * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ +package org.apache.cassandra.service.paxos.v1; + -import java.util.Collections; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.TimeUnit; -import com.google.common.base.Predicate; import com.google.common.collect.Iterables; import org.apache.cassandra.locator.InetAddressAndPort; @@ -36,11 +31,9 @@ import org.apache.cassandra.db.DecoratedKey; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.cassandra.db.SystemKeyspace; import org.apache.cassandra.net.Message; -import org.apache.cassandra.utils.UUIDGen; - -import static java.util.concurrent.TimeUnit.SECONDS; +import org.apache.cassandra.service.paxos.Commit; +import org.apache.cassandra.service.paxos.PrepareResponse; public class PrepareCallback extends AbstractPaxosCallback { @@ -85,27 +78,8 @@ public class PrepareCallback extends AbstractPaxosCallback latch.decrement(); } - public Iterable replicasMissingMostRecentCommit(TableMetadata metadata, int nowInSec) + public Iterable replicasMissingMostRecentCommit() { - // In general, we need every replicas that have answered to the prepare (a quorum) to agree on the MRC (see - // coment in StorageProxy.beginAndRepairPaxos(), but basically we need to make sure at least a quorum of nodes - // have learn a commit before commit a new one otherwise that previous commit is not guaranteed to have reach a - // quorum and further commit may proceed on incomplete information). - // However, if that commit is too hold, it may have been expired from some of the replicas paxos table (we don't - // keep the paxos state forever or that could grow unchecked), and we could end up in some infinite loop as - // explained on CASSANDRA-12043. To avoid that, we ignore a MRC that is too old, i.e. older than the TTL we set - // on paxos tables. For such old commit, we rely on hints and repair to ensure the commit has indeed be - // propagated to all nodes. - long paxosTtlSec = SystemKeyspace.paxosTtlSec(metadata); - if (mostRecentCommit.ballot.unix(SECONDS) + paxosTtlSec < nowInSec) - return Collections.emptySet(); - - return Iterables.filter(commitsByReplica.keySet(), new Predicate() - { - public boolean apply(InetAddressAndPort inetAddress) - { - return (!commitsByReplica.get(inetAddress).ballot.equals(mostRecentCommit.ballot)); - } - }); + return Iterables.filter(commitsByReplica.keySet(), inetAddress -> (!commitsByReplica.get(inetAddress).ballot.equals(mostRecentCommit.ballot))); } } diff --git a/src/java/org/apache/cassandra/service/paxos/PrepareVerbHandler.java b/src/java/org/apache/cassandra/service/paxos/v1/PrepareVerbHandler.java similarity index 60% rename from src/java/org/apache/cassandra/service/paxos/PrepareVerbHandler.java rename to src/java/org/apache/cassandra/service/paxos/v1/PrepareVerbHandler.java index 157630f277..2ebcfeb10f 100644 --- a/src/java/org/apache/cassandra/service/paxos/PrepareVerbHandler.java +++ b/src/java/org/apache/cassandra/service/paxos/v1/PrepareVerbHandler.java @@ -1,6 +1,4 @@ -package org.apache.cassandra.service.paxos; /* - * * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -8,20 +6,23 @@ package org.apache.cassandra.service.paxos; * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ + +package org.apache.cassandra.service.paxos.v1; import org.apache.cassandra.net.IVerbHandler; import org.apache.cassandra.net.Message; import org.apache.cassandra.net.MessagingService; +import org.apache.cassandra.service.paxos.Commit; +import org.apache.cassandra.service.paxos.PaxosState; +import org.apache.cassandra.service.paxos.PrepareResponse; public class PrepareVerbHandler implements IVerbHandler { @@ -29,7 +30,7 @@ public class PrepareVerbHandler implements IVerbHandler public static PrepareResponse doPrepare(Commit toPrepare) { - return PaxosState.prepare(toPrepare); + return PaxosState.legacyPrepare(toPrepare); } public void doVerb(Message message) diff --git a/src/java/org/apache/cassandra/service/paxos/ProposeCallback.java b/src/java/org/apache/cassandra/service/paxos/v1/ProposeCallback.java similarity index 86% rename from src/java/org/apache/cassandra/service/paxos/ProposeCallback.java rename to src/java/org/apache/cassandra/service/paxos/v1/ProposeCallback.java index 64eca68f0b..1c975cb30e 100644 --- a/src/java/org/apache/cassandra/service/paxos/ProposeCallback.java +++ b/src/java/org/apache/cassandra/service/paxos/v1/ProposeCallback.java @@ -1,6 +1,4 @@ -package org.apache.cassandra.service.paxos; /* - * * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -8,18 +6,18 @@ package org.apache.cassandra.service.paxos; * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ +package org.apache.cassandra.service.paxos.v1; + import java.util.concurrent.atomic.AtomicInteger; diff --git a/src/java/org/apache/cassandra/service/paxos/ProposeVerbHandler.java b/src/java/org/apache/cassandra/service/paxos/v1/ProposeVerbHandler.java similarity index 56% rename from src/java/org/apache/cassandra/service/paxos/ProposeVerbHandler.java rename to src/java/org/apache/cassandra/service/paxos/v1/ProposeVerbHandler.java index 5a20b67458..54c8c6771b 100644 --- a/src/java/org/apache/cassandra/service/paxos/ProposeVerbHandler.java +++ b/src/java/org/apache/cassandra/service/paxos/v1/ProposeVerbHandler.java @@ -1,6 +1,4 @@ -package org.apache.cassandra.service.paxos; /* - * * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -8,20 +6,22 @@ package org.apache.cassandra.service.paxos; * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ + +package org.apache.cassandra.service.paxos.v1; import org.apache.cassandra.net.IVerbHandler; import org.apache.cassandra.net.Message; import org.apache.cassandra.net.MessagingService; +import org.apache.cassandra.service.paxos.Commit; +import org.apache.cassandra.service.paxos.PaxosState; public class ProposeVerbHandler implements IVerbHandler { @@ -29,12 +29,13 @@ public class ProposeVerbHandler implements IVerbHandler public static Boolean doPropose(Commit proposal) { - return PaxosState.propose(proposal); + return PaxosState.legacyPropose(proposal); } public void doVerb(Message message) { - Message reply = message.responseWith(doPropose(message.payload)); + Boolean response = doPropose(message.payload); + Message reply = message.responseWith(response); MessagingService.instance().send(reply, message.from()); } } diff --git a/src/java/org/apache/cassandra/service/reads/AbstractReadExecutor.java b/src/java/org/apache/cassandra/service/reads/AbstractReadExecutor.java index 13627c8564..ea0b06c8f6 100644 --- a/src/java/org/apache/cassandra/service/reads/AbstractReadExecutor.java +++ b/src/java/org/apache/cassandra/service/reads/AbstractReadExecutor.java @@ -198,7 +198,7 @@ public abstract class AbstractReadExecutor // There are simply no extra replicas to speculate. // Handle this separately so it can record failed attempts to speculate due to lack of replicas - if (replicaPlan.contacts().size() == replicaPlan.candidates().size()) + if (replicaPlan.contacts().size() == replicaPlan.readCandidates().size()) { boolean recordFailedSpeculation = consistencyLevel != ConsistencyLevel.ALL; return new NeverSpeculatingReadExecutor(cfs, command, replicaPlan, queryStartNanoTime, recordFailedSpeculation); @@ -273,7 +273,7 @@ public abstract class AbstractReadExecutor // We're hitting additional targets for read repair (??). Since our "extra" replica is the least- // preferred by the snitch, we do an extra data read to start with against a replica more // likely to respond; better to let RR fail than the entire query. - super(cfs, command, replicaPlan, replicaPlan.blockFor() < replicaPlan.contacts().size() ? 2 : 1, queryStartNanoTime); + super(cfs, command, replicaPlan, replicaPlan.readQuorum() < replicaPlan.contacts().size() ? 2 : 1, queryStartNanoTime); } public void maybeTryAdditionalReplicas() @@ -367,7 +367,7 @@ public abstract class AbstractReadExecutor public void setResult(PartitionIterator result) { Preconditions.checkState(this.result == null, "Result can only be set once"); - this.result = DuplicateRowChecker.duringRead(result, this.replicaPlan.get().candidates().endpointList()); + this.result = DuplicateRowChecker.duringRead(result, this.replicaPlan.get().readCandidates().endpointList()); } /** diff --git a/src/java/org/apache/cassandra/service/reads/DataResolver.java b/src/java/org/apache/cassandra/service/reads/DataResolver.java index 6abb2add7b..e7e49521c1 100644 --- a/src/java/org/apache/cassandra/service/reads/DataResolver.java +++ b/src/java/org/apache/cassandra/service/reads/DataResolver.java @@ -21,8 +21,11 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.List; +import java.util.function.Supplier; import java.util.function.UnaryOperator; +import javax.annotation.Nullable; + import com.google.common.base.Joiner; import org.apache.cassandra.config.DatabaseDescriptor; @@ -50,24 +53,25 @@ import org.apache.cassandra.locator.ReplicaPlan; import org.apache.cassandra.net.Message; import org.apache.cassandra.schema.IndexMetadata; import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.service.reads.repair.NoopReadRepair; import org.apache.cassandra.service.reads.repair.ReadRepair; import org.apache.cassandra.service.reads.repair.RepairedDataTracker; import org.apache.cassandra.service.reads.repair.RepairedDataVerifier; import static com.google.common.collect.Iterables.*; -public class DataResolver, P extends ReplicaPlan.ForRead> extends ResponseResolver +public class DataResolver, P extends ReplicaPlan.ForRead> extends ResponseResolver { private final boolean enforceStrictLiveness; private final ReadRepair readRepair; private final boolean trackRepairedStatus; - public DataResolver(ReadCommand command, ReplicaPlan.Shared replicaPlan, ReadRepair readRepair, long queryStartNanoTime) + public DataResolver(ReadCommand command, Supplier replicaPlan, ReadRepair readRepair, long queryStartNanoTime) { this(command, replicaPlan, readRepair, queryStartNanoTime, false); } - public DataResolver(ReadCommand command, ReplicaPlan.Shared replicaPlan, ReadRepair readRepair, long queryStartNanoTime, boolean trackRepairedStatus) + public DataResolver(ReadCommand command, Supplier replicaPlan, ReadRepair readRepair, long queryStartNanoTime, boolean trackRepairedStatus) { super(command, replicaPlan, queryStartNanoTime); this.enforceStrictLiveness = command.metadata().enforceStrictLiveness(); @@ -87,13 +91,18 @@ public class DataResolver, P extends ReplicaPlan.ForRead< } public PartitionIterator resolve() + { + return resolve(null); + } + + public PartitionIterator resolve(@Nullable Runnable runOnShortRead) { // We could get more responses while this method runs, which is ok (we're happy to ignore any response not here // at the beginning of this method), so grab the response count once and use that through the method. Collection> messages = responses.snapshot(); assert !any(messages, msg -> msg.payload.isDigestResponse()); - E replicas = replicaPlan().candidates().select(transform(messages, Message::from), false); + E replicas = replicaPlan().readCandidates().select(transform(messages, Message::from), false); // If requested, inspect each response for a digest of the replica's repaired data set RepairedDataTracker repairedDataTracker = trackRepairedStatus @@ -115,7 +124,7 @@ public class DataResolver, P extends ReplicaPlan.ForRead< { ResolveContext context = new ResolveContext(replicas); return resolveWithReadRepair(context, - i -> shortReadProtectedResponse(i, context), + i -> shortReadProtectedResponse(i, context, runOnShortRead), UnaryOperator.identity(), repairedDataTracker); } @@ -181,13 +190,13 @@ public class DataResolver, P extends ReplicaPlan.ForRead< UnfilteredPartitionIterator getResponse(int i); } - private UnfilteredPartitionIterator shortReadProtectedResponse(int i, ResolveContext context) + private UnfilteredPartitionIterator shortReadProtectedResponse(int i, ResolveContext context, @Nullable Runnable onShortRead) { UnfilteredPartitionIterator originalResponse = responses.get(i).payload.makeIterator(command); return context.needShortReadProtection() ? ShortReadProtection.extend(context.replicas.get(i), - () -> responses.clearUnsafe(i), + () -> { responses.clearUnsafe(i); if (onShortRead != null) onShortRead.run(); }, originalResponse, command, context.mergedResultCounter, @@ -202,9 +211,9 @@ public class DataResolver, P extends ReplicaPlan.ForRead< RepairedDataTracker repairedDataTracker) { UnfilteredPartitionIterators.MergeListener listener = null; - if (context.needsReadRepair()) + if (context.needsReadRepair() && readRepair != NoopReadRepair.instance) { - P sources = replicaPlan.getWithContacts(context.replicas); + P sources = replicaPlan.get().withContacts(context.replicas); listener = wrapMergeListener(readRepair.getMergeListener(sources), sources, repairedDataTracker); } @@ -244,7 +253,7 @@ public class DataResolver, P extends ReplicaPlan.ForRead< PartitionIterator firstPhasePartitions = resolveInternal(firstPhaseContext, rfp.mergeController(), - i -> shortReadProtectedResponse(i, firstPhaseContext), + i -> shortReadProtectedResponse(i, firstPhaseContext, null), UnaryOperator.identity()); PartitionIterator completedPartitions = resolveWithReadRepair(secondPhaseContext, @@ -285,6 +294,7 @@ public class DataResolver, P extends ReplicaPlan.ForRead< Filter filter = new Filter(command.nowInSec(), command.metadata().enforceStrictLiveness()); FilteredPartitions filtered = FilteredPartitions.filter(merged, filter); PartitionIterator counted = Transformation.apply(preCountFilter.apply(filtered), context.mergedResultCounter); + return Transformation.apply(counted, new EmptyPartitionsDiscarder()); } diff --git a/src/java/org/apache/cassandra/service/reads/DigestResolver.java b/src/java/org/apache/cassandra/service/reads/DigestResolver.java index f66edea5ca..f79937421e 100644 --- a/src/java/org/apache/cassandra/service/reads/DigestResolver.java +++ b/src/java/org/apache/cassandra/service/reads/DigestResolver.java @@ -40,7 +40,7 @@ import org.apache.cassandra.utils.ByteBufferUtil; import static com.google.common.collect.Iterables.any; import static org.apache.cassandra.utils.Clock.Global.nanoTime; -public class DigestResolver, P extends ReplicaPlan.ForRead> extends ResponseResolver +public class DigestResolver, P extends ReplicaPlan.ForRead> extends ResponseResolver { private volatile Message dataResponse; diff --git a/src/java/org/apache/cassandra/service/reads/ReadCallback.java b/src/java/org/apache/cassandra/service/reads/ReadCallback.java index 802616214e..9b42296857 100644 --- a/src/java/org/apache/cassandra/service/reads/ReadCallback.java +++ b/src/java/org/apache/cassandra/service/reads/ReadCallback.java @@ -55,7 +55,7 @@ import static org.apache.cassandra.tracing.Tracing.isTracing; import static org.apache.cassandra.utils.Clock.Global.nanoTime; import static org.apache.cassandra.utils.concurrent.Condition.newOneTimeCondition; -public class ReadCallback, P extends ReplicaPlan.ForRead> implements RequestCallback +public class ReadCallback, P extends ReplicaPlan.ForRead> implements RequestCallback { protected static final Logger logger = LoggerFactory.getLogger(ReadCallback.class); @@ -81,7 +81,7 @@ public class ReadCallback, P extends ReplicaPlan.ForRead< this.resolver = resolver; this.queryStartNanoTime = queryStartNanoTime; this.replicaPlan = replicaPlan; - this.blockFor = replicaPlan.get().blockFor(); + this.blockFor = replicaPlan.get().readQuorum(); this.failureReasonByEndpoint = new ConcurrentHashMap<>(); // we don't support read repair (or rapid read protection) for range scans yet (CASSANDRA-6897) assert !(command instanceof PartitionRangeReadCommand) || blockFor >= replicaPlan().contacts().size(); diff --git a/src/java/org/apache/cassandra/service/reads/ResponseResolver.java b/src/java/org/apache/cassandra/service/reads/ResponseResolver.java index 6ae19ace9e..02e565d536 100644 --- a/src/java/org/apache/cassandra/service/reads/ResponseResolver.java +++ b/src/java/org/apache/cassandra/service/reads/ResponseResolver.java @@ -17,6 +17,8 @@ */ package org.apache.cassandra.service.reads; +import java.util.function.Supplier; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -27,22 +29,23 @@ import org.apache.cassandra.locator.ReplicaPlan; import org.apache.cassandra.net.Message; import org.apache.cassandra.utils.concurrent.Accumulator; -public abstract class ResponseResolver, P extends ReplicaPlan.ForRead> +public abstract class ResponseResolver, P extends ReplicaPlan.ForRead> { protected static final Logger logger = LoggerFactory.getLogger(ResponseResolver.class); protected final ReadCommand command; - protected final ReplicaPlan.Shared replicaPlan; + // TODO: this doesn't need to be a full ReplicaPlan; just a replica collection + protected final Supplier replicaPlan; // Accumulator gives us non-blocking thread-safety with optimal algorithmic constraints protected final Accumulator> responses; protected final long queryStartNanoTime; - public ResponseResolver(ReadCommand command, ReplicaPlan.Shared replicaPlan, long queryStartNanoTime) + public ResponseResolver(ReadCommand command, Supplier replicaPlan, long queryStartNanoTime) { this.command = command; this.replicaPlan = replicaPlan; - this.responses = new Accumulator<>(replicaPlan.get().candidates().size()); + this.responses = new Accumulator<>(replicaPlan.get().readCandidates().size()); this.queryStartNanoTime = queryStartNanoTime; } diff --git a/src/java/org/apache/cassandra/service/reads/ShortReadPartitionsProtection.java b/src/java/org/apache/cassandra/service/reads/ShortReadPartitionsProtection.java index 51043c352f..f49c9fb3a8 100644 --- a/src/java/org/apache/cassandra/service/reads/ShortReadPartitionsProtection.java +++ b/src/java/org/apache/cassandra/service/reads/ShortReadPartitionsProtection.java @@ -174,7 +174,7 @@ public class ShortReadPartitionsProtection extends Transformation, P extends ReplicaPlan.ForRead> + private , P extends ReplicaPlan.ForRead> UnfilteredPartitionIterator executeReadCommand(ReadCommand cmd, ReplicaPlan.Shared replicaPlan) { DataResolver resolver = new DataResolver<>(cmd, replicaPlan, (NoopReadRepair)NoopReadRepair.instance, queryStartNanoTime); diff --git a/src/java/org/apache/cassandra/service/reads/repair/AbstractReadRepair.java b/src/java/org/apache/cassandra/service/reads/repair/AbstractReadRepair.java index 04877c99b9..04788c0762 100644 --- a/src/java/org/apache/cassandra/service/reads/repair/AbstractReadRepair.java +++ b/src/java/org/apache/cassandra/service/reads/repair/AbstractReadRepair.java @@ -49,7 +49,8 @@ import org.apache.cassandra.tracing.Tracing; import static java.util.concurrent.TimeUnit.NANOSECONDS; -public abstract class AbstractReadRepair, P extends ReplicaPlan.ForRead> implements ReadRepair +public abstract class AbstractReadRepair, P extends ReplicaPlan.ForRead> + implements ReadRepair { protected static final Logger logger = LoggerFactory.getLogger(AbstractReadRepair.class); @@ -60,7 +61,7 @@ public abstract class AbstractReadRepair, P extends Repli private volatile DigestRepair digestRepair = null; - private static class DigestRepair, P extends ReplicaPlan.ForRead> + private static class DigestRepair, P extends ReplicaPlan.ForRead> { private final DataResolver dataResolver; private final ReadCallback readCallback; diff --git a/src/java/org/apache/cassandra/service/reads/repair/BlockingPartitionRepair.java b/src/java/org/apache/cassandra/service/reads/repair/BlockingPartitionRepair.java index df42f58f64..c8b0e29145 100644 --- a/src/java/org/apache/cassandra/service/reads/repair/BlockingPartitionRepair.java +++ b/src/java/org/apache/cassandra/service/reads/repair/BlockingPartitionRepair.java @@ -44,7 +44,7 @@ import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.locator.Replica; import org.apache.cassandra.locator.ReplicaPlan; import org.apache.cassandra.locator.Replicas; -import org.apache.cassandra.locator.InOurDcTester; +import org.apache.cassandra.locator.InOurDc; import org.apache.cassandra.metrics.ReadRepairMetrics; import org.apache.cassandra.net.RequestCallback; import org.apache.cassandra.net.Message; @@ -61,26 +61,26 @@ public class BlockingPartitionRepair extends AsyncFuture implements RequestCallback { private final DecoratedKey key; - private final ReplicaPlan.ForTokenWrite writePlan; + private final ReplicaPlan.ForWrite writePlan; private final Map pendingRepairs; private final CountDownLatch latch; private final Predicate shouldBlockOn; private volatile long mutationsSentTime; - public BlockingPartitionRepair(DecoratedKey key, Map repairs, ReplicaPlan.ForTokenWrite writePlan) + public BlockingPartitionRepair(DecoratedKey key, Map repairs, ReplicaPlan.ForWrite writePlan) { this(key, repairs, writePlan, - writePlan.consistencyLevel().isDatacenterLocal() ? InOurDcTester.endpoints() : Predicates.alwaysTrue()); + writePlan.consistencyLevel().isDatacenterLocal() ? InOurDc.endpoints() : Predicates.alwaysTrue()); } - public BlockingPartitionRepair(DecoratedKey key, Map repairs, ReplicaPlan.ForTokenWrite writePlan, Predicate shouldBlockOn) + public BlockingPartitionRepair(DecoratedKey key, Map repairs, ReplicaPlan.ForWrite writePlan, Predicate shouldBlockOn) { this.key = key; this.pendingRepairs = new ConcurrentHashMap<>(repairs); this.writePlan = writePlan; this.shouldBlockOn = shouldBlockOn; - int blockFor = writePlan.blockFor(); + int blockFor = writePlan.writeQuorum(); // here we remove empty repair mutations from the block for total, since // we're not sending them mutations for (Replica participant : writePlan.contacts()) @@ -101,7 +101,7 @@ public class BlockingPartitionRepair int blockFor() { - return writePlan.blockFor(); + return writePlan.writeQuorum(); } @VisibleForTesting diff --git a/src/java/org/apache/cassandra/service/reads/repair/BlockingReadRepair.java b/src/java/org/apache/cassandra/service/reads/repair/BlockingReadRepair.java index fdc8b50064..44092bd1d0 100644 --- a/src/java/org/apache/cassandra/service/reads/repair/BlockingReadRepair.java +++ b/src/java/org/apache/cassandra/service/reads/repair/BlockingReadRepair.java @@ -46,7 +46,7 @@ import static java.util.concurrent.TimeUnit.NANOSECONDS; * updates have been written to nodes needing correction. Breaks write * atomicity in some situations */ -public class BlockingReadRepair, P extends ReplicaPlan.ForRead> +public class BlockingReadRepair, P extends ReplicaPlan.ForRead> extends AbstractReadRepair { private static final Logger logger = LoggerFactory.getLogger(BlockingReadRepair.class); @@ -106,7 +106,7 @@ public class BlockingReadRepair, P extends ReplicaPlan.Fo } @Override - public void repairPartition(DecoratedKey partitionKey, Map mutations, ReplicaPlan.ForTokenWrite writePlan) + public void repairPartition(DecoratedKey partitionKey, Map mutations, ReplicaPlan.ForWrite writePlan) { BlockingPartitionRepair blockingRepair = new BlockingPartitionRepair(partitionKey, mutations, writePlan); blockingRepair.sendInitialRepairs(); diff --git a/src/java/org/apache/cassandra/service/reads/repair/NoopReadRepair.java b/src/java/org/apache/cassandra/service/reads/repair/NoopReadRepair.java index b65f3fcbc0..5cf72b33cf 100644 --- a/src/java/org/apache/cassandra/service/reads/repair/NoopReadRepair.java +++ b/src/java/org/apache/cassandra/service/reads/repair/NoopReadRepair.java @@ -34,7 +34,7 @@ import org.apache.cassandra.service.reads.DigestResolver; /** * Bypasses the read repair path for short read protection and testing */ -public class NoopReadRepair, P extends ReplicaPlan.ForRead> implements ReadRepair +public class NoopReadRepair, P extends ReplicaPlan.ForRead> implements ReadRepair { public static final NoopReadRepair instance = new NoopReadRepair(); @@ -75,7 +75,7 @@ public class NoopReadRepair, P extends ReplicaPlan.ForRea } @Override - public void repairPartition(DecoratedKey partitionKey, Map mutations, ReplicaPlan.ForTokenWrite writePlan) + public void repairPartition(DecoratedKey partitionKey, Map mutations, ReplicaPlan.ForWrite writePlan) { } diff --git a/src/java/org/apache/cassandra/service/reads/repair/PartitionIteratorMergeListener.java b/src/java/org/apache/cassandra/service/reads/repair/PartitionIteratorMergeListener.java index 9992bd5bcb..f77bd4d52c 100644 --- a/src/java/org/apache/cassandra/service/reads/repair/PartitionIteratorMergeListener.java +++ b/src/java/org/apache/cassandra/service/reads/repair/PartitionIteratorMergeListener.java @@ -33,11 +33,11 @@ import org.apache.cassandra.locator.ReplicaPlan; public class PartitionIteratorMergeListener> implements UnfilteredPartitionIterators.MergeListener { - private final ReplicaPlan.ForRead replicaPlan; + private final ReplicaPlan.ForRead replicaPlan; private final ReadCommand command; private final ReadRepair readRepair; - public PartitionIteratorMergeListener(ReplicaPlan.ForRead replicaPlan, ReadCommand command, ReadRepair readRepair) + public PartitionIteratorMergeListener(ReplicaPlan.ForRead replicaPlan, ReadCommand command, ReadRepair readRepair) { this.replicaPlan = replicaPlan; this.command = command; diff --git a/src/java/org/apache/cassandra/service/reads/repair/ReadOnlyReadRepair.java b/src/java/org/apache/cassandra/service/reads/repair/ReadOnlyReadRepair.java index d9293fb57b..72a12980e7 100644 --- a/src/java/org/apache/cassandra/service/reads/repair/ReadOnlyReadRepair.java +++ b/src/java/org/apache/cassandra/service/reads/repair/ReadOnlyReadRepair.java @@ -34,7 +34,7 @@ import org.apache.cassandra.metrics.ReadRepairMetrics; * Only performs the collection of data responses and reconciliation of them, doesn't send repair mutations * to replicas. This preserves write atomicity, but doesn't provide monotonic quorum reads */ -public class ReadOnlyReadRepair, P extends ReplicaPlan.ForRead> +public class ReadOnlyReadRepair, P extends ReplicaPlan.ForRead> extends AbstractReadRepair { ReadOnlyReadRepair(ReadCommand command, ReplicaPlan.Shared replicaPlan, long queryStartNanoTime) @@ -61,7 +61,7 @@ public class ReadOnlyReadRepair, P extends ReplicaPlan.Fo } @Override - public void repairPartition(DecoratedKey partitionKey, Map mutations, ReplicaPlan.ForTokenWrite writePlan) + public void repairPartition(DecoratedKey partitionKey, Map mutations, ReplicaPlan.ForWrite writePlan) { throw new UnsupportedOperationException("ReadOnlyReadRepair shouldn't be trying to repair partitions"); } diff --git a/src/java/org/apache/cassandra/service/reads/repair/ReadRepair.java b/src/java/org/apache/cassandra/service/reads/repair/ReadRepair.java index 9dcb969c36..a6a9be29f9 100644 --- a/src/java/org/apache/cassandra/service/reads/repair/ReadRepair.java +++ b/src/java/org/apache/cassandra/service/reads/repair/ReadRepair.java @@ -32,15 +32,15 @@ import org.apache.cassandra.locator.Replica; import org.apache.cassandra.locator.ReplicaPlan; import org.apache.cassandra.service.reads.DigestResolver; -public interface ReadRepair, P extends ReplicaPlan.ForRead> +public interface ReadRepair, P extends ReplicaPlan.ForRead> { public interface Factory { - , P extends ReplicaPlan.ForRead> + , P extends ReplicaPlan.ForRead> ReadRepair create(ReadCommand command, ReplicaPlan.Shared replicaPlan, long queryStartNanoTime); } - static , P extends ReplicaPlan.ForRead> + static , P extends ReplicaPlan.ForRead> ReadRepair create(ReadCommand command, ReplicaPlan.Shared replicaPlan, long queryStartNanoTime) { return command.metadata().params.readRepair.create(command, replicaPlan, queryStartNanoTime); @@ -92,5 +92,5 @@ public interface ReadRepair, P extends ReplicaPlan.ForRea * Repairs a partition _after_ receiving data responses. This method receives replica list, since * we will block repair only on the replicas that have responded. */ - void repairPartition(DecoratedKey partitionKey, Map mutations, ReplicaPlan.ForTokenWrite writePlan); + void repairPartition(DecoratedKey partitionKey, Map mutations, ReplicaPlan.ForWrite writePlan); } diff --git a/src/java/org/apache/cassandra/service/reads/repair/ReadRepairDiagnostics.java b/src/java/org/apache/cassandra/service/reads/repair/ReadRepairDiagnostics.java index b883a885b7..0443c3f602 100644 --- a/src/java/org/apache/cassandra/service/reads/repair/ReadRepairDiagnostics.java +++ b/src/java/org/apache/cassandra/service/reads/repair/ReadRepairDiagnostics.java @@ -38,22 +38,22 @@ final class ReadRepairDiagnostics { } - static void startRepair(AbstractReadRepair readRepair, ReplicaPlan.ForRead fullPlan, DigestResolver digestResolver) + static void startRepair(AbstractReadRepair readRepair, ReplicaPlan.ForRead fullPlan, DigestResolver digestResolver) { if (service.isEnabled(ReadRepairEvent.class, ReadRepairEventType.START_REPAIR)) service.publish(new ReadRepairEvent(ReadRepairEventType.START_REPAIR, readRepair, fullPlan.contacts().endpoints(), - fullPlan.candidates().endpoints(), digestResolver)); + fullPlan.readCandidates().endpoints(), digestResolver)); } static void speculatedRead(AbstractReadRepair readRepair, InetAddressAndPort endpoint, - ReplicaPlan.ForRead fullPlan) + ReplicaPlan.ForRead fullPlan) { if (service.isEnabled(ReadRepairEvent.class, ReadRepairEventType.SPECULATED_READ)) service.publish(new ReadRepairEvent(ReadRepairEventType.SPECULATED_READ, readRepair, Collections.singletonList(endpoint), - Lists.newArrayList(fullPlan.candidates().endpoints()), null)); + Lists.newArrayList(fullPlan.readCandidates().endpoints()), null)); } static void sendInitialRepair(BlockingPartitionRepair partitionRepair, InetAddressAndPort destination, Mutation mutation) diff --git a/src/java/org/apache/cassandra/service/reads/repair/ReadRepairStrategy.java b/src/java/org/apache/cassandra/service/reads/repair/ReadRepairStrategy.java index 8e0c1b54da..0d9caade35 100644 --- a/src/java/org/apache/cassandra/service/reads/repair/ReadRepairStrategy.java +++ b/src/java/org/apache/cassandra/service/reads/repair/ReadRepairStrategy.java @@ -26,7 +26,7 @@ public enum ReadRepairStrategy implements ReadRepair.Factory { NONE { - public , P extends ReplicaPlan.ForRead> + public , P extends ReplicaPlan.ForRead> ReadRepair create(ReadCommand command, ReplicaPlan.Shared replicaPlan, long queryStartNanoTime) { return new ReadOnlyReadRepair<>(command, replicaPlan, queryStartNanoTime); @@ -35,7 +35,7 @@ public enum ReadRepairStrategy implements ReadRepair.Factory BLOCKING { - public , P extends ReplicaPlan.ForRead> + public , P extends ReplicaPlan.ForRead> ReadRepair create(ReadCommand command, ReplicaPlan.Shared replicaPlan, long queryStartNanoTime) { return new BlockingReadRepair<>(command, replicaPlan, queryStartNanoTime); diff --git a/src/java/org/apache/cassandra/service/reads/repair/RowIteratorMergeListener.java b/src/java/org/apache/cassandra/service/reads/repair/RowIteratorMergeListener.java index 38d077ae8c..079080ab6f 100644 --- a/src/java/org/apache/cassandra/service/reads/repair/RowIteratorMergeListener.java +++ b/src/java/org/apache/cassandra/service/reads/repair/RowIteratorMergeListener.java @@ -68,8 +68,8 @@ public class RowIteratorMergeListener> private final PartitionUpdate.Builder[] repairs; private final Row.Builder[] currentRows; private final RowDiffListener diffListener; - private final ReplicaPlan.ForRead readPlan; - private final ReplicaPlan.ForTokenWrite writePlan; + private final ReplicaPlan.ForRead readPlan; + private final ReplicaPlan.ForWrite writePlan; // The partition level deletion for the merge row. private DeletionTime partitionLevelDeletion; @@ -82,7 +82,7 @@ public class RowIteratorMergeListener> private final ReadRepair readRepair; - public RowIteratorMergeListener(DecoratedKey partitionKey, RegularAndStaticColumns columns, boolean isReversed, ReplicaPlan.ForRead readPlan, ReadCommand command, ReadRepair readRepair) + public RowIteratorMergeListener(DecoratedKey partitionKey, RegularAndStaticColumns columns, boolean isReversed, ReplicaPlan.ForRead readPlan, ReadCommand command, ReadRepair readRepair) { this.partitionKey = partitionKey; this.columns = columns; diff --git a/src/java/org/apache/cassandra/streaming/StreamManager.java b/src/java/org/apache/cassandra/streaming/StreamManager.java index 8dcf2dd664..e258365376 100644 --- a/src/java/org/apache/cassandra/streaming/StreamManager.java +++ b/src/java/org/apache/cassandra/streaming/StreamManager.java @@ -278,7 +278,7 @@ public class StreamManager implements StreamManagerMBean return states.asMap().values(); } - public StreamingState getStreamingState(UUID id) + public StreamingState getStreamingState(TimeUUID id) { return states.getIfPresent(id); } diff --git a/src/java/org/apache/cassandra/streaming/StreamTransferTask.java b/src/java/org/apache/cassandra/streaming/StreamTransferTask.java index 980193b5f2..2e4d8bdc44 100644 --- a/src/java/org/apache/cassandra/streaming/StreamTransferTask.java +++ b/src/java/org/apache/cassandra/streaming/StreamTransferTask.java @@ -35,10 +35,9 @@ import org.slf4j.LoggerFactory; import org.apache.cassandra.concurrent.ScheduledExecutorPlus; import org.apache.cassandra.schema.TableId; import org.apache.cassandra.streaming.messages.OutgoingStreamMessage; +import org.apache.cassandra.utils.ExecutorUtils; import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory; -import static org.apache.cassandra.utils.ExecutorUtils.awaitTermination; -import static org.apache.cassandra.utils.ExecutorUtils.shutdown; /** * StreamTransferTask sends streams for a given table @@ -198,7 +197,6 @@ public class StreamTransferTask extends StreamTask @VisibleForTesting public static void shutdownAndWait(long timeout, TimeUnit units) throws InterruptedException, TimeoutException { - shutdown(timeoutExecutor); - awaitTermination(timeout, units, timeoutExecutor); + ExecutorUtils.shutdownAndWait(timeout, units, timeoutExecutor); } } diff --git a/src/java/org/apache/cassandra/streaming/async/StreamCompressionSerializer.java b/src/java/org/apache/cassandra/streaming/async/StreamCompressionSerializer.java index e34b2bdb86..f7d8101a8d 100644 --- a/src/java/org/apache/cassandra/streaming/async/StreamCompressionSerializer.java +++ b/src/java/org/apache/cassandra/streaming/async/StreamCompressionSerializer.java @@ -27,7 +27,6 @@ import io.netty.buffer.ByteBufAllocator; import net.jpountz.lz4.LZ4Compressor; import net.jpountz.lz4.LZ4SafeDecompressor; import org.apache.cassandra.io.util.DataInputPlus; -import org.apache.cassandra.net.AsyncStreamingOutputPlus; import org.apache.cassandra.streaming.StreamingDataOutputPlus; import static org.apache.cassandra.net.MessagingService.current_version; diff --git a/src/java/org/apache/cassandra/tools/nodetool/Repair.java b/src/java/org/apache/cassandra/tools/nodetool/Repair.java index c4f60af3c2..8e5aab255e 100644 --- a/src/java/org/apache/cassandra/tools/nodetool/Repair.java +++ b/src/java/org/apache/cassandra/tools/nodetool/Repair.java @@ -97,6 +97,12 @@ public class Repair extends NodeToolCmd @Option(title = "optimise_streams", name = {"-os", "--optimise-streams"}, description = "Use --optimise-streams to try to reduce the number of streams we do (EXPERIMENTAL, see CASSANDRA-3200).") private boolean optimiseStreams = false; + @Option(title = "skip-paxos", name = {"-skip-paxos", "--skip-paxos"}, description = "If the --skip-paxos flag is included, the paxos repair step is skipped. Paxos repair is also skipped for preview repairs.") + private boolean skipPaxos = false; + + @Option(title = "paxos-only", name = {"-paxos-only", "--paxos-only"}, description = "If the --paxos-only flag is included, no table data is repaired, only paxos operations..") + private boolean paxosOnly = false; + @Option(title = "ignore_unreplicated_keyspaces", name = {"-iuk","--ignore-unreplicated-keyspaces"}, description = "Use --ignore-unreplicated-keyspaces to ignore keyspaces which are not replicated, otherwise the repair will fail") private boolean ignoreUnreplicatedKeyspaces = false; @@ -152,6 +158,8 @@ public class Repair extends NodeToolCmd options.put(RepairOption.PREVIEW, getPreviewKind().toString()); options.put(RepairOption.OPTIMISE_STREAMS_KEY, Boolean.toString(optimiseStreams)); options.put(RepairOption.IGNORE_UNREPLICATED_KS, Boolean.toString(ignoreUnreplicatedKeyspaces)); + options.put(RepairOption.REPAIR_PAXOS_KEY, Boolean.toString(!skipPaxos && getPreviewKind() == PreviewKind.NONE)); + options.put(RepairOption.PAXOS_ONLY_KEY, Boolean.toString(paxosOnly && getPreviewKind() == PreviewKind.NONE)); if (!startToken.isEmpty() || !endToken.isEmpty()) { diff --git a/src/java/org/apache/cassandra/tracing/TraceStateImpl.java b/src/java/org/apache/cassandra/tracing/TraceStateImpl.java index d1740626f9..f8691eee5d 100644 --- a/src/java/org/apache/cassandra/tracing/TraceStateImpl.java +++ b/src/java/org/apache/cassandra/tracing/TraceStateImpl.java @@ -18,7 +18,6 @@ package org.apache.cassandra.tracing; import java.util.Arrays; -import java.util.Collections; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; @@ -29,7 +28,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.cassandra.concurrent.Stage; -import org.apache.cassandra.db.ConsistencyLevel; import org.apache.cassandra.db.Mutation; import org.apache.cassandra.exceptions.OverloadedException; import org.apache.cassandra.locator.InetAddressAndPort; diff --git a/src/java/org/apache/cassandra/utils/BiLongAccumulator.java b/src/java/org/apache/cassandra/utils/BiLongAccumulator.java index 2c3d6b5d4e..cdf2a4fd9e 100644 --- a/src/java/org/apache/cassandra/utils/BiLongAccumulator.java +++ b/src/java/org/apache/cassandra/utils/BiLongAccumulator.java @@ -20,5 +20,5 @@ package org.apache.cassandra.utils; public interface BiLongAccumulator { - long apply(T obj, A arguemnt, long v); + long apply(T obj, A argument, long v); } diff --git a/src/java/org/apache/cassandra/utils/CassandraVersion.java b/src/java/org/apache/cassandra/utils/CassandraVersion.java index 56290fb3c2..023127edfa 100644 --- a/src/java/org/apache/cassandra/utils/CassandraVersion.java +++ b/src/java/org/apache/cassandra/utils/CassandraVersion.java @@ -43,7 +43,7 @@ public class CassandraVersion implements Comparable * note: 3rd/4th groups matches to words but only allows number and checked after regexp test. * this is because 3rd and the last can be identical. **/ - private static final String VERSION_REGEXP = "(\\d+)\\.(\\d+)(?:\\.(\\w+))?(?:\\.(\\w+))?(\\-[-.\\w]+)?([.+][.\\w]+)?"; + private static final String VERSION_REGEXP = "(?\\d+)\\.(?\\d+)(\\.(?\\w+)(\\.(?\\w+))?)?(-(?[-.\\w]+))?([.+](?[.\\w]+))?"; private static final Pattern PATTERN_WORDS = Pattern.compile("\\w+"); @VisibleForTesting static final int NO_HOTFIX = -1; @@ -90,13 +90,13 @@ public class CassandraVersion implements Comparable try { - this.major = Integer.parseInt(matcher.group(1)); - this.minor = Integer.parseInt(matcher.group(2)); - this.patch = matcher.group(3) != null ? Integer.parseInt(matcher.group(3)) : 0; - this.hotfix = matcher.group(4) != null ? Integer.parseInt(matcher.group(4)) : NO_HOTFIX; + this.major = intPart(matcher, "major"); + this.minor = intPart(matcher, "minor"); + this.patch = intPart(matcher, "patch", 0); + this.hotfix = intPart(matcher, "hotfix", NO_HOTFIX); - String pr = matcher.group(5); - String bld = matcher.group(6); + String pr = matcher.group("prerelease"); + String bld = matcher.group("build"); this.preRelease = pr == null || pr.isEmpty() ? null : parseIdentifiers(version, pr); this.build = bld == null || bld.isEmpty() ? null : parseIdentifiers(version, bld); @@ -107,6 +107,17 @@ public class CassandraVersion implements Comparable } } + private static int intPart(Matcher matcher, String group) + { + return Integer.parseInt(matcher.group(group)); + } + + private static int intPart(Matcher matcher, String group, int orElse) + { + String value = matcher.group(group); + return value == null ? orElse : Integer.parseInt(value); + } + private CassandraVersion getFamilyLowerBound() { return patch == 0 && hotfix == NO_HOTFIX && preRelease != null && preRelease.length == 0 && build == null @@ -117,7 +128,6 @@ public class CassandraVersion implements Comparable private static String[] parseIdentifiers(String version, String str) { // Drop initial - or + - str = str.substring(1); String[] parts = StringUtils.split(str, ".-"); for (String part : parts) { diff --git a/src/java/org/apache/cassandra/utils/CloseableIterator.java b/src/java/org/apache/cassandra/utils/CloseableIterator.java index 57034ae609..a4cb857daf 100644 --- a/src/java/org/apache/cassandra/utils/CloseableIterator.java +++ b/src/java/org/apache/cassandra/utils/CloseableIterator.java @@ -18,9 +18,54 @@ package org.apache.cassandra.utils; import java.util.Iterator; +import java.util.NoSuchElementException; +import java.util.function.Supplier; // so we can instantiate anonymous classes implementing both interfaces public interface CloseableIterator extends Iterator, AutoCloseable { public void close(); + + public static CloseableIterator wrap(Iterator iter) + { + return new CloseableIterator() + { + public void close() + { + // noop + } + + public boolean hasNext() + { + return iter.hasNext(); + } + + public T next() + { + return iter.next(); + } + }; + } + + public static CloseableIterator empty() + { + return new CloseableIterator() + { + public void close() + { + // noop + } + + public boolean hasNext() + { + return false; + } + + public T next() + { + throw new NoSuchElementException(); + } + }; + } + } diff --git a/src/java/org/apache/cassandra/utils/CollectionSerializer.java b/src/java/org/apache/cassandra/utils/CollectionSerializer.java new file mode 100644 index 0000000000..4f8e8b068c --- /dev/null +++ b/src/java/org/apache/cassandra/utils/CollectionSerializer.java @@ -0,0 +1,125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.utils; + +import java.io.IOException; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.RandomAccess; +import java.util.Set; +import java.util.function.IntFunction; + +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; + +import org.apache.cassandra.db.TypeSizes; +import org.apache.cassandra.io.IVersionedSerializer; +import org.apache.cassandra.io.util.DataInputPlus; +import org.apache.cassandra.io.util.DataOutputPlus; + +public class CollectionSerializer +{ + + public static void serializeCollection(IVersionedSerializer valueSerializer, Collection values, DataOutputPlus out, int version) throws IOException + { + out.writeUnsignedVInt(values.size()); + for (V value : values) + valueSerializer.serialize(value, out, version); + } + + public static & RandomAccess> void serializeList(IVersionedSerializer valueSerializer, L values, DataOutputPlus out, int version) throws IOException + { + int size = values.size(); + out.writeUnsignedVInt(size); + for (int i = 0 ; i < size ; ++i) + valueSerializer.serialize(values.get(i), out, version); + } + + public static void serializeMap(IVersionedSerializer keySerializer, IVersionedSerializer valueSerializer, Map map, DataOutputPlus out, int version) throws IOException + { + out.writeUnsignedVInt(map.size()); + for (Map.Entry e : map.entrySet()) + { + keySerializer.serialize(e.getKey(), out, version); + valueSerializer.serialize(e.getValue(), out, version); + } + } + + public static > C deserializeCollection(IVersionedSerializer serializer, IntFunction factory, DataInputPlus in, int version) throws IOException + { + int size = (int) in.readUnsignedVInt(); + C result = factory.apply(size); + while (size-- > 0) + result.add(serializer.deserialize(in, version)); + return result; + } + + public static > M deserializeMap(IVersionedSerializer keySerializer, IVersionedSerializer valueSerializer, IntFunction factory, DataInputPlus in, int version) throws IOException + { + int size = (int) in.readUnsignedVInt(); + M result = factory.apply(size); + while (size-- > 0) + { + K key = keySerializer.deserialize(in, version); + V value = valueSerializer.deserialize(in, version); + result.put(key, value); + } + return result; + } + + public static long serializedSizeCollection(IVersionedSerializer valueSerializer, Collection values, int version) + { + long size = TypeSizes.sizeofUnsignedVInt(values.size()); + for (V value : values) + size += valueSerializer.serializedSize(value, version); + return size; + } + + public static & RandomAccess> long serializedSizeList(IVersionedSerializer valueSerializer, L values, int version) throws IOException + { + int items = values.size(); + long size = TypeSizes.sizeofUnsignedVInt(items); + for (int i = 0 ; i < items ; ++i) + size += valueSerializer.serializedSize(values.get(i), version); + return size; + } + + + public static long serializedSizeMap(IVersionedSerializer keySerializer, IVersionedSerializer valueSerializer, Map map, int version) + { + long size = TypeSizes.sizeofUnsignedVInt(map.size()); + for (Map.Entry e : map.entrySet()) + size += keySerializer.serializedSize(e.getKey(), version) + + valueSerializer.serializedSize(e.getValue(), version); + return size; + } + + public static IntFunction> newHashSet() + { + return i -> i == 0 ? Collections.emptySet() : Sets.newHashSetWithExpectedSize(i); + } + + public static IntFunction> newHashMap() + { + return i -> i == 0 ? Collections.emptyMap() : Maps.newHashMapWithExpectedSize(i); + } + +} diff --git a/src/java/org/apache/cassandra/utils/EstimatedHistogram.java b/src/java/org/apache/cassandra/utils/EstimatedHistogram.java index ed3dccc641..198f92286f 100644 --- a/src/java/org/apache/cassandra/utils/EstimatedHistogram.java +++ b/src/java/org/apache/cassandra/utils/EstimatedHistogram.java @@ -20,6 +20,7 @@ package org.apache.cassandra.utils; import java.io.IOException; import java.util.Arrays; import java.util.concurrent.atomic.AtomicLongArray; +import java.util.function.DoubleToLongFunction; import com.google.common.base.Objects; import org.slf4j.Logger; @@ -30,7 +31,7 @@ import org.apache.cassandra.io.ISerializer; import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputPlus; -public class EstimatedHistogram +public class EstimatedHistogram implements DoubleToLongFunction { public static final EstimatedHistogramSerializer serializer = new EstimatedHistogramSerializer(); @@ -384,6 +385,12 @@ public class EstimatedHistogram return Objects.hashCode(getBucketOffsets(), getBuckets(false)); } + @Override + public long applyAsLong(double value) + { + return percentile(value); + } + public static class EstimatedHistogramSerializer implements ISerializer { private static final Logger logger = LoggerFactory.getLogger(EstimatedHistogramSerializer.class); diff --git a/src/java/org/apache/cassandra/utils/MonotonicClock.java b/src/java/org/apache/cassandra/utils/MonotonicClock.java index d4590c981a..ad9ee81ca4 100644 --- a/src/java/org/apache/cassandra/utils/MonotonicClock.java +++ b/src/java/org/apache/cassandra/utils/MonotonicClock.java @@ -143,14 +143,14 @@ public interface MonotonicClock private static final long UPDATE_INTERVAL_MS = Long.getLong(UPDATE_INTERVAL_PROPERTY, 10000); @VisibleForTesting - static class AlmostSameTime implements MonotonicClockTranslation + public static class AlmostSameTime implements MonotonicClockTranslation { final long millisSinceEpoch; final long monotonicNanos; final long error; // maximum error of millis measurement (in nanos) @VisibleForTesting - AlmostSameTime(long millisSinceEpoch, long monotonicNanos, long errorNanos) + public AlmostSameTime(long millisSinceEpoch, long monotonicNanos, long errorNanos) { this.millisSinceEpoch = millisSinceEpoch; this.monotonicNanos = monotonicNanos; diff --git a/src/java/org/apache/cassandra/utils/Nemesis.java b/src/java/org/apache/cassandra/utils/Nemesis.java index b5110c44c0..840eba3ac3 100644 --- a/src/java/org/apache/cassandra/utils/Nemesis.java +++ b/src/java/org/apache/cassandra/utils/Nemesis.java @@ -34,7 +34,7 @@ import static org.apache.cassandra.utils.Nemesis.Traffic.HIGH; * TODO: Support @Nemesis on methods, to insert nemesis points either before or after invocations of the method */ @Retention(RetentionPolicy.RUNTIME) -@Target({ ElementType.FIELD }) +@Target({ ElementType.FIELD, ElementType.METHOD }) public @interface Nemesis { enum Traffic { LOW, HIGH } diff --git a/src/java/org/apache/cassandra/utils/NullableSerializer.java b/src/java/org/apache/cassandra/utils/NullableSerializer.java new file mode 100644 index 0000000000..67e2d6a0a9 --- /dev/null +++ b/src/java/org/apache/cassandra/utils/NullableSerializer.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.utils; + +import java.io.IOException; + +import org.apache.cassandra.db.TypeSizes; +import org.apache.cassandra.io.IVersionedSerializer; +import org.apache.cassandra.io.util.DataInputPlus; +import org.apache.cassandra.io.util.DataOutputPlus; + +public class NullableSerializer +{ + + public static void serializeNullable(IVersionedSerializer serializer, T value, DataOutputPlus out, int version) throws IOException + { + out.writeBoolean(value != null); + if (value != null) + serializer.serialize(value, out, version); + } + + public static T deserializeNullable(IVersionedSerializer serializer, DataInputPlus in, int version) throws IOException + { + return in.readBoolean() ? serializer.deserialize(in, version) : null; + } + + public static long serializedSizeNullable(IVersionedSerializer serializer, T value, int version) + { + return value != null + ? TypeSizes.sizeof(true) + serializer.serializedSize(value, version) + : TypeSizes.sizeof(false); + } + + public static IVersionedSerializer wrap(IVersionedSerializer wrap) + { + return new IVersionedSerializer() { + public void serialize(T t, DataOutputPlus out, int version) throws IOException + { + serializeNullable(wrap, t, out, version); + } + + public T deserialize(DataInputPlus in, int version) throws IOException + { + return deserializeNullable(wrap, in, version); + } + + public long serializedSize(T t, int version) + { + return serializedSizeNullable(wrap, t, version); + } + }; + } + +} diff --git a/src/java/org/apache/cassandra/utils/TimeUUID.java b/src/java/org/apache/cassandra/utils/TimeUUID.java index c0841e70c0..8d79096ecd 100644 --- a/src/java/org/apache/cassandra/utils/TimeUUID.java +++ b/src/java/org/apache/cassandra/utils/TimeUUID.java @@ -83,12 +83,12 @@ public class TimeUUID implements Serializable, Comparable private static final long MAX_CLOCK_SEQ_AND_NODE = 0x7f7f7f7f7f7f7f7fL; - final long rawTimestamp, lsb; + final long uuidTimestamp, lsb; - public TimeUUID(long rawTimestamp, long lsb) + public TimeUUID(long uuidTimestamp, long lsb) { // we don't validate that this is a true TIMEUUID to avoid problems with historical mixing of UUID with TimeUUID - this.rawTimestamp = rawTimestamp; + this.uuidTimestamp = uuidTimestamp; this.lsb = lsb; } @@ -97,11 +97,6 @@ public class TimeUUID implements Serializable, Comparable return new TimeUUID(unixMicrosToRawTimestamp(unixMicros), uniqueLsb); } - public static TimeUUID atUnixMicrosWithLsb(long unixMicros, long uniqueLsb, boolean isSerial) - { - return new TimeUUID(unixMicrosToRawTimestamp(unixMicros) + (isSerial ? 2 : 1), uniqueLsb); - } - public static UUID atUnixMicrosWithLsbAsUUID(long unixMicros, long uniqueLsb) { return new UUID(rawTimestampToMsb(unixMicrosToRawTimestamp(unixMicros)), uniqueLsb); @@ -187,7 +182,7 @@ public class TimeUUID implements Serializable, Comparable public UUID asUUID() { - return new UUID(rawTimestampToMsb(rawTimestamp), lsb); + return new UUID(rawTimestampToMsb(uuidTimestamp), lsb); } /** @@ -203,21 +198,21 @@ public class TimeUUID implements Serializable, Comparable */ public long unixMicros() { - return rawTimestampToUnixMicros(rawTimestamp); + return rawTimestampToUnixMicros(uuidTimestamp); } /** * The UUID-format timestamp, i.e. 10x micros-resolution, as of UUIDGen.UUID_EPOCH_UNIX_MILLIS * The tenths of a microsecond are used to store a flag value. */ - public long rawTimestamp() + public long uuidTimestamp() { - return rawTimestamp & 0x0FFFFFFFFFFFFFFFL; + return uuidTimestamp & 0x0FFFFFFFFFFFFFFFL; } public long msb() { - return rawTimestampToMsb(rawTimestamp); + return rawTimestampToMsb(uuidTimestamp); } public long lsb() @@ -260,7 +255,7 @@ public class TimeUUID implements Serializable, Comparable @Override public int hashCode() { - return (int) ((rawTimestamp ^ (rawTimestamp >> 32) * 31) + (lsb ^ (lsb >> 32))); + return (int) ((uuidTimestamp ^ (uuidTimestamp >> 32) * 31) + (lsb ^ (lsb >> 32))); } @Override @@ -272,12 +267,12 @@ public class TimeUUID implements Serializable, Comparable public boolean equals(TimeUUID that) { - return that != null && rawTimestamp == that.rawTimestamp && lsb == that.lsb; + return that != null && uuidTimestamp == that.uuidTimestamp && lsb == that.lsb; } public boolean equals(UUID that) { - return that != null && rawTimestamp == that.timestamp() && lsb == that.getLeastSignificantBits(); + return that != null && uuidTimestamp == that.timestamp() && lsb == that.getLeastSignificantBits(); } @Override @@ -288,19 +283,19 @@ public class TimeUUID implements Serializable, Comparable public static String toString(TimeUUID ballot) { - return ballot == null ? "null" : ballot.rawTimestamp() + ":" + ballot; + return ballot == null ? "null" : ballot.uuidTimestamp() + ":" + ballot; } public static String toString(TimeUUID ballot, String kind) { - return ballot == null ? "null" : String.format("%s(%d:%s)", kind, ballot.rawTimestamp(), ballot); + return ballot == null ? "null" : String.format("%s(%d:%s)", kind, ballot.uuidTimestamp(), ballot); } @Override public int compareTo(TimeUUID that) { - return this.rawTimestamp != that.rawTimestamp - ? Long.compare(this.rawTimestamp, that.rawTimestamp) + return this.uuidTimestamp != that.uuidTimestamp + ? Long.compare(this.uuidTimestamp, that.uuidTimestamp) : Long.compare(this.lsb, that.lsb); } diff --git a/src/java/org/apache/cassandra/utils/UUIDGen.java b/src/java/org/apache/cassandra/utils/UUIDGen.java index 14ab23083b..351a5d3a03 100644 --- a/src/java/org/apache/cassandra/utils/UUIDGen.java +++ b/src/java/org/apache/cassandra/utils/UUIDGen.java @@ -19,6 +19,7 @@ package org.apache.cassandra.utils; import java.nio.ByteBuffer; import java.util.UUID; +import java.util.concurrent.TimeUnit; /** * The goods are here: www.ietf.org/rfc/rfc4122.txt. diff --git a/src/java/org/apache/cassandra/utils/VoidSerializer.java b/src/java/org/apache/cassandra/utils/VoidSerializer.java new file mode 100644 index 0000000000..1bc11675ea --- /dev/null +++ b/src/java/org/apache/cassandra/utils/VoidSerializer.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.utils; + +import java.io.IOException; + +import org.apache.cassandra.io.IVersionedSerializer; +import org.apache.cassandra.io.util.DataInputPlus; +import org.apache.cassandra.io.util.DataOutputPlus; + +public class VoidSerializer implements IVersionedSerializer +{ + public static final VoidSerializer serializer = new VoidSerializer(); + private VoidSerializer() {} + public void serialize(Void v, DataOutputPlus out, int version) throws IOException {} + public Void deserialize(DataInputPlus in, int version) throws IOException { return null; } + public long serializedSize(Void v, int version) { return 0; } +} diff --git a/src/java/org/apache/cassandra/utils/binlog/ExternalArchiver.java b/src/java/org/apache/cassandra/utils/binlog/ExternalArchiver.java index b3ce4846c0..3f1d6bc51a 100644 --- a/src/java/org/apache/cassandra/utils/binlog/ExternalArchiver.java +++ b/src/java/org/apache/cassandra/utils/binlog/ExternalArchiver.java @@ -40,8 +40,8 @@ import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException; import static java.util.concurrent.TimeUnit.MILLISECONDS; -import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis; import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory; +import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis; /** * Archives binary log files immediately when they are rolled using a configure archive command. diff --git a/src/java/org/apache/cassandra/utils/concurrent/AbstractFuture.java b/src/java/org/apache/cassandra/utils/concurrent/AbstractFuture.java index 026f9f23c3..83cd7d3f8b 100644 --- a/src/java/org/apache/cassandra/utils/concurrent/AbstractFuture.java +++ b/src/java/org/apache/cassandra/utils/concurrent/AbstractFuture.java @@ -316,7 +316,6 @@ public abstract class AbstractFuture implements Future return this; } - /** * Support {@link com.google.common.util.concurrent.Futures#transformAsync(ListenableFuture, AsyncFunction, Executor)} natively * diff --git a/src/java/org/apache/cassandra/utils/concurrent/ConditionAsConsumer.java b/src/java/org/apache/cassandra/utils/concurrent/ConditionAsConsumer.java new file mode 100644 index 0000000000..985daf7103 --- /dev/null +++ b/src/java/org/apache/cassandra/utils/concurrent/ConditionAsConsumer.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.utils.concurrent; + +import java.util.function.Consumer; + +public interface ConditionAsConsumer extends Condition, Consumer +{ + @Override + default void accept(T t) { signal(); } + + public static class Async extends Condition.Async implements ConditionAsConsumer { } + + public static ConditionAsConsumer newConditionAsConsumer() + { + return new Async(); + } +} diff --git a/src/java/org/apache/cassandra/utils/concurrent/IntrusiveStack.java b/src/java/org/apache/cassandra/utils/concurrent/IntrusiveStack.java index 6f944d2476..8717335d60 100644 --- a/src/java/org/apache/cassandra/utils/concurrent/IntrusiveStack.java +++ b/src/java/org/apache/cassandra/utils/concurrent/IntrusiveStack.java @@ -22,8 +22,11 @@ import java.util.Iterator; import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; import java.util.function.BiFunction; import java.util.function.Consumer; +import java.util.function.Function; import net.nicoulaj.compilecommand.annotations.Inline; +import org.apache.cassandra.utils.BiLongAccumulator; +import org.apache.cassandra.utils.LongAccumulator; /** * An efficient stack/list that is expected to be ordinarily either empty or close to, and for which @@ -62,21 +65,45 @@ public class IntrusiveStack> implements Iterable T next; @Inline - protected static > void push(AtomicReferenceFieldUpdater headUpdater, O owner, T prepend) + protected static > T push(AtomicReferenceFieldUpdater headUpdater, O owner, T prepend) { - push(headUpdater, owner, prepend, (prev, next) -> { + return push(headUpdater, owner, prepend, (prev, next) -> { next.next = prev; return next; }); } - protected static > void push(AtomicReferenceFieldUpdater headUpdater, O owner, T prepend, BiFunction combine) + protected static > T push(AtomicReferenceFieldUpdater headUpdater, O owner, T prepend, BiFunction combine) { while (true) { T head = headUpdater.get(owner); if (headUpdater.compareAndSet(owner, head, combine.apply(head, prepend))) - return; + return head; + } + } + + protected interface Setter + { + public boolean compareAndSet(O owner, T expect, T update); + } + + @Inline + protected static > T push(Function getter, Setter setter, O owner, T prepend) + { + return push(getter, setter, owner, prepend, (prev, next) -> { + next.next = prev; + return next; + }); + } + + protected static > T push(Function getter, Setter setter, O owner, T prepend, BiFunction combine) + { + while (true) + { + T head = getter.apply(owner); + if (setter.compareAndSet(owner, head, combine.apply(head, prepend))) + return head; } } @@ -86,6 +113,18 @@ public class IntrusiveStack> implements Iterable headUpdater.lazySet(owner, combine.apply(head, prepend)); } + protected static , O> void pushExclusive(AtomicReferenceFieldUpdater headUpdater, O owner, T prepend) + { + prepend.next = headUpdater.get(owner); + headUpdater.lazySet(owner, prepend); + } + + protected static > T pushExclusive(T head, T prepend) + { + prepend.next = head; + return prepend; + } + protected static , O> Iterable iterable(AtomicReferenceFieldUpdater headUpdater, O owner) { return iterable(headUpdater.get(owner)); @@ -112,6 +151,17 @@ public class IntrusiveStack> implements Iterable return size; } + protected static > long accumulate(T list, LongAccumulator accumulator, long initialValue) + { + long value = initialValue; + while (list != null) + { + value = accumulator.apply(list, initialValue); + list = list.next; + } + return value; + } + // requires exclusive ownership (incl. with readers) protected T reverse() { diff --git a/src/java/org/apache/cassandra/utils/concurrent/Ref.java b/src/java/org/apache/cassandra/utils/concurrent/Ref.java index f40e08fc81..f728a2023c 100644 --- a/src/java/org/apache/cassandra/utils/concurrent/Ref.java +++ b/src/java/org/apache/cassandra/utils/concurrent/Ref.java @@ -30,6 +30,7 @@ import java.util.*; import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; +import java.util.function.Consumer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -191,7 +192,7 @@ public final class Ref implements RefCounted private static final AtomicIntegerFieldUpdater releasedUpdater = AtomicIntegerFieldUpdater.newUpdater(State.class, "released"); - public State(final GlobalState globalState, Ref reference, ReferenceQueue q) + State(final GlobalState globalState, Ref reference, ReferenceQueue q) { super(reference, q); this.globalState = globalState; diff --git a/src/java/org/apache/cassandra/utils/concurrent/SyncFuture.java b/src/java/org/apache/cassandra/utils/concurrent/SyncFuture.java index a7b34738cf..2a3598aa03 100644 --- a/src/java/org/apache/cassandra/utils/concurrent/SyncFuture.java +++ b/src/java/org/apache/cassandra/utils/concurrent/SyncFuture.java @@ -59,7 +59,7 @@ import static org.apache.cassandra.utils.concurrent.Awaitable.SyncAwaitable.wait */ public class SyncFuture extends AbstractFuture { - public SyncFuture() + protected SyncFuture() { super(); } diff --git a/src/java/org/apache/cassandra/utils/concurrent/SyncPromise.java b/src/java/org/apache/cassandra/utils/concurrent/SyncPromise.java new file mode 100644 index 0000000000..fc5fb8124f --- /dev/null +++ b/src/java/org/apache/cassandra/utils/concurrent/SyncPromise.java @@ -0,0 +1,269 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.utils.concurrent; + +import java.util.concurrent.Executor; +import java.util.function.Consumer; + +import com.google.common.util.concurrent.FutureCallback; + +import io.netty.util.concurrent.Future; // checkstyle: permit this import +import io.netty.util.concurrent.GenericFutureListener; + +/** + * Extends {@link SyncFuture} to implement the {@link Promise} interface. + */ +public class SyncPromise extends SyncFuture implements Promise +{ + public static class WithExecutor extends SyncPromise + { + final Executor notifyExecutor; + protected WithExecutor(Executor notifyExecutor) + { + this.notifyExecutor = notifyExecutor; + } + + protected WithExecutor(Executor notifyExecutor, FailureHolder initialState) + { + super(initialState); + this.notifyExecutor = notifyExecutor; + } + + protected WithExecutor(Executor notifyExecutor, GenericFutureListener> listener) + { + super(listener); + this.notifyExecutor = notifyExecutor; + } + + @Override + public Executor notifyExecutor() + { + return notifyExecutor; + } + } + + public SyncPromise() {} + + SyncPromise(FailureHolder initialState) + { + super(initialState); + } + + public SyncPromise(GenericFutureListener> listener) + { + super(listener); + } + + SyncPromise(FailureHolder initialState, GenericFutureListener> listener) + { + super(initialState, listener); + } + + public static SyncPromise withExecutor(Executor executor) + { + return new SyncPromise.WithExecutor<>(executor); + } + + public static SyncPromise uncancellable() + { + return new SyncPromise<>(UNCANCELLABLE); + } + + public static SyncPromise uncancellable(Executor executor) + { + return new WithExecutor<>(executor, UNCANCELLABLE); + } + + public static SyncPromise uncancellable(GenericFutureListener> listener) + { + return new SyncPromise<>(UNCANCELLABLE, listener); + } + + /** + * Complete the promise successfully if not already complete + * @throws IllegalStateException if already set + */ + @Override + public Promise setSuccess(V v) + { + if (!trySuccess(v)) + throw new IllegalStateException("complete already: " + this); + return this; + } + + /** + * Complete the promise successfully if not already complete + * @return true iff completed promise + */ + @Override + public boolean trySuccess(V v) + { + return super.trySuccess(v); + } + + /** + * Complete the promise abnormally if not already complete + * @throws IllegalStateException if already set + */ + @Override + public Promise setFailure(Throwable throwable) + { + if (!tryFailure(throwable)) + throw new IllegalStateException("complete already: " + this); + return this; + } + + /** + * Complete the promise abnormally if not already complete + * @return true iff completed promise + */ + @Override + public boolean tryFailure(Throwable throwable) + { + return super.tryFailure(throwable); + } + + /** + * Prevent a future caller from cancelling this promise + * @return true if the promise is now uncancellable (whether or not we did this) + */ + @Override + public boolean setUncancellable() + { + return super.setUncancellable(); + } + + /** + * Prevent a future caller from cancelling this promise + * @return true iff this invocation set it to uncancellable, whether or not now uncancellable + */ + @Override + public boolean setUncancellableExclusive() + { + return super.setUncancellableExclusive(); + } + + @Override + public boolean isUncancellable() + { + return super.isUncancellable(); + } + + /** + * waits for completion; in case of failure rethrows the original exception without a new wrapping exception + * so may cause problems for reporting stack traces + */ + @Override + public Promise sync() throws InterruptedException + { + super.sync(); + return this; + } + + /** + * waits for completion; in case of failure rethrows the original exception without a new wrapping exception + * so may cause problems for reporting stack traces + */ + @Override + public Promise syncUninterruptibly() + { + super.syncUninterruptibly(); + return this; + } + + @Override + public SyncPromise addListener(GenericFutureListener> listener) + { + super.addListener(listener); + return this; + } + + @Override + public SyncPromise addListeners(GenericFutureListener>... listeners) + { + super.addListeners(listeners); + return this; + } + + @Override + public SyncPromise removeListener(GenericFutureListener> listener) + { + throw new UnsupportedOperationException(); + } + + @Override + public SyncPromise removeListeners(GenericFutureListener>... listeners) + { + throw new UnsupportedOperationException(); + } + + @Override + public SyncPromise addCallback(FutureCallback callback) + { + super.addCallback(callback); + return this; + } + + @Override + public SyncPromise addCallback(FutureCallback callback, Executor executor) + { + super.addCallback(callback, executor); + return this; + } + + @Override + public SyncPromise addCallback(Consumer onSuccess, Consumer onFailure) + { + super.addCallback(onSuccess, onFailure); + return this; + } + + /** + * Wait for this promise to complete + * @throws InterruptedException if interrupted + */ + @Override + public SyncPromise await() throws InterruptedException + { + super.await(); + return this; + } + + /** + * Wait uninterruptibly for this promise to complete + */ + @Override + public SyncPromise awaitUninterruptibly() + { + super.awaitUninterruptibly(); + return this; + } + + /** + * Wait for this promise to complete, throwing any interrupt as an UncheckedInterruptedException + * @throws UncheckedInterruptedException if interrupted + */ + @Override + public SyncPromise awaitThrowUncheckedOnInterrupt() throws UncheckedInterruptedException + { + super.awaitThrowUncheckedOnInterrupt(); + return this; + } +} + diff --git a/src/java/org/apache/cassandra/utils/memory/BufferPool.java b/src/java/org/apache/cassandra/utils/memory/BufferPool.java index f302f4ff1f..cc28ea7915 100644 --- a/src/java/org/apache/cassandra/utils/memory/BufferPool.java +++ b/src/java/org/apache/cassandra/utils/memory/BufferPool.java @@ -1485,8 +1485,7 @@ public class BufferPool @VisibleForTesting public void shutdownLocalCleaner(long timeout, TimeUnit unit) throws InterruptedException, TimeoutException { - shutdownNow(of(localPoolCleaner)); - awaitTermination(timeout, unit, of(localPoolCleaner)); + shutdownAndWait(timeout, unit, of(localPoolCleaner)); } @VisibleForTesting diff --git a/test/burn/org/apache/cassandra/utils/memory/LongBufferPoolTest.java b/test/burn/org/apache/cassandra/utils/memory/LongBufferPoolTest.java index 9bb217c7e6..3c110d7060 100644 --- a/test/burn/org/apache/cassandra/utils/memory/LongBufferPoolTest.java +++ b/test/burn/org/apache/cassandra/utils/memory/LongBufferPoolTest.java @@ -101,6 +101,7 @@ public class LongBufferPoolTest { DebugChunk.get(chunk).lastRecycled = recycleRound; } + public synchronized void check() { for (BufferPool.Chunk chunk : normalChunks) diff --git a/test/conf/cassandra.yaml b/test/conf/cassandra.yaml index 58afdb932f..4ca9238428 100644 --- a/test/conf/cassandra.yaml +++ b/test/conf/cassandra.yaml @@ -63,4 +63,4 @@ track_warnings: abort_threshold_kb: 8192 row_index_size: warn_threshold_kb: 4096 - abort_threshold_kb: 8192 + abort_threshold_kb: 8192 \ No newline at end of file diff --git a/test/conf/logback-simulator.xml b/test/conf/logback-simulator.xml index a7e286fd80..25c9de8a6a 100644 --- a/test/conf/logback-simulator.xml +++ b/test/conf/logback-simulator.xml @@ -40,11 +40,15 @@ + + + + diff --git a/test/data/serialization/4.1/gms.EndpointState.bin b/test/data/serialization/4.1/gms.EndpointState.bin new file mode 100644 index 0000000000000000000000000000000000000000..083dbb773621f0b3183c5b2f2fb266eb6572ee19 GIT binary patch literal 73 zcmZQzU`SZN3q=s8uEr1}@7q!y*71SA%fWR_&+=cN{b GoB{wnh$F55 literal 0 HcmV?d00001 diff --git a/test/data/serialization/4.1/service.SyncComplete.bin b/test/data/serialization/4.1/service.SyncComplete.bin new file mode 100644 index 0000000000000000000000000000000000000000..39a6243972eb2a33c3239a72d27a9cd32c9c47e3 GIT binary patch literal 256 zcmZQ9F;>0c<1P56t3x-PL#*vKI-h~lJGHX7ATc@BkbyI}Brz`~u_(omfq{XM6-4|8 z6KwSi3{28VAc|Q!i4i1*Dg+W??0&kvu;1_TY|%F#S-zYB3Ne6;0D%-BW^{R(thZ)` hYs$Vg3%@(RE|>%q`T~|^VBlgzcNRLI3}=GP1_178L^S{a literal 0 HcmV?d00001 diff --git a/test/data/serialization/4.1/service.SyncRequest.bin b/test/data/serialization/4.1/service.SyncRequest.bin new file mode 100644 index 0000000000000000000000000000000000000000..f853b20f9ce4f086e604938ae4dd654138117b94 GIT binary patch literal 111 zcmZQ9F;>0c<1P56t3x-PL#*vKI-h~lJGHX7ATc@BkbyI}Brz`~u_(omfq{XM6-4|8 Y6KwTBrgRdBVuDc2(n)A)fLsO!06f1Mo&W#< literal 0 HcmV?d00001 diff --git a/test/data/serialization/4.1/service.ValidationComplete.bin b/test/data/serialization/4.1/service.ValidationComplete.bin new file mode 100644 index 0000000000000000000000000000000000000000..bb4de439f6e0e3e1c7142e73ecc3ea16700af294 GIT binary patch literal 597 zcmZQ9F;>0c<1P56t3x-PL#*vKI-h~lJGHX7ATc@BkbyI}Brz`~u_(omfq{XM6-4|8 z6O16T4oCn&1DFI7j0{@&Md^BpK)o5MddZ2!#X!wPiFzp+C3-0c<1P56t3x-PL#*vKI-h~lJGHX7ATc@BkbyI}Brz`~u_(omfq{XM6-4}p H5-gVhu|gJN literal 0 HcmV?d00001 diff --git a/test/data/serialization/4.1/utils.EstimatedHistogram.bin b/test/data/serialization/4.1/utils.EstimatedHistogram.bin new file mode 100644 index 0000000000000000000000000000000000000000..e878eda0ce8d45a08740a90d3270bfd8611bfe83 GIT binary patch literal 97500 zcmeI%f1HhV{x|S*t~183F&JYoe*a8Xl1eIxlBANPl1fM=E48v#R%N9Uk}XNCB(>H` zvMR}1tCA!YYWra&wNh(Kwzaady6^is&pC6>JnsL#kMDn<{pWL@^FA|ko!9kxUuJg` ziNuwO^na4hvZJD-vVk*I!Fd&2sexN-$oso-iATu{M<6sL=6X?u{=U&Ocm8dY94DLnGuHzL}SL7dh_JBlr5QDBYbiANfsD zjwfC8T#g_8PnR68TA!QGC#!PuOjfJ)@9g&{&pdZVj_+)-xZHK_^E&q)CK450VTTjp znOz>@|9cgl**STHXLe2;iSW$M$s;_o{|C?P|NC%$>$Q&Ac}B;)l=C-nG^gFH3k-K z$)49rowp&|bxvuW^EXJn!vD<8o;Q5v*L$v~KJ-0ObdanHT)RVLOJi33+MYFQrKV{geH+E%t{*F)9EXwlQg-<<~ zeZ9M1Yq#x`@_Dl8uFXfz%JO+5|NX<{EKk4inB(5f^7gA99@)O!(W$5JxpPF8pDA9} z>GpD`PFyi^;gKwVxNy(aS=W;{@9N^4x0cTqCc@ z&yP;;f6v&zW}H0X)+0||QU1R{Bu7s_zHQ#k<@1-ecbW6kRT)P$w=L;g`boKOySCH% zX64VX=uoYvE?!su_hf2To$ovB`BUcodA;tqa_eWyGp^Kp_~_#c-^)0^ZBgTGwKrv4 z`P$z5A71ud##KIsKzh;M0tYhBfW+ z=&{u^E~z>2#U>|o&$x8_mX9wVePPB;*B`faUgPqAFWNMDeCWPugEI5u)?W7WDU0$m z?!5bkbzR;#E#n?LKi{)t>e7rasy=Sn^=Hh>c<}a_BR?xylkv~qt9@Z%!}9#5&iU+t zCHFSV%%A$I$ES~u-kiLDw`)8o_ zpj~D2)WMzGCw;S^dztfI`RLRe?z+E>iApctxO3tu<1)_wcty7ki_8D{t#Gm4vU zZojpBe%|?4zVhzS0h#&b2P(gKO^>-5-#ESNt!FlzlJOnCt9yCf7ej$(053$ID!t*2aBd$W&4;!Hm6!t?N zp&y|iaRT}vjj$i`2>YQCp&yymh+oJf^g}H2pz!<%{fMg&_QOW#1BLyNN9afBN1T8@ zNF(frJi>lxMCeCmHR2cY2>lR?JSaRrLOlu0q%k8=(&r_Cp?_AE6&{0{S40upjaW`=Jq`ADPvNU&tf$LoD*3 z@canh5e96=tt;BoPa(^BkYGf!hUE(=tpKX;urD={Sb>hC_FzxKjJEc z{jd@GKw&@R5&9AO5htJz(g^z@kFXya5&Dr?jrfH;LO;YJ4+_tZ(2uwZVLxnyK2X>X zd4zt1e#8mrgEYc^$Rq5BMudK3RwI5PkI)aX$b-W3BlIJ#Lf8)*p$`=HLmr_Yp&xMq z`XG(4AMyzMp%I}UnbnA2$RqSaEb^f6{0RMss}T0XM(6{D{g6lKN9aeKfIdhg?1wzU zerQDKM`ktR7xD=G5Q{u0JU>D|;wpswuo3z|VL#*%`VsmOC!i0~2>T(Aupb%``jJ_U z_=P+|Kg1#r3eS(wkGKk9KWv0PP}mQ7gnop6#0ltwG{Sz!BkYGpgnndJBYq)|&=0Z5 zgTnJ8^dqi9*bf__4;1!89-$whA8`WuAdRpe@(BB(5uqQM)reonBlJTo@}Thi2>pnw z5cb1H=mUlQkVoi8=trD@K1d_%hdjc5Xhi5oW;Nm$@(BG9i##YiKSDp^Dun&85&A%3 zKjabm5&97)pbydr`yr389~u$*ky(xSg*-w(#3Bz0&yUcLxC&uEY=k~g*bjMxeuRF+ z3Fw0~!hXmj?1x5#eq>f7ej$(053$ID!t*2aBd$W&4;!Hm6!t?Np&y|iaRT}vjj$i` z2>YQCp&yymh+oJf^g}H2pz!<%{fMg&_QOW#1BLyNN9afBN1T8@NF(frJi>lxMCeCm zHR2cY2>lR?JSaRrLOl zu0q%k8=(&r_Cp?_AE6&{0{S40upjaW`=Jq`ADPvNU&tf$LoD*3@can zh5e96=tt;BoPa(^BkYGf!hUE(=tpKX;urD={Sb>hC_FzxKjJEc{jd@GKw&@R5&9AO z5htJz(g^z@kFXya5&Dr?jrfH;LO;YJ4+_tZ(2uwZVLxnyK2X>Xd4zt1e#8mrgEYc^ z$Rq5BMudK3RwI5PkI)aX$b-W3BlIJ#Lf8)*p$`=HLmr_Yp&xMq`XG(4AMyzMp%I}U znbnA2$RqSaEb^f6{0RMss}T0XM(6{D{gB81T0atr)Za5<|Nmc9^X0N@DCWr7#F=uo zb>=xcIV(B4I`f^qoRyvZoK>8IoK>B}on_ZGnyWjxHDx8ZyE7@oy?lzfbjHGi7h@iP7i zuVSAw`6?f0zL`7l2Yfv5;*PwRPv8Tr*U6)NV)~5AdV3O=_2guBUlMYU}E)6EBRS02V^@F?!aRT#NBxs_u$!l7SHFic`>`rXa%2RzJ`18 z20oX!aBtqu=kadV%V-~;Z+?g`NS~>QFXRH2TdIUFGH=HHSkDp{v-_Ip&vHy$!mc|p zfW0qqDPNU76PNMzd^z93?qg~KUtvCl2k}h4lIODDlUm4En=j?Typpfsbv%SO@lbxB z^(wWKuQmUQhx2!Q9UozzJ1?JaNS}GNcm$X7jogYyatHn?cVRux)6>LF=6aDB&F(|u zW*(A06YfKv-;?;6`B?KYd9g#g>BJZI@8Y&Rg*)-x+?DU)UVJb2V>wpR^Td7T z!_B9$`5EEFh9j3_-P)) zzh%8l=zYE(Bz|Wu_rzlMITOEUKR4mNyTE4dB7#2wk^sC*i~Y~GVsb6@@=59BpGjQ_-=SZ^zj z<3F2E;&nWYUtzsY$hEQ_C0;Xk-xBLtPKnpqbtN{iGw}v*NuP<0yq(|V-MoqS@mqX| z-RCM1znwm-6!1G-!tZi3b{$pP@mBMbcpG=)_qaE|&;9u?d==|mmFxLK^IP~Mp1>dT z6qZYsnfzDtxx9lH@+Z8Mcd{NPKIL`hyI9VN&)DZme9nGO;&1#_`b@Y_Rlegd%#ZM% z^jS5ZzvNo%=Tt4_zniz>z1)HS!Cm-kK8wHM3s`TeUdG>=58?ehlK;tL+2^Wy2mi}_ zDj(nn`QJQ`5Aq`Zo|m&8S6$8jF<;MzS&tJxu-+u(T-AL_{MTGAiKFZ~5APv#mt zg9~{M*W?APht-yFZSxnnh}Uu*-pF#QwvFqW@8A;N!}VD2lJ(hrPBuupx?Gc`>~ka= zvhPba;uh&M*_hk2`&PX(H#P6h&A1OY=K<{JRv*kQ%|~!69>cA9BDdjt_!yqWdQ|-} zKGytcZpX{`C%lS%&g!r7aps%31AoBB^Dgeld-()DzRT#NBxs_u$!l7SHFi zc`>`rf)#v@`5NxU8~9w_!o7JrpU1mdFAMhZ`R0fCg7jG<;tRQe~Fx$O7USjbnK zFXh3!lCR-)JcKv#P=25Fs>V*f*8D3T&foEMe1v`O!hF6VeHPZ@5nReQaw{Im9r&l* zh4s8pPm?#9>qT-jyAR2mc}V(9x(|hZPx5ExW6j6#9egWKW$!C|kbiDIkH_&M{sk}R z@w}RExn3owvHO^u&T>o6VAq+vpM54Z_o3&Ge5!I zc>(v~CwTxbF@hKKQ=cogex?Q#5P^GUppr|~PS*Jb~|N^Lz#zGm*eCD*f@lCQJtN^W3h z@(tdSK9d`HJHN@hc@yvBxA+jd&qWcxoj!{S_#H0ccexq6j-qzF)%+yh#@+Zm?#=IW zfBp+!#d=qCJ%4C^3xC8D_+y^Jaw(e0e>I=WJJ_BkKjEe3J6R8tpYl5MT`cG1XY6w& zKW9HD`8WP5eJ0(fqVM<%^CP?`eb&k6FS!=`Idw|;@8+#|FL&U7a2NiX&*E?R0@j;4 zm+`meLwG-rmKD3(`Sj^MkleJL?^TR63M+p4$&#*es9!;PfMTCsoax&y`(RnZa$E^@-RMw zM{zeE$7k{+?#|P=2hZlScs`%ai`jLStl)FZ*KjZ1z~}N7?#K%2%b&=rX>ZFXvm>eXKWu zuP~p&gLo!i$#dE7ske}?Hebqvc_m-N>v#xn;-UOL>s7s-e69IcJe-Y%!-1YPM zhV)s#7LVXkzL8t;NbbNt)VEMf?k1&f|GC-^S}%FY3R;zcl}dC-UcfJAcityZ%AGGkyAJCJ{~I zYUaPj2_?xJd2m`ulWUbUmL9Dhs-zfY~IEX^A7fV8|>jn%=hzLKFp7%&(b`8j0^cU zT%Yx$bGuZ82&BmWweO(Ao?B4JzC5@XY_ma zb0hbqbXofA(KF_&cqzZi&+=yWeWf4pbLP8vIq&5^@Bv=INBQ~m*{}-hNy8$3(Yz6_ zq*YPxdh4ng;YePMXUNd*! zqV+7N=yi5o(FS%#Z}67%8Exe4{3h?_O}vla;zR5{H;VY}^x3F@-{BH|mz%NcXw;6k znxDkmxEsI6z4?9a&wt^oSnnEL&mWrK!XNPj{+Oq*TpG>fznahG9lVe~;ibHj^)ULB z*O~8PIY*zd&lP>neopi^{wjS&?o*@h_zUwRyeEA&&gU<=7W+AkOZo5St#~hY;D2xz z{+iF?Z}p@>uq{8sEYHGM~x^_(A?R&*Ouxh0#yGi;;q?}EvrC+3y=EWSbZ%FxuCT-30%sX)< z?#k|alU`ieydPKLL0pxGv+HUynyZ_S=K`M0HFyRW@*J+o3s?`EEaBSbFK`jB0^AX`P1Bvm+?<{75kh`U*+S>H**L6 zfRE>0+>!V434DO{y6I6qF?}}E+tf*{C#jR!eM!l^nH*B5nESn{E__=0Or6R-+1H!( z<5Yi#PDOyoG!7c0P}H zvtBma$LE_L;tSGe^N26x0+w6z628d18TVs7OI^(FYpOrXF?9*M?$iMGzSO0BRr*X_ z#@F-Zd<(me%_r~`=2LhO&*Uq4F8e*r7xLBSOL;J_b{|qV^N{qJavxgw zJ*l6Wk2N2|ckrz|mA$XUgZy*zc|496@h^BekLT5V8?R@*Xz>pJ()=Ty$e;7={58Ao z76)brGR=6aQy#_nTk zI?F9JgI#Cpe)jd$OrDcIQxEV0p2bV}*ZcyzuPxW|L*^TKHgDsHc?bKwE%)#v=KFar zALd8XXRADZj0^cUT%YxH~hxd>1e0z5EA0z$^GD zKc7BZS7AMAUBoY%H{zAthF{{2>~pj}jbAqJ$*Z|9|B(ms8Xm@f;!&)(t;g}7%_s3X zp2n}RUZ>>RT8~n%nY(YP^(?2<>+HHx8`zn8gSVv5)JERUZ}M*5#QXRyKE&>Gn~2{| zpKS{G9WLQ_xf#2THtl$;`ANKuyYYM6o8RaD{1?88^{&nJ{Gs_R{1H#!k9i8qrOizK ztNC2s!3+5lUdlUJ4^yA=I`dsD=hSEHbEQ6KKPUA!{wjT@+^06*@fYStcu)E~CZE5I J&*VJ|{ttaffFu9_ literal 0 HcmV?d00001 diff --git a/test/distributed/org/apache/cassandra/distributed/api/ConsistencyLevel.java b/test/distributed/org/apache/cassandra/distributed/api/ConsistencyLevel.java new file mode 100644 index 0000000000..c28de537d6 --- /dev/null +++ b/test/distributed/org/apache/cassandra/distributed/api/ConsistencyLevel.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.distributed.api; + +public enum ConsistencyLevel +{ + ANY(0), + ONE(1), + TWO(2), + THREE(3), + QUORUM(4), + ALL(5), + LOCAL_QUORUM(6), + EACH_QUORUM(7), + SERIAL(8), + LOCAL_SERIAL(9), + LOCAL_ONE(10), + NODE_LOCAL(11); + + public final int code; + ConsistencyLevel(int code) + { + this.code = code; + } +} diff --git a/test/distributed/org/apache/cassandra/distributed/api/IClassTransformer.java b/test/distributed/org/apache/cassandra/distributed/api/IClassTransformer.java new file mode 100644 index 0000000000..dd26822958 --- /dev/null +++ b/test/distributed/org/apache/cassandra/distributed/api/IClassTransformer.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.distributed.api; + +public interface IClassTransformer { + + /** + * Modify the bytecode of the provided class. Provides the original bytecode and the fully qualified name of the class. + * Note, bytecode may be null indicating the class definition could not be found. In this case a synthetic definition + * may be returned, or null. + */ + byte[] transform(String name, byte[] bytecode); + + default IClassTransformer initialise() { return this; } +} diff --git a/test/distributed/org/apache/cassandra/distributed/api/ICoordinator.java b/test/distributed/org/apache/cassandra/distributed/api/ICoordinator.java new file mode 100644 index 0000000000..6415a303de --- /dev/null +++ b/test/distributed/org/apache/cassandra/distributed/api/ICoordinator.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.distributed.api; + +import java.util.Iterator; +import java.util.UUID; +import java.util.concurrent.Future; + +import org.apache.cassandra.distributed.shared.FutureUtils; + +// The cross-version API requires that a Coordinator can be constructed without any constructor arguments +public interface ICoordinator +{ + default Object[][] execute(String query, ConsistencyLevel consistencyLevel, Object... boundValues) + { + return executeWithResult(query, consistencyLevel, boundValues).toObjectArrays(); + } + + default Object[][] execute(String query, ConsistencyLevel serialConsistencyLevel, ConsistencyLevel commitConsistencyLevel, Object... boundValues) + { + return executeWithResult(query, serialConsistencyLevel, commitConsistencyLevel, boundValues).toObjectArrays(); + } + + SimpleQueryResult executeWithResult(String query, ConsistencyLevel consistencyLevel, Object... boundValues); + + default SimpleQueryResult executeWithResult(String query, ConsistencyLevel serialConsistencyLevel, ConsistencyLevel commitConsistencyLevel, Object... boundValues) + { + throw new UnsupportedOperationException(); + } + + default Iterator executeWithPaging(String query, ConsistencyLevel consistencyLevel, int pageSize, Object... boundValues) + { + return executeWithPagingWithResult(query, consistencyLevel, pageSize, boundValues).map(Row::toObjectArray); + } + + QueryResult executeWithPagingWithResult(String query, ConsistencyLevel consistencyLevel, int pageSize, Object... boundValues); + + default Future asyncExecuteWithTracing(UUID sessionId, String query, ConsistencyLevel consistencyLevel, Object... boundValues) + { + return FutureUtils.map(asyncExecuteWithTracingWithResult(sessionId, query, consistencyLevel, boundValues), r -> r.toObjectArrays()); + } + + Future asyncExecuteWithTracingWithResult(UUID sessionId, String query, ConsistencyLevel consistencyLevel, Object... boundValues); + + default Object[][] executeWithTracing(UUID sessionId, String query, ConsistencyLevel consistencyLevel, Object... boundValues) + { + return executeWithTracingWithResult(sessionId, query, consistencyLevel, boundValues).toObjectArrays(); + } + + default SimpleQueryResult executeWithTracingWithResult(UUID sessionId, String query, ConsistencyLevel consistencyLevel, Object... boundValues) + { + return FutureUtils.waitOn(asyncExecuteWithTracingWithResult(sessionId, query, consistencyLevel, boundValues)); + } + + IInstance instance(); +} diff --git a/test/distributed/org/apache/cassandra/distributed/api/IMessage.java b/test/distributed/org/apache/cassandra/distributed/api/IMessage.java new file mode 100644 index 0000000000..cf7e920041 --- /dev/null +++ b/test/distributed/org/apache/cassandra/distributed/api/IMessage.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.distributed.api; + +import java.io.Serializable; +import java.net.InetSocketAddress; + +/** + * A cross-version interface for delivering internode messages via message sinks. + *

    + * Message implementations should be serializable so we could load into instances. + */ +public interface IMessage extends Serializable +{ + int verb(); + + byte[] bytes(); + + // TODO: need to make this a long + int id(); + + int version(); + + InetSocketAddress from(); + + default long expiresAtNanos() { return -1L; } +} diff --git a/test/distributed/org/apache/cassandra/distributed/api/QueryResult.java b/test/distributed/org/apache/cassandra/distributed/api/QueryResult.java new file mode 100644 index 0000000000..90cc83fedd --- /dev/null +++ b/test/distributed/org/apache/cassandra/distributed/api/QueryResult.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.distributed.api; + +import java.util.Iterator; +import java.util.List; +import java.util.function.Function; +import java.util.function.Predicate; + +/** + * A table of data representing a complete query result. + *

    + * A QueryResult is different from {@link java.sql.ResultSet} in several key ways: + * + *

      + *
    • represents a complete result rather than a cursor
      • + *
    • returns a {@link Row} to access the current row of data
      • + *
    • relies on object pooling; {@link #hasNext()} may return the same object just with different data, accessing a + * {@link Row} from a previous {@link #hasNext()} call has undefined behavior.
      • + *
    • includes {@link #filter(Predicate)}, this will do client side filtering since Apache Cassandra is more + * restrictive on server side filtering
      • + *
    + * + *

    Unsafe patterns

    + *

    + * Below are a few unsafe patterns which may lead to unexpected results + * + * {@code + * while (rs.hasNext()) { + * list.add(rs.next()); + * } + * } + * + * {@code + * rs.forEach(list::add) + * } + *

    + * Both cases have the same issue; reference to a row from a previous call to {@link #hasNext()}. Since the same {@link Row} + * object can be used across different calls to {@link #hasNext()} this would mean any attempt to access after the fact + * points to newer data. If this behavior is not desirable and access is needed between calls, then {@link Row#copy()} + * should be used; this will clone the {@link Row} and return a new object pointing to the same data. + */ +public interface QueryResult extends Iterator +{ + List names(); + + List warnings(); + + default QueryResult filter(Predicate fn) + { + return QueryResults.filter(this, fn); + } + + default Iterator map(Function fn) + { + return new Iterator() + { + @Override + public boolean hasNext() + { + return QueryResult.this.hasNext(); + } + + @Override + public A next() + { + return fn.apply(QueryResult.this.next()); + } + }; + } +} diff --git a/test/distributed/org/apache/cassandra/distributed/api/QueryResults.java b/test/distributed/org/apache/cassandra/distributed/api/QueryResults.java new file mode 100644 index 0000000000..081d06a525 --- /dev/null +++ b/test/distributed/org/apache/cassandra/distributed/api/QueryResults.java @@ -0,0 +1,224 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.distributed.api; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.NoSuchElementException; +import java.util.Objects; +import java.util.function.Predicate; + +public final class QueryResults +{ + private static final SimpleQueryResult EMPTY = new SimpleQueryResult(new String[0], null); + + private QueryResults() {} + + public static SimpleQueryResult empty() + { + return EMPTY; + } + + public static QueryResult fromIterator(String[] names, Iterator iterator) + { + Objects.requireNonNull(names, "names"); + Objects.requireNonNull(iterator, "iterator"); + return new IteratorQueryResult(names, iterator); + } + + public static QueryResult fromObjectArrayIterator(String[] names, Iterator iterator) + { + Row row = new Row(names); + return fromIterator(names, new Iterator() + { + @Override + public boolean hasNext() + { + return iterator.hasNext(); + } + + @Override + public Row next() + { + row.setResults(iterator.next()); + return row; + } + }); + } + + public static QueryResult filter(QueryResult result, Predicate fn) + { + return new FilterQueryResult(result, fn); + } + + public static Builder builder() + { + return new Builder(); + } + + public static final class Builder + { + private static final int UNSET = -1; + + private int numColumns = UNSET; + private String[] names; + private final List results = new ArrayList<>(); + private final List warnings = new ArrayList<>(); + + public Builder columns(String... columns) + { + if (columns != null) + { + if (numColumns == UNSET) + numColumns = columns.length; + + if (numColumns != columns.length) + throw new AssertionError("Attempted to add column names with different column count; " + + "expected " + numColumns + " columns but given " + Arrays.toString(columns)); + } + + names = columns; + return this; + } + + public Builder row(Object... values) + { + if (numColumns == UNSET) + numColumns = values.length; + + if (numColumns != values.length) + throw new AssertionError("Attempted to add row with different column count; " + + "expected " + numColumns + " columns but given " + Arrays.toString(values)); + results.add(values); + return this; + } + + public Builder warning(String message) + { + warnings.add(message); + return this; + } + + public SimpleQueryResult build() + { + if (names == null) + { + if (numColumns == UNSET) + return QueryResults.empty(); + names = new String[numColumns]; + for (int i = 0; i < numColumns; i++) + names[i] = "unknown"; + } + + return new SimpleQueryResult(names, results.toArray(new Object[0][]), warnings); + } + } + + private static final class IteratorQueryResult implements QueryResult + { + private final List names; + private final Iterator iterator; + + private IteratorQueryResult(String[] names, Iterator iterator) + { + this(Collections.unmodifiableList(Arrays.asList(names)), iterator); + } + + private IteratorQueryResult(List names, Iterator iterator) + { + this.names = names; + this.iterator = iterator; + } + + @Override + public List names() + { + return names; + } + + @Override + public List warnings() + { + throw new UnsupportedOperationException("Warnings are not yet supported for " + getClass().getSimpleName()); + } + + @Override + public boolean hasNext() + { + return iterator.hasNext(); + } + + @Override + public Row next() + { + return iterator.next(); + } + } + + private static final class FilterQueryResult implements QueryResult + { + private final QueryResult delegate; + private final Predicate filter; + private Row current; + + private FilterQueryResult(QueryResult delegate, Predicate filter) + { + this.delegate = delegate; + this.filter = filter; + } + + @Override + public List names() + { + return delegate.names(); + } + + @Override + public List warnings() + { + return delegate.warnings(); + } + + @Override + public boolean hasNext() + { + while (delegate.hasNext()) + { + Row row = delegate.next(); + if (filter.test(row)) + { + current = row; + return true; + } + } + current = null; + return false; + } + + @Override + public Row next() + { + if (current == null) + throw new NoSuchElementException(); + return current; + } + } +} diff --git a/test/distributed/org/apache/cassandra/distributed/api/SimpleQueryResult.java b/test/distributed/org/apache/cassandra/distributed/api/SimpleQueryResult.java new file mode 100644 index 0000000000..2b71e8b8b1 --- /dev/null +++ b/test/distributed/org/apache/cassandra/distributed/api/SimpleQueryResult.java @@ -0,0 +1,163 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.distributed.api; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.NoSuchElementException; +import java.util.Objects; +import java.util.function.Predicate; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** + * A table of data representing a complete query result. + *

    + * A QueryResult is different from {@link java.sql.ResultSet} in several key ways: + * + *

    + * + *

    Unsafe patterns

    + *

    + * Below are a few unsafe patterns which may lead to unexpected results + * + * {@code + * while (rs.hasNext()) { + * list.add(rs.next()); + * } + * } + * + * {@code + * rs.forEach(list::add) + * } + *

    + * Both cases have the same issue; reference to a row from a previous call to {@link #hasNext()}. Since the same {@link Row} + * object can be used accross different calls to {@link #hasNext()} this would mean any attempt to access after the fact + * points to newer data. If this behavior is not desirable and access is needed between calls, then {@link Row#copy()} + * should be used; this will clone the {@link Row} and return a new object pointing to the same data. + */ +public class SimpleQueryResult implements QueryResult +{ + private final String[] names; + private final Object[][] results; + private final List warnings; + private final Predicate filter; + private final Row row; + private int offset = -1; + + public SimpleQueryResult(String[] names, Object[][] results) + { + this(names, results, Collections.emptyList()); + } + + public SimpleQueryResult(String[] names, Object[][] results, List warnings) + { + this.names = Objects.requireNonNull(names, "names"); + this.results = results; + this.warnings = Objects.requireNonNull(warnings, "warnings"); + this.row = new Row(names); + this.filter = ignore -> true; + } + + private SimpleQueryResult(String[] names, Object[][] results, Predicate filter, int offset) + { + this.names = names; + this.results = results; + this.warnings = Collections.emptyList(); + this.filter = filter; + this.offset = offset; + this.row = new Row(names); + } + + public List names() + { + return Collections.unmodifiableList(Arrays.asList(names)); + } + + @Override + public List warnings() + { + return Collections.unmodifiableList(warnings); + } + + public SimpleQueryResult filter(Predicate fn) + { + return new SimpleQueryResult(names, results, filter.and(fn), offset); + } + + /** + * Reset the cursor to the start of the query result; if the query result has not been iterated, this has no effect. + */ + public void reset() + { + offset = -1; + row.setResults(null); + } + + /** + * Get all rows as a 2d array. Any calls to {@link #filter(Predicate)} will be ignored and the array returned will + * be the full set from the query. + */ + public Object[][] toObjectArrays() + { + return results; + } + + @Override + public boolean hasNext() + { + if (results == null) + return false; + while ((offset += 1) < results.length) + { + row.setResults(results[offset]); + if (filter.test(row)) + { + return true; + } + } + row.setResults(null); + return false; + } + + @Override + public Row next() + { + // no null check needed for results since offset only increments IFF results is not null + if (offset < 0 || offset >= results.length) + throw new NoSuchElementException(); + return row; + } + + @Override + public String toString() { + if (results == null) + return "[]"; + return Stream.of(results) + .map(Arrays::toString) + .collect(Collectors.joining(",", "[", "]")); + } +} diff --git a/test/distributed/org/apache/cassandra/distributed/impl/AbstractCluster.java b/test/distributed/org/apache/cassandra/distributed/impl/AbstractCluster.java index 70d9d01e0e..1a76ce0134 100644 --- a/test/distributed/org/apache/cassandra/distributed/impl/AbstractCluster.java +++ b/test/distributed/org/apache/cassandra/distributed/impl/AbstractCluster.java @@ -38,11 +38,13 @@ import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.UUID; +import java.util.concurrent.Callable; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.Executor; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; +import java.util.function.BiFunction; import java.util.function.BiPredicate; import java.util.function.Consumer; import java.util.function.Function; @@ -81,7 +83,6 @@ import org.apache.cassandra.distributed.api.LogAction; import org.apache.cassandra.distributed.api.NodeToolResult; import org.apache.cassandra.distributed.api.TokenSupplier; import org.apache.cassandra.distributed.shared.InstanceClassLoader; -import org.apache.cassandra.distributed.shared.MessageFilters; import org.apache.cassandra.distributed.shared.Metrics; import org.apache.cassandra.distributed.shared.NetworkTopology; import org.apache.cassandra.distributed.shared.ShutdownException; @@ -98,6 +99,7 @@ import org.reflections.scanners.TypeAnnotationsScanner; import org.reflections.util.ConfigurationBuilder; import static java.util.stream.Stream.of; +import static org.apache.cassandra.distributed.impl.IsolatedExecutor.DEFAULT_SHUTDOWN_EXECUTOR; import static org.apache.cassandra.distributed.shared.NetworkTopology.addressAndPort; import static org.apache.cassandra.utils.Shared.Recursive.ALL; import static org.apache.cassandra.utils.Shared.Recursive.NONE; @@ -168,6 +170,7 @@ public abstract class AbstractCluster implements ICluster uncaughtExceptions = new CopyOnWriteArrayList<>(); private final ThreadGroup clusterThreadGroup = new ThreadGroup(clusterId.toString()); + private final ShutdownExecutor shutdownExecutor; private volatile IMessageSink messageSink; @@ -178,6 +181,7 @@ public abstract class AbstractCluster implements ICluster { private INodeProvisionStrategy.Strategy nodeProvisionStrategy = INodeProvisionStrategy.Strategy.MultipleNetworkInterfaces; + private ShutdownExecutor shutdownExecutor = DEFAULT_SHUTDOWN_EXECUTOR; public AbstractBuilder(Factory factory) { @@ -190,6 +194,12 @@ public abstract class AbstractCluster implements ICluster implements ICluster implements ICluster 1 ? "_" + generation : "")); if (instanceInitializer != null) instanceInitializer.initialise(classLoader, threadGroup, config.num(), generation); @@ -237,13 +248,25 @@ public abstract class AbstractCluster implements ICluster)Instance::new, classLoader) - .apply(config.forVersion(version.version), classLoader, root.getFileSystem()); + instance = Instance.transferAdhocPropagate((SerializableQuadFunction)Instance::new, classLoader) + .apply(config.forVersion(version.version), classLoader, root.getFileSystem(), shutdownExecutor); } catch (InvocationTargetException e) { - instance = Instance.transferAdhoc((SerializableBiFunction)Instance::new, classLoader) - .apply(config.forVersion(version.version), classLoader); + try + { + instance = Instance.transferAdhocPropagate((SerializableTriFunction)Instance::new, classLoader) + .apply(config.forVersion(version.version), classLoader, root.getFileSystem()); + } + catch (InvocationTargetException e2) + { + instance = Instance.transferAdhoc((SerializableBiFunction)Instance::new, classLoader) + .apply(config.forVersion(version.version), classLoader); + } + catch (IllegalAccessException e2) + { + throw new RuntimeException(e); + } } catch (IllegalAccessException e) { @@ -465,6 +488,7 @@ public abstract class AbstractCluster implements ICluster(); this.instanceMap = new ConcurrentHashMap<>(); this.initialVersion = builder.getVersion(); diff --git a/test/distributed/org/apache/cassandra/distributed/impl/Coordinator.java b/test/distributed/org/apache/cassandra/distributed/impl/Coordinator.java index 754c2a2d34..980ee19973 100644 --- a/test/distributed/org/apache/cassandra/distributed/impl/Coordinator.java +++ b/test/distributed/org/apache/cassandra/distributed/impl/Coordinator.java @@ -63,7 +63,7 @@ public class Coordinator implements ICoordinator @Override public SimpleQueryResult executeWithResult(String query, ConsistencyLevel consistencyLevel, Object... boundValues) { - return instance().sync(() -> executeInternal(query, consistencyLevel, boundValues)).call(); + return instance().sync(() -> unsafeExecuteInternal(query, consistencyLevel, boundValues)).call(); } public Future asyncExecuteWithTracingWithResult(UUID sessionId, String query, ConsistencyLevel consistencyLevelOrigin, Object... boundValues) @@ -72,7 +72,7 @@ public class Coordinator implements ICoordinator try { Tracing.instance.newSession(TimeUUID.fromUuid(sessionId), Collections.emptyMap()); - return executeInternal(query, consistencyLevelOrigin, boundValues); + return unsafeExecuteInternal(query, consistencyLevelOrigin, boundValues); } finally { @@ -81,17 +81,33 @@ public class Coordinator implements ICoordinator }).call(); } - static org.apache.cassandra.db.ConsistencyLevel toCassandraCL(ConsistencyLevel cl) + public static org.apache.cassandra.db.ConsistencyLevel toCassandraCL(ConsistencyLevel cl) { - return org.apache.cassandra.db.ConsistencyLevel.fromCode(cl.ordinal()); + try + { + return org.apache.cassandra.db.ConsistencyLevel.fromCode(cl.code); + } + catch (NoSuchFieldError e) + { + return org.apache.cassandra.db.ConsistencyLevel.fromCode(cl.ordinal()); + } } - private SimpleQueryResult executeInternal(String query, ConsistencyLevel consistencyLevelOrigin, Object[] boundValues) + protected static org.apache.cassandra.db.ConsistencyLevel toCassandraSerialCL(ConsistencyLevel cl) + { + return toCassandraCL(cl == null ? ConsistencyLevel.SERIAL : cl); + } + + public static SimpleQueryResult unsafeExecuteInternal(String query, ConsistencyLevel consistencyLevel, Object[] boundValues) + { + return unsafeExecuteInternal(query, null, consistencyLevel, boundValues); + } + + public static SimpleQueryResult unsafeExecuteInternal(String query, ConsistencyLevel serialConsistencyLevel, ConsistencyLevel commitConsistencyLevel, Object[] boundValues) { ClientState clientState = makeFakeClientState(); CQLStatement prepared = QueryProcessor.getStatement(query, clientState); List boundBBValues = new ArrayList<>(); - ConsistencyLevel consistencyLevel = ConsistencyLevel.valueOf(consistencyLevelOrigin.name()); for (Object boundValue : boundValues) boundBBValues.add(ByteBufferUtil.objectToBytes(boundValue)); @@ -104,12 +120,12 @@ public class Coordinator implements ICoordinator try { ResultMessage res = prepared.execute(QueryState.forInternalCalls(), - QueryOptions.create(toCassandraCL(consistencyLevel), + QueryOptions.create(toCassandraCL(commitConsistencyLevel), boundBBValues, false, Integer.MAX_VALUE, null, - null, + toCassandraSerialCL(serialConsistencyLevel), ProtocolVersion.CURRENT, null), nanoTime()); @@ -142,6 +158,12 @@ public class Coordinator implements ICoordinator return instance; } + @Override + public SimpleQueryResult executeWithResult(String query, ConsistencyLevel serialConsistencyLevel, ConsistencyLevel commitConsistencyLevel, Object... boundValues) + { + return instance.sync(() -> unsafeExecuteInternal(query, serialConsistencyLevel, commitConsistencyLevel, boundValues)).call(); + } + @Override public QueryResult executeWithPagingWithResult(String query, ConsistencyLevel consistencyLevelOrigin, int pageSize, Object... boundValues) { diff --git a/test/distributed/org/apache/cassandra/distributed/impl/DelegatingInvokableInstance.java b/test/distributed/org/apache/cassandra/distributed/impl/DelegatingInvokableInstance.java index 79bf946cc2..23b37682af 100644 --- a/test/distributed/org/apache/cassandra/distributed/impl/DelegatingInvokableInstance.java +++ b/test/distributed/org/apache/cassandra/distributed/impl/DelegatingInvokableInstance.java @@ -52,7 +52,7 @@ public abstract class DelegatingInvokableInstance implements IInvokableInstance @Override public InetSocketAddress broadcastAddress() { - return delegate().broadcastAddress(); + return config().broadcastAddress(); } @Override diff --git a/test/distributed/org/apache/cassandra/distributed/impl/DirectStreamingConnectionFactory.java b/test/distributed/org/apache/cassandra/distributed/impl/DirectStreamingConnectionFactory.java index e598dc1796..72105d8f40 100644 --- a/test/distributed/org/apache/cassandra/distributed/impl/DirectStreamingConnectionFactory.java +++ b/test/distributed/org/apache/cassandra/distributed/impl/DirectStreamingConnectionFactory.java @@ -23,8 +23,8 @@ import java.net.InetSocketAddress; import java.nio.ByteBuffer; import java.nio.channels.ClosedChannelException; import java.nio.channels.FileChannel; -import java.util.concurrent.ScheduledFuture; -import java.util.concurrent.TimeUnit; +import java.util.ArrayDeque; +import java.util.Queue; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Function; import java.util.function.IntFunction; @@ -34,6 +34,7 @@ import org.apache.cassandra.distributed.api.ICluster; import org.apache.cassandra.distributed.api.IInvokableInstance; import org.apache.cassandra.io.util.BufferedDataOutputStreamPlus; import org.apache.cassandra.io.util.RebufferingInputStream; +import org.apache.cassandra.net.OutboundConnectionSettings; import org.apache.cassandra.streaming.StreamDeserializingTask; import org.apache.cassandra.streaming.StreamingDataInputPlus; import org.apache.cassandra.streaming.StreamingDataOutputPlus; @@ -41,12 +42,13 @@ import org.apache.cassandra.streaming.StreamingChannel; import org.apache.cassandra.streaming.StreamingDataOutputPlusFixed; import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.concurrent.ImmediateFuture; -import org.apache.cassandra.utils.concurrent.NotScheduledFuture; import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException; import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory; +import static org.apache.cassandra.locator.InetAddressAndPort.getByAddress; import static org.apache.cassandra.net.MessagingService.*; +// TODO: Simulator should schedule based on some streaming data rate public class DirectStreamingConnectionFactory { static class DirectConnection @@ -54,12 +56,14 @@ public class DirectStreamingConnectionFactory private static final AtomicInteger nextId = new AtomicInteger(); final int protocolVersion; + final long sendBufferSize; // TODO rename private static class Buffer { + final Queue pending = new ArrayDeque<>(); boolean isClosed; - byte[] pending; + int pendingBytes = 0; } @SuppressWarnings({"InnerClassMayBeStatic","unused"}) // helpful for debugging @@ -86,14 +90,15 @@ public class DirectStreamingConnectionFactory { synchronized (out) { - while (out.pending != null && !out.isClosed) + while (out.pendingBytes > 0 && count + out.pendingBytes > sendBufferSize && !out.isClosed) out.wait(); if (out.isClosed) throw new ClosedChannelException(); buffer.flip(); - out.pending = ByteBufferUtil.getArray(buffer); + out.pendingBytes += buffer.remaining(); + out.pending.add(ByteBufferUtil.getArray(buffer)); buffer.clear(); out.notify(); @@ -182,16 +187,18 @@ public class DirectStreamingConnectionFactory { synchronized (in) { - byte[] bytes; - while ((bytes = in.pending) == null && !in.isClosed) + while (in.pendingBytes == 0 && !in.isClosed) in.wait(); - if (bytes == null) + if (in.pendingBytes == 0) throw new ClosedChannelException(); - in.pending = null; - buffer = ByteBuffer.wrap(bytes); + byte[] bytes = in.pending.poll(); + if (bytes == null) + throw new IllegalStateException(); + in.pendingBytes -= bytes.length; + buffer = ByteBuffer.wrap(bytes); in.notify(); } } @@ -322,9 +329,10 @@ public class DirectStreamingConnectionFactory private final DirectStreamingChannel outToRecipient, outToOriginator; - DirectConnection(int protocolVersion, InetSocketAddress originator, InetSocketAddress recipient) + DirectConnection(int protocolVersion, long sendBufferSize, InetSocketAddress originator, InetSocketAddress recipient) { this.protocolVersion = protocolVersion; + this.sendBufferSize = sendBufferSize; Buffer buffer1 = new Buffer(), buffer2 = new Buffer(); outToRecipient = new DirectStreamingChannel(recipient, buffer1, buffer2); outToOriginator = new DirectStreamingChannel(originator, buffer2, buffer1); @@ -349,7 +357,11 @@ public class DirectStreamingConnectionFactory @Override public StreamingChannel create(InetSocketAddress to, int messagingVersion, StreamingChannel.Kind kind) { - DirectConnection connection = new DirectConnection(messagingVersion, from, to); + long sendBufferSize = new OutboundConnectionSettings(getByAddress(to)).socketSendBufferSizeInBytes(); + if (sendBufferSize <= 0) + sendBufferSize = 1 << 14; + + DirectConnection connection = new DirectConnection(messagingVersion, sendBufferSize, from, to); IInvokableInstance instance = cluster.get(to); instance.unsafeAcceptOnThisThread((channel, version) -> executorFactory().startThread(channel.description(), new StreamDeserializingTask(null, channel, version)), connection.get(from), messagingVersion); diff --git a/test/distributed/org/apache/cassandra/distributed/impl/Instance.java b/test/distributed/org/apache/cassandra/distributed/impl/Instance.java index da4592a86a..760d3b1af4 100644 --- a/test/distributed/org/apache/cassandra/distributed/impl/Instance.java +++ b/test/distributed/org/apache/cassandra/distributed/impl/Instance.java @@ -32,7 +32,6 @@ import java.util.List; import java.util.Map; import java.util.Objects; import java.util.UUID; -import java.util.concurrent.CompletableFuture; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; @@ -118,9 +117,12 @@ import org.apache.cassandra.service.PendingRangeCalculatorService; import org.apache.cassandra.service.QueryState; import org.apache.cassandra.service.StorageService; import org.apache.cassandra.service.StorageServiceMBean; +import org.apache.cassandra.service.paxos.PaxosRepair; +import org.apache.cassandra.service.paxos.PaxosState; import org.apache.cassandra.service.reads.trackwarnings.CoordinatorWarnings; import org.apache.cassandra.service.snapshot.SnapshotManager; import org.apache.cassandra.streaming.StreamManager; +import org.apache.cassandra.service.paxos.uncommitted.UncommittedTableData; import org.apache.cassandra.streaming.StreamReceiveTask; import org.apache.cassandra.streaming.StreamTransferTask; import org.apache.cassandra.streaming.async.NettyStreamingChannel; @@ -169,7 +171,12 @@ public class Instance extends IsolatedExecutor implements IInvokableInstance Instance(IInstanceConfig config, ClassLoader classLoader, FileSystem fileSystem) { - super("node" + config.num(), classLoader, executorFactory().pooled("isolatedExecutor", Integer.MAX_VALUE)); + this(config, classLoader, fileSystem, null); + } + + Instance(IInstanceConfig config, ClassLoader classLoader, FileSystem fileSystem, ShutdownExecutor shutdownExecutor) + { + super("node" + config.num(), classLoader, executorFactory().pooled("isolatedExecutor", Integer.MAX_VALUE), shutdownExecutor); this.config = config; if (fileSystem != null) File.unsafeSetFilesystem(fileSystem); @@ -341,12 +348,12 @@ public class Instance extends IsolatedExecutor implements IInvokableInstance { MessagingService.instance().outboundSink.add((message, to) -> { if (isShutdown()) - return false; + return false; // TODO: Simulator needs this to trigger a failure IMessage serialzied = serializeMessage(message.from(), to, message); int fromNum = config.num(); // since this instance is sending the message, from will always be this instance IInstance toInstance = cluster.get(fromCassandraInetAddressAndPort(to)); if (toInstance == null) - return false; + return false; // TODO: Simulator needs this to trigger a failure int toNum = toInstance.config().num(); return cluster.filters().permitOutbound(fromNum, toNum, serialzied); }); @@ -371,6 +378,7 @@ public class Instance extends IsolatedExecutor implements IInvokableInstance ByteArrayUtil.EMPTY_BYTE_ARRAY, messageOut.id(), toVersion, + messageOut.expiresAtNanos(), fromCassandraInetAddressAndPort(from)); } @@ -393,7 +401,7 @@ public class Instance extends IsolatedExecutor implements IInvokableInstance { reserialize(batch, out, toVersion); byte[] bytes = out.toByteArray(); - return new MessageImpl(messageOut.verb().id, bytes, messageOut.id(), toVersion, fromCassandraInetAddressAndPort(from)); + return new MessageImpl(messageOut.verb().id, bytes, messageOut.id(), toVersion, messageOut.expiresAtNanos(), fromCassandraInetAddressAndPort(from)); } } } @@ -404,7 +412,7 @@ public class Instance extends IsolatedExecutor implements IInvokableInstance throw new AssertionError(String.format("Message serializedSize(%s) does not match what was written with serialize(out, %s) for verb %s and serializer %s; " + "expected %s, actual %s", toVersion, toVersion, messageOut.verb(), Message.serializer.getClass(), messageOut.serializedSize(toVersion), bytes.length)); - return new MessageImpl(messageOut.verb().id, bytes, messageOut.id(), toVersion, fromCassandraInetAddressAndPort(from)); + return new MessageImpl(messageOut.verb().id, bytes, messageOut.id(), toVersion, messageOut.expiresAtNanos(), fromCassandraInetAddressAndPort(from)); } catch (IOException e) { @@ -594,6 +602,15 @@ public class Instance extends IsolatedExecutor implements IInvokableInstance // Re-populate token metadata after commit log recover (new peers might be loaded onto system keyspace #10293) StorageService.instance.populateTokenMetadata(); + try + { + PaxosState.maybeRebuildUncommittedState(); + } + catch (IOException e) + { + throw new RuntimeException(e); + } + Verb.HINT_REQ.unsafeSetSerializer(DTestSerializer::new); if (config.has(NETWORK)) @@ -636,6 +653,7 @@ public class Instance extends IsolatedExecutor implements IInvokableInstance else { Stream peers = cluster.stream().filter(instance -> ((IInstance) instance).isValid()); + SystemKeyspace.setLocalHostId(config.hostId()); if (config.has(BLANK_GOSSIP)) peers.forEach(peer -> GossipHelper.statusToBlank((IInvokableInstance) peer).accept(this)); else if (cluster instanceof Cluster) @@ -713,7 +731,7 @@ public class Instance extends IsolatedExecutor implements IInvokableInstance @Override public Future shutdown(boolean graceful) { - if (!graceful) + if (!graceful && config.has(NETWORK)) MessagingService.instance().shutdown(1L, MINUTES, false, true); Future future = async((ExecutorService executor) -> { @@ -745,13 +763,16 @@ public class Instance extends IsolatedExecutor implements IInvokableInstance () -> SecondaryIndexManager.shutdownAndWait(1L, MINUTES), () -> IndexSummaryManager.instance.shutdownAndWait(1L, MINUTES), () -> ColumnFamilyStore.shutdownExecutorsAndWait(1L, MINUTES), - () -> PendingRangeCalculatorService.instance.shutdownAndWait(1L, MINUTES), () -> BufferPools.shutdownLocalCleaner(1L, MINUTES), + () -> PaxosRepair.shutdownAndWait(1L, MINUTES), () -> Ref.shutdownReferenceReaper(1L, MINUTES), + () -> UncommittedTableData.shutdownAndWait(1L, MINUTES), () -> Memtable.MEMORY_POOL.shutdownAndWait(1L, MINUTES), () -> DiagnosticSnapshotService.instance.shutdownAndWait(1L, MINUTES), () -> SSTableReader.shutdownBlocking(1L, MINUTES), () -> shutdownAndWait(Collections.singletonList(ActiveRepairService.repairCommandExecutor())), + () -> ScheduledExecutors.shutdownNowAndWait(1L, MINUTES), + () -> ActiveRepairService.instance.shutdownNowAndWait(1L, MINUTES), () -> SnapshotManager.shutdownAndWait(1L, MINUTES) ); @@ -760,10 +781,10 @@ public class Instance extends IsolatedExecutor implements IInvokableInstance error = parallelRun(error, executor, CommitLog.instance::shutdownBlocking, // can only shutdown message once, so if the test shutsdown an instance, then ignore the failure - (IgnoreThrowingRunnable) () -> MessagingService.instance().shutdown(1L, MINUTES, false, true) + (IgnoreThrowingRunnable) () -> MessagingService.instance().shutdown(1L, MINUTES, false, config.has(NETWORK)) ); error = parallelRun(error, executor, - () -> { try { GlobalEventExecutor.INSTANCE.awaitInactivity(1L, MINUTES); } catch (IllegalStateException ignore) {} }, + () -> { if (config.has(NETWORK)) { try { GlobalEventExecutor.INSTANCE.awaitInactivity(1L, MINUTES); } catch (IllegalStateException ignore) {} } }, () -> Stage.shutdownAndWait(1L, MINUTES), () -> SharedExecutorPool.SHARED.shutdownAndWait(1L, MINUTES) ); @@ -786,8 +807,17 @@ public class Instance extends IsolatedExecutor implements IInvokableInstance Throwables.maybeFail(error); }).apply(isolatedExecutor); - return CompletableFuture.runAsync(ThrowingRunnable.toRunnable(future::get), isolatedExecutor) - .thenRun(super::shutdown); + return isolatedExecutor.submit(() -> { + try + { + future.get(); + return null; + } + finally + { + super.shutdown(); + } + }); } @Override @@ -972,8 +1002,7 @@ public class Instance extends IsolatedExecutor implements IInvokableInstance private static void shutdownAndWait(List executors) throws TimeoutException, InterruptedException { - ExecutorUtils.shutdownNow(executors); - ExecutorUtils.awaitTermination(1L, MINUTES, executors); + ExecutorUtils.shutdownNowAndWait(1L, MINUTES, executors); } private static Throwable parallelRun(Throwable accumulate, ExecutorService runOn, ThrowingRunnable ... runnables) diff --git a/test/distributed/org/apache/cassandra/distributed/impl/IsolatedExecutor.java b/test/distributed/org/apache/cassandra/distributed/impl/IsolatedExecutor.java index c9c0087971..68ff1e71c6 100644 --- a/test/distributed/org/apache/cassandra/distributed/impl/IsolatedExecutor.java +++ b/test/distributed/org/apache/cassandra/distributed/impl/IsolatedExecutor.java @@ -46,7 +46,6 @@ import org.slf4j.LoggerFactory; import ch.qos.logback.classic.LoggerContext; import io.netty.util.concurrent.FastThreadLocal; import org.apache.cassandra.concurrent.ExecutorFactory; -import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.distributed.api.IIsolatedExecutor; import org.apache.cassandra.utils.ExecutorUtils; import org.apache.cassandra.utils.Throwables; @@ -59,18 +58,56 @@ public class IsolatedExecutor implements IIsolatedExecutor private final String name; final ClassLoader classLoader; private final DynamicFunction transfer; + private final ShutdownExecutor shutdownExecutor; + public static final ShutdownExecutor DEFAULT_SHUTDOWN_EXECUTOR = (name, classLoader, shuttingDown, onTermination) -> { + /* Use a thread pool with a core pool size of zero to terminate the thread as soon as possible + ** so the instance class loader can be garbage collected. Uses a custom thread factory + ** rather than NamedThreadFactory to avoid calling FastThreadLocal.removeAll() in 3.0 and up + ** as it was observed crashing during test failures and made it harder to find the real cause. + */ + ThreadFactory threadFactory = (Runnable r) -> { + Thread t = new Thread(r, name + "_shutdown"); + t.setDaemon(true); + return t; + }; + + ExecutorService shutdownExecutor = new ThreadPoolExecutor(0, Integer.MAX_VALUE, 0, SECONDS, + new LinkedBlockingQueue<>(), threadFactory); + return shutdownExecutor.submit(() -> { + try + { + ExecutorUtils.awaitTermination(60, TimeUnit.SECONDS, shuttingDown); + return onTermination.call(); + } + finally + { + shutdownExecutor.shutdownNow(); + } + }); + }; + + // retained for backwards compatibility + @SuppressWarnings("unused") public IsolatedExecutor(String name, ClassLoader classLoader, ExecutorFactory executorFactory) { - this(name, classLoader, executorFactory.pooled("isolatedExecutor", Integer.MAX_VALUE)); + this(name, classLoader, executorFactory.pooled("isolatedExecutor", Integer.MAX_VALUE), DEFAULT_SHUTDOWN_EXECUTOR); } - IsolatedExecutor(String name, ClassLoader classLoader, ExecutorService executorService) + // retained for backwards compatibility + @SuppressWarnings("unused") + public IsolatedExecutor(String name, ClassLoader classLoader, ExecutorService executorService) + { + this(name, classLoader, executorService, DEFAULT_SHUTDOWN_EXECUTOR); + } + + IsolatedExecutor(String name, ClassLoader classLoader, ExecutorService executorService, ShutdownExecutor shutdownExecutor) { this.name = name; this.isolatedExecutor = executorService; this.classLoader = classLoader; this.transfer = transferTo(classLoader); + this.shutdownExecutor = shutdownExecutor; } protected IsolatedExecutor(IsolatedExecutor from, ExecutorService executor) @@ -79,6 +116,7 @@ public class IsolatedExecutor implements IIsolatedExecutor this.isolatedExecutor = executor; this.classLoader = from.classLoader; this.transfer = from.transfer; + this.shutdownExecutor = from.shutdownExecutor; } public IIsolatedExecutor with(ExecutorService executor) @@ -89,40 +127,19 @@ public class IsolatedExecutor implements IIsolatedExecutor public Future shutdown() { isolatedExecutor.shutdownNow(); + return shutdownExecutor.shutdown(name, classLoader, isolatedExecutor, () -> { - /* Use a thread pool with a core pool size of zero to terminate the thread as soon as possible - ** so the instance class loader can be garbage collected. Uses a custom thread factory - ** rather than NamedThreadFactory to avoid calling FastThreadLocal.removeAll() in 3.0 and up - ** as it was observed crashing during test failures and made it harder to find the real cause. - */ - ThreadFactory threadFactory = (Runnable r) -> { - Thread t = new Thread(r, name + "_shutdown"); - t.setDaemon(true); - return t; - }; - ExecutorService shutdownExecutor = new ThreadPoolExecutor(0, Integer.MAX_VALUE, 0, SECONDS, - new LinkedBlockingQueue<>(), threadFactory); - return shutdownExecutor.submit(() -> { - try - { - ExecutorUtils.awaitTermination(60, TimeUnit.SECONDS, isolatedExecutor); + // Shutdown logging last - this is not ideal as the logging subsystem is initialized + // outsize of this class, however doing it this way provides access to the full + // logging system while termination is taking place. + LoggerContext loggerContext = (LoggerContext) LoggerFactory.getILoggerFactory(); + loggerContext.stop(); - // Shutdown logging last - this is not ideal as the logging subsystem is initialized - // outsize of this class, however doing it this way provides access to the full - // logging system while termination is taking place. - LoggerContext loggerContext = (LoggerContext) LoggerFactory.getILoggerFactory(); - loggerContext.stop(); + FastThreadLocal.destroy(); - FastThreadLocal.destroy(); - - // Close the instance class loader after shutting down the isolatedExecutor and logging - // in case error handling triggers loading additional classes - ((URLClassLoader) classLoader).close(); - } - finally - { - shutdownExecutor.shutdownNow(); - } + // Close the instance class loader after shutting down the isolatedExecutor and logging + // in case error handling triggers loading additional classes + ((URLClassLoader) classLoader).close(); return null; }); } diff --git a/test/distributed/org/apache/cassandra/distributed/impl/MessageFilters.java b/test/distributed/org/apache/cassandra/distributed/impl/MessageFilters.java new file mode 100644 index 0000000000..84db349802 --- /dev/null +++ b/test/distributed/org/apache/cassandra/distributed/impl/MessageFilters.java @@ -0,0 +1,207 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.distributed.impl; + +import java.util.Arrays; +import java.util.List; +import java.util.Objects; +import java.util.concurrent.CopyOnWriteArrayList; + +import org.apache.cassandra.distributed.api.IMessage; +import org.apache.cassandra.distributed.api.IMessageFilters; + +// note: as a pure implementation class of an interface, this should not live in dtest-api +public class MessageFilters implements IMessageFilters +{ + private final List inboundFilters = new CopyOnWriteArrayList<>(); + private final List outboundFilters = new CopyOnWriteArrayList<>(); + + public boolean permitInbound(int from, int to, IMessage msg) + { + return permit(inboundFilters, from, to, msg); + } + + public boolean permitOutbound(int from, int to, IMessage msg) + { + return permit(outboundFilters, from, to, msg); + } + + @Override + public boolean hasInbound() + { + return !inboundFilters.isEmpty(); + } + + @Override + public boolean hasOutbound() + { + return !outboundFilters.isEmpty(); + } + + private static boolean permit(List filters, int from, int to, IMessage msg) + { + for (Filter filter : filters) + { + if (filter.matches(from, to, msg)) + return false; + } + return true; + } + + public static class Filter implements IMessageFilters.Filter + { + final int[] from; + final int[] to; + final int[] verbs; + final Matcher matcher; + final List parent; + + Filter(int[] from, int[] to, int[] verbs, Matcher matcher, List parent) + { + if (from != null) + { + from = from.clone(); + Arrays.sort(from); + } + if (to != null) + { + to = to.clone(); + Arrays.sort(to); + } + if (verbs != null) + { + verbs = verbs.clone(); + Arrays.sort(verbs); + } + this.from = from; + this.to = to; + this.verbs = verbs; + this.matcher = matcher; + this.parent = Objects.requireNonNull(parent, "parent"); + } + + public int hashCode() + { + return (from == null ? 0 : Arrays.hashCode(from)) + + (to == null ? 0 : Arrays.hashCode(to)) + + (verbs == null ? 0 : Arrays.hashCode(verbs) + + parent.hashCode()); + } + + public boolean equals(Object that) + { + return that instanceof Filter && equals((Filter) that); + } + + public boolean equals(Filter that) + { + return Arrays.equals(from, that.from) + && Arrays.equals(to, that.to) + && Arrays.equals(verbs, that.verbs) + && parent.equals(that.parent); + } + + public Filter off() + { + parent.remove(this); + return this; + } + + public Filter on() + { + parent.add(this); + return this; + } + + public boolean matches(int from, int to, IMessage msg) + { + return (this.from == null || Arrays.binarySearch(this.from, from) >= 0) + && (this.to == null || Arrays.binarySearch(this.to, to) >= 0) + && (this.verbs == null || Arrays.binarySearch(this.verbs, msg.verb()) >= 0) + && (this.matcher == null || this.matcher.matches(from, to, msg)); + } + } + + public class Builder implements IMessageFilters.Builder + { + int[] from; + int[] to; + int[] verbs; + Matcher matcher; + boolean inbound; + + private Builder(boolean inbound) + { + this.inbound = inbound; + } + + public Builder from(int... nums) + { + from = nums; + return this; + } + + public Builder to(int... nums) + { + to = nums; + return this; + } + + public IMessageFilters.Builder verbs(int... verbs) + { + this.verbs = verbs; + return this; + } + + public IMessageFilters.Builder allVerbs() + { + this.verbs = null; + return this; + } + + public IMessageFilters.Builder inbound(boolean inbound) + { + this.inbound = inbound; + return this; + } + + public IMessageFilters.Builder messagesMatching(Matcher matcher) + { + this.matcher = matcher; + return this; + } + + public IMessageFilters.Filter drop() + { + return new Filter(from, to, verbs, matcher, inbound ? inboundFilters : outboundFilters).on(); + } + } + + public IMessageFilters.Builder inbound(boolean inbound) + { + return new Builder(inbound); + } + + @Override + public void reset() + { + inboundFilters.clear(); + outboundFilters.clear(); + } +} diff --git a/test/distributed/org/apache/cassandra/distributed/impl/MessageImpl.java b/test/distributed/org/apache/cassandra/distributed/impl/MessageImpl.java index 607e890ff1..758d413583 100644 --- a/test/distributed/org/apache/cassandra/distributed/impl/MessageImpl.java +++ b/test/distributed/org/apache/cassandra/distributed/impl/MessageImpl.java @@ -30,14 +30,16 @@ public class MessageImpl implements IMessage public final byte[] bytes; public final long id; public final int version; + public final long expiresAtNanos; public final InetSocketAddress from; - public MessageImpl(int verb, byte[] bytes, long id, int version, InetSocketAddress from) + public MessageImpl(int verb, byte[] bytes, long id, int version, long expiresAtNanos, InetSocketAddress from) { this.verb = verb; this.bytes = bytes; this.id = id; this.version = version; + this.expiresAtNanos = expiresAtNanos; this.from = from; } @@ -61,6 +63,12 @@ public class MessageImpl implements IMessage return version; } + @Override + public long expiresAtNanos() + { + return expiresAtNanos; + } + public InetSocketAddress from() { return from; diff --git a/test/distributed/org/apache/cassandra/distributed/impl/ShutdownExecutor.java b/test/distributed/org/apache/cassandra/distributed/impl/ShutdownExecutor.java new file mode 100644 index 0000000000..15ed4a2ddb --- /dev/null +++ b/test/distributed/org/apache/cassandra/distributed/impl/ShutdownExecutor.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.distributed.impl; + +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; + +import org.apache.cassandra.utils.Shared; + +@Shared +public interface ShutdownExecutor +{ + Future shutdown(String name, ClassLoader classLoader, ExecutorService shuttingDown, Callable onTermination); +} diff --git a/test/distributed/org/apache/cassandra/distributed/impl/UnsafeGossipHelper.java b/test/distributed/org/apache/cassandra/distributed/impl/UnsafeGossipHelper.java index 9680b2d403..97fa7bcbaa 100644 --- a/test/distributed/org/apache/cassandra/distributed/impl/UnsafeGossipHelper.java +++ b/test/distributed/org/apache/cassandra/distributed/impl/UnsafeGossipHelper.java @@ -42,6 +42,7 @@ import org.apache.cassandra.service.PendingRangeCalculatorService; import org.apache.cassandra.service.StorageService; import org.apache.cassandra.utils.FBUtilities; +import static com.google.common.collect.Iterables.getOnlyElement; import static java.util.Collections.singleton; import static org.apache.cassandra.distributed.impl.DistributedTestSnitch.toCassandraInetAddressAndPort; import static org.apache.cassandra.locator.InetAddressAndPort.getByAddress; @@ -86,8 +87,27 @@ public class UnsafeGossipHelper Token token; if (FBUtilities.getBroadcastAddressAndPort().equals(addressAndPort)) { - String str = tokenString == null ? Iterables.getOnlyElement(DatabaseDescriptor.getInitialTokens()) : tokenString; - token = DatabaseDescriptor.getPartitioner().getTokenFactory().fromString(str); + // try grabbing saved tokens so that - if we're leaving - we get the ones we may have adopted as part of a range movement + // if that fails, grab them from config (as we're probably joining and should just use the default token) + Token.TokenFactory tokenFactory = DatabaseDescriptor.getPartitioner().getTokenFactory(); + if (tokenString == null) + { + Token tmp; + try + { + tmp = getOnlyElement(SystemKeyspace.getSavedTokens()); + } + catch (Throwable t) + { + tmp = tokenFactory.fromString(getOnlyElement(DatabaseDescriptor.getInitialTokens())); + } + token = tmp; + } + else + { + token = tokenFactory.fromString(tokenString); + } + SystemKeyspace.setLocalHostId(hostId); SystemKeyspace.updateTokens(singleton(token)); } diff --git a/test/distributed/org/apache/cassandra/distributed/test/CASCommonTestCases.java b/test/distributed/org/apache/cassandra/distributed/test/CASCommonTestCases.java new file mode 100644 index 0000000000..151abb742b --- /dev/null +++ b/test/distributed/org/apache/cassandra/distributed/test/CASCommonTestCases.java @@ -0,0 +1,204 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.distributed.test; + +import org.junit.Assert; +import org.junit.Test; + +import org.apache.cassandra.distributed.Cluster; +import org.apache.cassandra.distributed.api.IMessageFilters; +import org.apache.cassandra.exceptions.CasWriteTimeoutException; + +import static org.apache.cassandra.distributed.shared.AssertUtils.assertRows; +import static org.apache.cassandra.distributed.shared.AssertUtils.row; +import static org.apache.cassandra.net.Verb.PAXOS2_PREPARE_REQ; +import static org.apache.cassandra.net.Verb.PAXOS2_PROPOSE_REQ; +import static org.apache.cassandra.net.Verb.PAXOS_PREPARE_REQ; +import static org.apache.cassandra.net.Verb.PAXOS_PROPOSE_REQ; + +public abstract class CASCommonTestCases extends CASTestBase +{ + protected abstract Cluster getCluster(); + + @Test + public void simpleUpdate() throws Throwable + { + String tableName = tableName(); + String fullTableName = KEYSPACE + "." + tableName; + getCluster().schemaChange("CREATE TABLE " + fullTableName + " (pk int, ck int, v int, PRIMARY KEY (pk, ck))"); + + getCluster().coordinator(1).execute("INSERT INTO " + fullTableName + " (pk, ck, v) VALUES (1, 1, 1) IF NOT EXISTS", org.apache.cassandra.distributed.api.ConsistencyLevel.QUORUM); + assertRows(getCluster().coordinator(1).execute("SELECT * FROM " + fullTableName + " WHERE pk = 1", org.apache.cassandra.distributed.api.ConsistencyLevel.SERIAL), + row(1, 1, 1)); + getCluster().coordinator(1).execute("UPDATE " + fullTableName + " SET v = 3 WHERE pk = 1 and ck = 1 IF v = 2", org.apache.cassandra.distributed.api.ConsistencyLevel.QUORUM); + assertRows(getCluster().coordinator(1).execute("SELECT * FROM " + fullTableName + " WHERE pk = 1", org.apache.cassandra.distributed.api.ConsistencyLevel.SERIAL), + row(1, 1, 1)); + getCluster().coordinator(1).execute("UPDATE " + fullTableName + " SET v = 2 WHERE pk = 1 and ck = 1 IF v = 1", org.apache.cassandra.distributed.api.ConsistencyLevel.QUORUM); + assertRows(getCluster().coordinator(1).execute("SELECT * FROM " + fullTableName + " WHERE pk = 1", org.apache.cassandra.distributed.api.ConsistencyLevel.SERIAL), + row(1, 1, 2)); + } + + @Test + public void incompletePrepare() throws Throwable + { + String tableName = tableName(); + String fullTableName = KEYSPACE + "." + tableName; + getCluster().schemaChange("CREATE TABLE " + fullTableName + " (pk int, ck int, v int, PRIMARY KEY (pk, ck))"); + + IMessageFilters.Filter drop = getCluster().filters().verbs(PAXOS2_PREPARE_REQ.id, PAXOS_PREPARE_REQ.id).from(1).to(2, 3).drop(); + try + { + getCluster().coordinator(1).execute("INSERT INTO " + fullTableName + " (pk, ck, v) VALUES (1, 1, 1) IF NOT EXISTS", org.apache.cassandra.distributed.api.ConsistencyLevel.QUORUM); + Assert.assertTrue(false); + } + catch (RuntimeException t) + { + if (!t.getClass().getName().equals(CasWriteTimeoutException.class.getName())) + throw new AssertionError(t); + } + drop.off(); + getCluster().coordinator(1).execute("UPDATE " + fullTableName + " SET v = 2 WHERE pk = 1 and ck = 1 IF v = 1", org.apache.cassandra.distributed.api.ConsistencyLevel.QUORUM); + assertRows(getCluster().coordinator(1).execute("SELECT * FROM " + fullTableName + " WHERE pk = 1", org.apache.cassandra.distributed.api.ConsistencyLevel.SERIAL)); + } + + @Test + public void incompletePropose() throws Throwable + { + String tableName = tableName(); + String fullTableName = KEYSPACE + "." + tableName; + getCluster().schemaChange("CREATE TABLE " + fullTableName + " (pk int, ck int, v int, PRIMARY KEY (pk, ck))"); + + IMessageFilters.Filter drop1 = getCluster().filters().verbs(PAXOS2_PROPOSE_REQ.id, PAXOS_PROPOSE_REQ.id).from(1).to(2, 3).drop(); + try + { + getCluster().coordinator(1).execute("INSERT INTO " + fullTableName + " (pk, ck, v) VALUES (1, 1, 1) IF NOT EXISTS", org.apache.cassandra.distributed.api.ConsistencyLevel.QUORUM); + Assert.assertTrue(false); + } + catch (RuntimeException t) + { + if (!t.getClass().getName().equals(CasWriteTimeoutException.class.getName())) + throw new AssertionError(t); + } + drop1.off(); + // make sure we encounter one of the in-progress proposals so we complete it + drop(getCluster(), 1, to(2), to(), to()); + getCluster().coordinator(1).execute("UPDATE " + fullTableName + " SET v = 2 WHERE pk = 1 and ck = 1 IF v = 1", org.apache.cassandra.distributed.api.ConsistencyLevel.QUORUM); + assertRows(getCluster().coordinator(1).execute("SELECT * FROM " + fullTableName + " WHERE pk = 1", org.apache.cassandra.distributed.api.ConsistencyLevel.SERIAL), + row(1, 1, 2)); + } + + @Test + public void incompleteCommit() throws Throwable + { + String tableName = tableName(); + String fullTableName = KEYSPACE + "." + tableName; + getCluster().schemaChange("CREATE TABLE " + fullTableName + " (pk int, ck int, v int, PRIMARY KEY (pk, ck))"); + + try (AutoCloseable drop = drop(getCluster(), 1, to(), to(), to(2, 3))) + { + getCluster().coordinator(1).execute("INSERT INTO " + fullTableName + " (pk, ck, v) VALUES (1, 1, 1) IF NOT EXISTS", org.apache.cassandra.distributed.api.ConsistencyLevel.QUORUM); + Assert.assertTrue(false); + } + catch (RuntimeException t) + { + if (!t.getClass().getName().equals(CasWriteTimeoutException.class.getName())) + throw new AssertionError(t); + } + + // make sure we see one of the successful commits + try (AutoCloseable drop = drop(getCluster(), 1, to(2), to(2), to())) + { + getCluster().coordinator(1).execute("UPDATE " + fullTableName + " SET v = 2 WHERE pk = 1 and ck = 1 IF v = 1", org.apache.cassandra.distributed.api.ConsistencyLevel.QUORUM); + assertRows(getCluster().coordinator(1).execute("SELECT * FROM " + fullTableName + " WHERE pk = 1", org.apache.cassandra.distributed.api.ConsistencyLevel.SERIAL), + row(1, 1, 2)); + } + } + + /** + * - Prepare A to {1, 2, 3} + * - Propose A to {1} + */ + @Test + public void testRepairIncompletePropose() throws Throwable + { + String tableName = tableName(); + String fullTableName = KEYSPACE + "." + tableName; + getCluster().schemaChange("CREATE TABLE " + fullTableName + " (pk int, ck int, v int, PRIMARY KEY (pk, ck))"); + + for (int repairWithout = 1 ; repairWithout <= 3 ; ++repairWithout) + { + try (AutoCloseable drop = drop(getCluster(), 1, to(), to(2, 3), to())) + { + getCluster().coordinator(1).execute("INSERT INTO " + fullTableName + " (pk, ck, v) VALUES (?, 1, 1) IF NOT EXISTS", org.apache.cassandra.distributed.api.ConsistencyLevel.QUORUM, repairWithout); + Assert.assertTrue(false); + } + catch (RuntimeException t) + { + if (!t.getClass().getName().equals(CasWriteTimeoutException.class.getName())) + throw new AssertionError(t); + } + int repairWith = repairWithout == 3 ? 2 : 3; + repair(getCluster(), tableName, repairWithout, repairWith, repairWithout); + + try (AutoCloseable drop = drop(getCluster(), repairWith, to(repairWithout), to(), to())) + { + Object[][] rows = getCluster().coordinator(1).execute("SELECT * FROM " + fullTableName + " WHERE pk = ?", org.apache.cassandra.distributed.api.ConsistencyLevel.QUORUM, repairWithout); + if (repairWithout == 1) assertRows(rows); // invalidated + else assertRows(rows, row(repairWithout, 1, 1)); // finished + } + } + } + + /** + * - Prepare A to {1, 2, 3} + * - Propose A to {1, 2} + * - Commit A to {1} + * - Repair using {2, 3} + */ + @Test + public void testRepairIncompleteCommit() throws Throwable + { + String tableName = tableName(); + String fullTableName = KEYSPACE + "." + tableName; + getCluster().schemaChange("CREATE TABLE " + fullTableName + " (pk int, ck int, v int, PRIMARY KEY (pk, ck))"); + + for (int repairWithout = 1 ; repairWithout <= 3 ; ++repairWithout) + { + try (AutoCloseable drop = drop(getCluster(), 1, to(), to(3), to(2, 3))) + { + getCluster().coordinator(1).execute("INSERT INTO " + fullTableName + " (pk, ck, v) VALUES (?, 1, 1) IF NOT EXISTS", org.apache.cassandra.distributed.api.ConsistencyLevel.QUORUM, repairWithout); + Assert.assertTrue(false); + } + catch (RuntimeException t) + { + if (!t.getClass().getName().equals(CasWriteTimeoutException.class.getName())) + throw new AssertionError(t); + } + + int repairWith = repairWithout == 3 ? 2 : 3; + repair(getCluster(), tableName, repairWithout, repairWith, repairWithout); + try (AutoCloseable drop = drop(getCluster(), repairWith, to(repairWithout), to(), to())) + { + //TODO dtest api is missing one with message? booo... removed "" + repairWithout, + assertRows(getCluster().coordinator(repairWith).execute("SELECT * FROM " + fullTableName + " WHERE pk = ?", org.apache.cassandra.distributed.api.ConsistencyLevel.QUORUM, repairWithout), + row(repairWithout, 1, 1)); + } + } + } +} diff --git a/test/distributed/org/apache/cassandra/distributed/test/CASContentionTest.java b/test/distributed/org/apache/cassandra/distributed/test/CASContentionTest.java new file mode 100644 index 0000000000..aafbc45277 --- /dev/null +++ b/test/distributed/org/apache/cassandra/distributed/test/CASContentionTest.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.distributed.test; + +import com.google.common.util.concurrent.Uninterruptibles; +import org.apache.cassandra.concurrent.Stage; +import org.apache.cassandra.distributed.Cluster; +import org.apache.cassandra.distributed.api.IInstanceConfig; +import org.apache.cassandra.service.paxos.ContentionStrategy; +import org.apache.cassandra.utils.FBUtilities; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; + +import static org.apache.cassandra.distributed.api.ConsistencyLevel.QUORUM; +import static org.apache.cassandra.net.Verb.PAXOS2_PREPARE_REQ; + +public class CASContentionTest extends CASTestBase +{ + private static Cluster THREE_NODES; + + @BeforeClass + public static void beforeClass() throws Throwable + { + System.setProperty("cassandra.paxos.use_self_execution", "false"); + TestBaseImpl.beforeClass(); + Consumer conf = config -> config + .set("paxos_variant", "v2") + .set("write_request_timeout_in_ms", 20000L) + .set("cas_contention_timeout_in_ms", 20000L) + .set("request_timeout_in_ms", 20000L); + THREE_NODES = init(Cluster.create(3, conf)); + } + + @AfterClass + public static void afterClass() + { + if (THREE_NODES != null) + THREE_NODES.close(); + } + + @Test + public void testDynamicContentionTracing() throws Throwable + { + try + { + + String tableName = tableName("tbl"); + THREE_NODES.schemaChange("CREATE TABLE " + KEYSPACE + '.' + tableName + " (pk int, v int, PRIMARY KEY (pk))"); + + CountDownLatch haveStarted = new CountDownLatch(1); + CountDownLatch haveInvalidated = new CountDownLatch(1); + THREE_NODES.verbs(PAXOS2_PREPARE_REQ).from(1).messagesMatching((from, to, verb) -> { + haveStarted.countDown(); + Uninterruptibles.awaitUninterruptibly(haveInvalidated); + return false; + }).drop(); + THREE_NODES.get(1).runOnInstance(() -> ContentionStrategy.setStrategy("trace=1")); + Future insert = THREE_NODES.get(1).async(() -> { + THREE_NODES.coordinator(1).execute("INSERT INTO " + KEYSPACE + '.' + tableName + " (pk, v) VALUES (1, 1) IF NOT EXISTS", QUORUM); + }).call(); + haveStarted.await(); + THREE_NODES.coordinator(2).execute("INSERT INTO " + KEYSPACE + '.' + tableName + " (pk, v) VALUES (1, 1) IF NOT EXISTS", QUORUM); + haveInvalidated.countDown(); + THREE_NODES.filters().reset(); + insert.get(); + Uninterruptibles.sleepUninterruptibly(1L, TimeUnit.SECONDS); + THREE_NODES.forEach(i -> i.runOnInstance(() -> FBUtilities.waitOnFuture(Stage.TRACING.submit(() -> {})))); + Object[][] result = THREE_NODES.coordinator(1).execute("SELECT parameters FROM system_traces.sessions", QUORUM); + Assert.assertEquals(1, result.length); + Assert.assertEquals(1, result[0].length); + Assert.assertTrue(Map.class.isAssignableFrom(result[0][0].getClass())); + Map params = (Map) result[0][0]; + Assert.assertEquals("SERIAL", params.get("consistency")); + Assert.assertEquals(tableName, params.get("table")); + Assert.assertEquals(KEYSPACE, params.get("keyspace")); + Assert.assertEquals("1", params.get("partitionKey")); + Assert.assertEquals("write", params.get("kind")); + } + finally + { + THREE_NODES.filters().reset(); + } + } + + +} diff --git a/test/distributed/org/apache/cassandra/distributed/test/CASMultiDCTest.java b/test/distributed/org/apache/cassandra/distributed/test/CASMultiDCTest.java new file mode 100644 index 0000000000..e36f34b837 --- /dev/null +++ b/test/distributed/org/apache/cassandra/distributed/test/CASMultiDCTest.java @@ -0,0 +1,150 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.distributed.test; + +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Consumer; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import org.apache.cassandra.cql3.QueryProcessor; +import org.apache.cassandra.cql3.UntypedResultSet; +import org.apache.cassandra.distributed.Cluster; +import org.apache.cassandra.distributed.api.ConsistencyLevel; +import org.apache.cassandra.distributed.api.Feature; +import org.apache.cassandra.distributed.api.IInstanceConfig; +import org.apache.cassandra.service.paxos.PaxosCommit; +import org.apache.cassandra.utils.ByteBufferUtil; + +import static org.apache.cassandra.distributed.api.ConsistencyLevel.*; + +public class CASMultiDCTest +{ + private static Cluster CLUSTER; + private static final String KEYSPACE = "ks"; + private static final String TABLE = "tbl"; + private static final String KS_TBL = KEYSPACE + '.' + TABLE; + + private static final AtomicInteger nextKey = new AtomicInteger(); + + @BeforeClass + public static void beforeClass() throws Throwable + { + TestBaseImpl.beforeClass(); + Consumer conf = config -> config + .with(Feature.NETWORK) + .set("paxos_variant", "v2") + .set("write_request_timeout_in_ms", 1000L) + .set("cas_contention_timeout_in_ms", 1000L) + .set("request_timeout_in_ms", 1000L); + + Cluster.Builder builder = new Cluster.Builder(); + builder.withNodes(4); + builder.withDCs(2); + builder.withConfig(conf); + CLUSTER = builder.start(); + CLUSTER.schemaChange("CREATE KEYSPACE " + KEYSPACE + " WITH replication = {'class': 'NetworkTopologyStrategy', 'datacenter1':2, 'datacenter2':2};"); + CLUSTER.schemaChange("CREATE TABLE " + KS_TBL + " (k int, c int, v int, primary key (k, v))"); + CLUSTER.forEach(i -> i.runOnInstance(() -> { + Assert.assertTrue(PaxosCommit.getEnableDcLocalCommit()); // should be enabled by default + })); + } + + @Before + public void setUp() + { + CLUSTER.forEach(i -> i.runOnInstance(() -> { + PaxosCommit.setEnableDcLocalCommit(true); + })); + } + + private static void testLocalSerialCommit(ConsistencyLevel serialCL, ConsistencyLevel commitCL, int key, boolean expectRemoteCommit) + { + for (int i=0; i { + UntypedResultSet result = QueryProcessor.executeInternal("SELECT * FROM system.paxos WHERE row_key=?", ByteBufferUtil.bytes(key)); + Assert.assertEquals(0, result.size()); + }); + } + + CLUSTER.coordinator(1).execute("INSERT INTO " + KS_TBL + " (k, c, v) VALUES (?, ?, ?) IF NOT EXISTS", serialCL, commitCL, key, key, key); + + int numCommitted = 0; + int numWritten = 0; + for (int i=0; i { + int numPaxosRows = QueryProcessor.executeInternal("SELECT * FROM system.paxos WHERE row_key=?", ByteBufferUtil.bytes(key)).size(); + Assert.assertTrue(numPaxosRows == 0 || numPaxosRows == 1); + if (!expectRemoteCommit) + Assert.assertEquals(expectPaxosRows ? 1 : 0, numPaxosRows); + int numTableRows = QueryProcessor.executeInternal("SELECT * FROM " + KS_TBL + " WHERE k=?", ByteBufferUtil.bytes(key)).size(); + Assert.assertTrue(numTableRows == 0 || numTableRows == 1); + return (numPaxosRows > 0 ? 1 : 0) | (numTableRows > 0 ? 2 : 0); + }); + if ((flags & 1) != 0) + numCommitted++; + if ((flags & 2) != 0) + numWritten++; + } + Assert.assertTrue(String.format("numWritten: %s < 3", numWritten), numWritten >= 3); + if (expectRemoteCommit) + Assert.assertTrue(String.format("numCommitted: %s < 3", numCommitted), numCommitted >= 3); + else + Assert.assertEquals(2, numCommitted); + } + + @Test + public void testLocalSerialLocalCommit() + { + testLocalSerialCommit(LOCAL_SERIAL, LOCAL_QUORUM, nextKey.getAndIncrement(), false); + } + + @Test + public void testLocalSerialQuorumCommit() + { + testLocalSerialCommit(LOCAL_SERIAL, QUORUM, nextKey.getAndIncrement(), false); + } + + @Test + public void testSerialLocalCommit() + { + testLocalSerialCommit(SERIAL, LOCAL_QUORUM, nextKey.getAndIncrement(), true); + } + + @Test + public void testSerialQuorumCommit() + { + testLocalSerialCommit(SERIAL, QUORUM, nextKey.getAndIncrement(), true); + } + + @Test + public void testDcLocalCommitDisabled() + { + CLUSTER.forEach(i -> i.runOnInstance(() -> { + PaxosCommit.setEnableDcLocalCommit(false); + })); + testLocalSerialCommit(LOCAL_SERIAL, QUORUM, nextKey.getAndIncrement(), true); + } +} diff --git a/test/distributed/org/apache/cassandra/distributed/test/CASTest.java b/test/distributed/org/apache/cassandra/distributed/test/CASTest.java index 4513a8ce6a..a7e99130e5 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/CASTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/CASTest.java @@ -20,33 +20,41 @@ package org.apache.cassandra.distributed.test; import java.io.IOException; import java.util.function.BiConsumer; +import java.util.function.Consumer; +import org.junit.AfterClass; import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; import org.junit.Ignore; import org.junit.Test; -import org.apache.cassandra.db.marshal.Int32Type; -import org.apache.cassandra.dht.Murmur3Partitioner; -import org.apache.cassandra.dht.Token; import org.apache.cassandra.distributed.Cluster; -import org.apache.cassandra.distributed.api.ConsistencyLevel; import org.apache.cassandra.distributed.api.ICoordinator; -import org.apache.cassandra.distributed.api.IInstance; +import org.apache.cassandra.distributed.api.IInstanceConfig; import org.apache.cassandra.distributed.api.IMessageFilters; -import org.apache.cassandra.distributed.impl.UnsafeGossipHelper; +import org.apache.cassandra.exceptions.CasWriteTimeoutException; +import static org.apache.cassandra.distributed.api.ConsistencyLevel.ANY; +import static org.apache.cassandra.distributed.api.ConsistencyLevel.ONE; +import static org.apache.cassandra.distributed.api.ConsistencyLevel.QUORUM; +import static org.apache.cassandra.distributed.api.ConsistencyLevel.SERIAL; import static org.apache.cassandra.distributed.shared.AssertUtils.assertRows; -import static org.apache.cassandra.distributed.shared.AssertUtils.fail; import static org.apache.cassandra.distributed.shared.AssertUtils.row; +import static org.apache.cassandra.net.Verb.PAXOS2_COMMIT_AND_PREPARE_REQ; +import static org.apache.cassandra.net.Verb.PAXOS2_PREPARE_REFRESH_REQ; +import static org.apache.cassandra.net.Verb.PAXOS2_PREPARE_REQ; +import static org.apache.cassandra.net.Verb.PAXOS2_PROPOSE_REQ; import static org.apache.cassandra.net.Verb.PAXOS_COMMIT_REQ; import static org.apache.cassandra.net.Verb.PAXOS_PREPARE_REQ; import static org.apache.cassandra.net.Verb.PAXOS_PROPOSE_REQ; import static org.apache.cassandra.net.Verb.READ_REQ; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; -public class CASTest extends TestBaseImpl +public class CASTest extends CASCommonTestCases { /** * The {@code cas_contention_timeout} used during the tests @@ -58,212 +66,128 @@ public class CASTest extends TestBaseImpl */ private static final String REQUEST_TIMEOUT = "1000ms"; - @Test - public void simpleUpdate() throws Throwable - { - try (Cluster cluster = init(Cluster.create(3))) - { - cluster.schemaChange("CREATE TABLE " + KEYSPACE + ".tbl (pk int, ck int, v int, PRIMARY KEY (pk, ck))"); + private static Cluster THREE_NODES; + private static Cluster FOUR_NODES; - cluster.coordinator(1).execute("INSERT INTO " + KEYSPACE + ".tbl (pk, ck, v) VALUES (1, 1, 1) IF NOT EXISTS", ConsistencyLevel.QUORUM); - assertRows(cluster.coordinator(1).execute("SELECT * FROM " + KEYSPACE + ".tbl WHERE pk = 1", ConsistencyLevel.SERIAL), - row(1, 1, 1)); - cluster.coordinator(1).execute("UPDATE " + KEYSPACE + ".tbl SET v = 3 WHERE pk = 1 and ck = 1 IF v = 2", ConsistencyLevel.QUORUM); - assertRows(cluster.coordinator(1).execute("SELECT * FROM " + KEYSPACE + ".tbl WHERE pk = 1", ConsistencyLevel.SERIAL), - row(1, 1, 1)); - cluster.coordinator(1).execute("UPDATE " + KEYSPACE + ".tbl SET v = 2 WHERE pk = 1 and ck = 1 IF v = 1", ConsistencyLevel.QUORUM); - assertRows(cluster.coordinator(1).execute("SELECT * FROM " + KEYSPACE + ".tbl WHERE pk = 1", ConsistencyLevel.SERIAL), - row(1, 1, 2)); - } + @BeforeClass + public static void beforeClass() throws Throwable + { + System.setProperty("cassandra.paxos.use_self_execution", "false"); + TestBaseImpl.beforeClass(); + Consumer conf = config -> config + .set("paxos_variant", "v2") + .set("write_request_timeout", REQUEST_TIMEOUT) + .set("cas_contention_timeout", CONTENTION_TIMEOUT) + .set("request_timeout", REQUEST_TIMEOUT); + THREE_NODES = init(Cluster.create(3, conf)); + FOUR_NODES = init(Cluster.create(4, conf), 3); } - @Test - public void incompletePrepare() throws Throwable + @AfterClass + public static void afterClass() { - try (Cluster cluster = init(Cluster.create(3, config -> config.set("write_request_timeout", REQUEST_TIMEOUT) - .set("cas_contention_timeout", CONTENTION_TIMEOUT)))) - { - cluster.schemaChange("CREATE TABLE " + KEYSPACE + ".tbl (pk int, ck int, v int, PRIMARY KEY (pk, ck))"); - - IMessageFilters.Filter drop = cluster.filters().verbs(PAXOS_PREPARE_REQ.id).from(1).to(2, 3).drop(); - try - { - cluster.coordinator(1).execute("INSERT INTO " + KEYSPACE + ".tbl (pk, ck, v) VALUES (1, 1, 1) IF NOT EXISTS", ConsistencyLevel.QUORUM); - Assert.fail(); - } - catch (RuntimeException e) - { - Assert.assertEquals("CAS operation timed out - encountered contentions: 0", e.getMessage()); - } - drop.off(); - cluster.coordinator(1).execute("UPDATE " + KEYSPACE + ".tbl SET v = 2 WHERE pk = 1 and ck = 1 IF v = 1", ConsistencyLevel.QUORUM); - assertRows(cluster.coordinator(1).execute("SELECT * FROM " + KEYSPACE + ".tbl WHERE pk = 1", ConsistencyLevel.SERIAL)); - } + if (THREE_NODES != null) + THREE_NODES.close(); + if (FOUR_NODES != null) + FOUR_NODES.close(); } - @Test - public void incompletePropose() throws Throwable + @Before + public void before() { - try (Cluster cluster = init(Cluster.create(3, config -> config.set("write_request_timeout", REQUEST_TIMEOUT) - .set("cas_contention_timeout", CONTENTION_TIMEOUT)))) + THREE_NODES.filters().reset(); + FOUR_NODES.filters().reset(); + // tests add/remove nodes from the ring, so attempt to add them back + for (int i = 1 ; i <= 4 ; ++i) { - cluster.schemaChange("CREATE TABLE " + KEYSPACE + ".tbl (pk int, ck int, v int, PRIMARY KEY (pk, ck))"); - - IMessageFilters.Filter drop1 = cluster.filters().verbs(PAXOS_PROPOSE_REQ.id).from(1).to(2, 3).drop(); - try + for (int j = 1; j <= 4; j++) { - cluster.coordinator(1).execute("INSERT INTO " + KEYSPACE + ".tbl (pk, ck, v) VALUES (1, 1, 1) IF NOT EXISTS", ConsistencyLevel.QUORUM); - Assert.fail(); + FOUR_NODES.get(i).acceptsOnInstance(CASTestBase::addToRingNormal).accept(FOUR_NODES.get(j)); } - catch (RuntimeException e) - { - Assert.assertEquals("CAS operation timed out - encountered contentions: 0", e.getMessage()); - } - drop1.off(); - // make sure we encounter one of the in-progress proposals so we complete it - cluster.filters().verbs(PAXOS_PREPARE_REQ.id).from(1).to(2).drop(); - cluster.coordinator(1).execute("UPDATE " + KEYSPACE + ".tbl SET v = 2 WHERE pk = 1 and ck = 1 IF v = 1", ConsistencyLevel.QUORUM); - assertRows(cluster.coordinator(1).execute("SELECT * FROM " + KEYSPACE + ".tbl WHERE pk = 1", ConsistencyLevel.SERIAL), - row(1, 1, 2)); } } - @Test - public void incompleteCommit() throws Throwable - { - try (Cluster cluster = init(Cluster.create(3, config -> config.set("write_request_timeout", REQUEST_TIMEOUT) - .set("cas_contention_timeout", CONTENTION_TIMEOUT)))) - { - cluster.schemaChange("CREATE TABLE " + KEYSPACE + ".tbl (pk int, ck int, v int, PRIMARY KEY (pk, ck))"); - - IMessageFilters.Filter drop1 = cluster.filters().verbs(PAXOS_COMMIT_REQ.id).from(1).to(2, 3).drop(); - try - { - cluster.coordinator(1).execute("INSERT INTO " + KEYSPACE + ".tbl (pk, ck, v) VALUES (1, 1, 1) IF NOT EXISTS", ConsistencyLevel.QUORUM); - Assert.fail(); - } - catch (RuntimeException e) - { - Assert.assertEquals("CAS operation timed out - encountered contentions: 0", e.getMessage()); - } - drop1.off(); - // make sure we see one of the successful commits - cluster.filters().verbs(PAXOS_PROPOSE_REQ.id).from(1).to(2).drop(); - cluster.coordinator(1).execute("UPDATE " + KEYSPACE + ".tbl SET v = 2 WHERE pk = 1 and ck = 1 IF v = 1", ConsistencyLevel.QUORUM); - assertRows(cluster.coordinator(1).execute("SELECT * FROM " + KEYSPACE + ".tbl WHERE pk = 1", ConsistencyLevel.SERIAL), - row(1, 1, 2)); - } - } - - private int[] paxosAndReadVerbs() { - return new int[] { PAXOS_PREPARE_REQ.id, PAXOS_PROPOSE_REQ.id, PAXOS_COMMIT_REQ.id, READ_REQ.id }; - } - /** - * Base test to ensure that if a write times out but with a proposal accepted by some nodes (less then quorum), and - * a following SERIAL operation does not observe that write (the node having accepted it do not participate in that - * following operation), then that write is never applied, even when the nodes having accepted the original proposal - * participate. + * A write and a read that are able to witness different (i.e. non-linearizable) histories + * See CASSANDRA-12126 * - *

    In other words, if an operation timeout, it may or may not be applied, but that "fate" is persistently decided - * by the very SERIAL operation that "succeed" (in the sense of 'not timing out or throwing some other exception'). - * - * @param postTimeoutOperation1 a SERIAL operation executed after an initial write that inserts the row [0, 0] times - * out. It is executed with a QUORUM of nodes that have _not_ see the timed out - * proposal, and so that operation should expect that the [0, 0] write has not taken - * place. - * @param postTimeoutOperation2 a 2nd SERIAL operation executed _after_ {@code postTimeoutOperation1}, with no - * write executed between the 2 operation. Contrarily to the 1st operation, the QORUM - * for this operation _will_ include the node that got the proposal for the [0, 0] - * insert but didn't participated to {@code postTimeoutOperation1}}. That operation - * should also no witness that [0, 0] write (since {@code postTimeoutOperation1} - * didn't). - * @param loseCommitOfOperation1 if {@code true}, the test will also drop the "commits" messages for - * {@code postTimeoutOperation1}. In general, the test should behave the same with or - * without that flag since a value is decided as soon as it has been "accepted by - * quorum" and the commits should always be properly replayed. + * - A Promised by {1, 2, 3} + * - A Acccepted by {1} + * - B (=>!A) Promised and Proposed to {2, 3} + * - Read from (or attempt C (=>!B)) to {1, 2} -> witness either A or B, not both */ - private void consistencyAfterWriteTimeoutTest(BiConsumer postTimeoutOperation1, - BiConsumer postTimeoutOperation2, - boolean loseCommitOfOperation1) throws IOException + @Test + public void testIncompleteWriteSupersededByConflictingRejectedCondition() throws Throwable { - // It's unclear why (haven't dug), but in some of the instance of this test method, there is a consistent 2+ - // seconds pauses between the prepare and propose phases during the execution of 'postTimeoutOperation2'. This - // does not happen on 3.0 and there is no report of such long pauses otherwise, so an hypothesis is that this - // is due to the in-jvm dtest framework. This is is why we use a 4 seconds timeout here. Given this test is - // not about performance, this is probably ok, even if we ideally should dug into the underlying reason. - try (Cluster cluster = init(Cluster.create(3, config -> config.set("write_request_timeout", "4000ms") - .set("cas_contention_timeout", CONTENTION_TIMEOUT)))) + String tableName = tableName("tbl"); + THREE_NODES.schemaChange("CREATE TABLE " + KEYSPACE + "." + tableName + " (pk int, ck int, v int, PRIMARY KEY (pk, ck))"); + + IMessageFilters.Filter drop1 = THREE_NODES.filters().verbs(PAXOS2_PROPOSE_REQ.id, PAXOS_PROPOSE_REQ.id).from(1).to(2, 3).drop(); + try { - String table = KEYSPACE + ".t"; - cluster.schemaChange("CREATE TABLE " + table + " (k int PRIMARY KEY, v int)"); - - // We do a CAS insertion, but have with the PROPOSE message dropped on node 1 and 2. The CAS will not get - // through and should timeout. Importantly, node 3 does receive and answer the PROPOSE. - IMessageFilters.Filter dropProposeFilter = cluster.filters() - .inbound() - .verbs(PAXOS_PROPOSE_REQ.id) - .from(3) - .to(1, 2) - .drop(); - try - { - // NOTE: the consistency below is the "commit" one, so it doesn't matter at all here. - // NOTE 2: we use node 3 as coordinator because message filters don't currently work for locally - // delivered messages and as we want to drop messages to 1 and 2, we can't use them. - cluster.coordinator(3) - .execute("INSERT INTO " + table + "(k, v) VALUES (0, 0) IF NOT EXISTS", ConsistencyLevel.ONE); - fail("The insertion should have timed-out"); - } - catch (Exception e) - { - // We expect a write timeout. If we get one, the test can continue, otherwise, we rethrow. Note that we - // look at the root cause because the dtest framework effectively wrap the exception in a RuntimeException - // (we could just look at the immediate cause, but this feel a bit more resilient this way). - // TODO: we can't use an instanceof below because the WriteTimeoutException we get is from a different class - // loader than the one the test run under, and that's our poor-man work-around. This kind of things should - // be improved at the dtest API level. - if (!e.getClass().getSimpleName().equals("CasWriteTimeoutException")) - throw e; - } - finally - { - dropProposeFilter.off(); - } - - // Isolates node 3 and executes the SERIAL operation. As neither node 1 or 2 got the initial insert proposal, - // there is nothing to "replay" and the operation should assert the table is still empty. - IMessageFilters.Filter ignoreNode3Filter = cluster.filters().verbs(paxosAndReadVerbs()).to(3).drop(); - IMessageFilters.Filter dropCommitFilter = null; - if (loseCommitOfOperation1) - { - dropCommitFilter = cluster.filters().verbs(PAXOS_COMMIT_REQ.id).to(1, 2).drop(); - } - try - { - postTimeoutOperation1.accept(table, cluster.coordinator(1)); - } - finally - { - ignoreNode3Filter.off(); - if (dropCommitFilter != null) - dropCommitFilter.off(); - } - - // Node 3 is now back and we isolate node 2 to ensure the next read hits node 1 and 3. - // What we want to ensure is that despite node 3 having the initial insert in its paxos state in a position of - // being replayed, that insert is _not_ replayed (it would contradict serializability since the previous - // operation asserted nothing was inserted). It is this execution that failed before CASSANDRA-12126. - IMessageFilters.Filter ignoreNode2Filter = cluster.filters().verbs(paxosAndReadVerbs()).to(2).drop(); - try - { - postTimeoutOperation2.accept(table, cluster.coordinator(1)); - } - finally - { - ignoreNode2Filter.off(); - } + THREE_NODES.coordinator(1).execute("INSERT INTO " + KEYSPACE + "." + tableName + " (pk, ck, v) VALUES (1, 1, 1) IF NOT EXISTS", QUORUM); + fail(); } + catch (Throwable t) + { + if (!t.getClass().getName().equals(CasWriteTimeoutException.class.getName())) + throw t; + } + drop(THREE_NODES, 2, to(1), to(1), to()); + assertRows(THREE_NODES.coordinator(2).execute("UPDATE " + KEYSPACE + "." + tableName + " SET v = 2 WHERE pk = 1 and ck = 1 IF v = 1", QUORUM), + row(false)); + drop1.off(); + drop(THREE_NODES, 1, to(2), to(), to()); + assertRows(THREE_NODES.coordinator(1).execute("SELECT * FROM " + KEYSPACE + "." + tableName + " WHERE pk = 1", SERIAL)); + } + + /** + * Two reads that are able to witness different (i.e. non-linearizable) histories + * - A Promised by {1, 2, 3} + * - A Accepted by {1} + * - Read from {2, 3} -> do not witness A? + * - Read from {1, 2} -> witnesses A? + * See CASSANDRA-12126 + */ + @Ignore + @Test + public void testIncompleteWriteSupersededByRead() throws Throwable + { + String tableName = tableName(); + String fullTableName = KEYSPACE + "." + tableName; + THREE_NODES.schemaChange("CREATE TABLE " + fullTableName + " (pk int, ck int, v int, PRIMARY KEY (pk, ck))"); + + IMessageFilters.Filter drop1 = THREE_NODES.filters().verbs(PAXOS2_PROPOSE_REQ.id, PAXOS_PROPOSE_REQ.id).from(1).to(2, 3).drop(); + try + { + THREE_NODES.coordinator(1).execute("INSERT INTO " + fullTableName + " (pk, ck, v) VALUES (1, 1, 1) IF NOT EXISTS", QUORUM); + fail(); + } + catch (Throwable t) + { + if (!t.getClass().getName().equals(CasWriteTimeoutException.class.getName())) + throw t; + } + drop(THREE_NODES, 2, to(1), to(), to()); + assertRows(THREE_NODES.coordinator(2).execute("SELECT * FROM " + fullTableName + " WHERE pk = 1", SERIAL)); + drop1.off(); + + drop(THREE_NODES, 1, to(2), to(), to()); + assertRows(THREE_NODES.coordinator(1).execute("SELECT * FROM " + fullTableName + " WHERE pk = 1", SERIAL)); + } + + private static int[] paxosAndReadVerbs() + { + return new int[] { + PAXOS_PREPARE_REQ.id, + PAXOS2_PREPARE_REQ.id, + PAXOS2_PREPARE_REFRESH_REQ.id, + PAXOS2_COMMIT_AND_PREPARE_REQ.id, + PAXOS_PROPOSE_REQ.id, + PAXOS2_PROPOSE_REQ.id, + PAXOS_COMMIT_REQ.id, + READ_REQ.id + }; } /** @@ -276,11 +200,73 @@ public class CASTest extends TestBaseImpl public void readConsistencyAfterWriteTimeoutTest() throws IOException { BiConsumer operation = - (table, coordinator) -> assertRows(coordinator.execute("SELECT * FROM " + table + " WHERE k=0", - ConsistencyLevel.SERIAL)); + (table, coordinator) -> assertRows(coordinator.execute("SELECT * FROM " + table + " WHERE k=0", + SERIAL)); - consistencyAfterWriteTimeoutTest(operation, operation, false); - consistencyAfterWriteTimeoutTest(operation, operation, true); + consistencyAfterWriteTimeoutTest(operation, operation, false, THREE_NODES); + consistencyAfterWriteTimeoutTest(operation, operation, true, THREE_NODES); + } + + /** + * Tests that a sequence of reads exploit the fast read optimisation, as does a fail write, but that + * a read after a failed write that does not propose successfully does not + */ + @Test + public void fastReadsAndFailedWrites() throws IOException + { + String tableName = tableName("t"); + String table = KEYSPACE + "." + tableName; + THREE_NODES.schemaChange("CREATE TABLE " + table + " (k int PRIMARY KEY, v int)"); + + // We do a CAS insertion, but have with the PROPOSE message dropped on node 1 and 2. The CAS will not get + // through and should timeout. Importantly, node 3 does receive and answer the PROPOSE. + IMessageFilters.Filter dropProposeFilter = THREE_NODES.filters() + .verbs(PAXOS_PROPOSE_REQ.id, PAXOS2_PROPOSE_REQ.id, + PAXOS_COMMIT_REQ.id, PAXOS2_COMMIT_AND_PREPARE_REQ.id) + .to(1, 2) + .drop(); + + try + { + // shouldn't timeout + THREE_NODES.coordinator(1).execute("SELECT * FROM " + table + " WHERE k = 1", SERIAL); + THREE_NODES.coordinator(1).execute("SELECT * FROM " + table + " WHERE k = 1", SERIAL); + THREE_NODES.coordinator(1).execute("SELECT * FROM " + table + " WHERE k = 1", SERIAL); + THREE_NODES.coordinator(1).execute("UPDATE " + table + " SET v = 1 WHERE k = 1 IF EXISTS", ANY); + try + { + THREE_NODES.coordinator(1).execute("SELECT * FROM " + table + " WHERE k = 1", SERIAL); + Assert.fail(); + } + catch (AssertionError propagate) + { + throw propagate; + } + catch (Throwable maybeIgnore) + { + if (!maybeIgnore.getClass().getSimpleName().equals("ReadTimeoutException")) + throw maybeIgnore; + } + } + finally + { + dropProposeFilter.off(); + } + + THREE_NODES.coordinator(1).execute("SELECT * FROM " + table + " WHERE k = 1", SERIAL); + + try + { + dropProposeFilter.on(); + // shouldn't timeout + THREE_NODES.coordinator(1).execute("SELECT * FROM " + table + " WHERE k = 1", SERIAL); + THREE_NODES.coordinator(1).execute("SELECT * FROM " + table + " WHERE k = 1", SERIAL); + THREE_NODES.coordinator(1).execute("SELECT * FROM " + table + " WHERE k = 1", SERIAL); + } + finally + { + dropProposeFilter.off(); + } } /** @@ -296,10 +282,10 @@ public class CASTest extends TestBaseImpl // Note: we use CL.ANY so that the operation don't timeout in the case where we "lost" the operation1 commits. // The commit CL shouldn't have impact on this test anyway, so this doesn't diminishes the test. BiConsumer operation = - (table, coordinator) -> assertCasNotApplied(coordinator.execute("UPDATE " + table + " SET v = 1 WHERE k = 0 IF v = 0", - ConsistencyLevel.ANY)); - consistencyAfterWriteTimeoutTest(operation, operation, false); - consistencyAfterWriteTimeoutTest(operation, operation, true); + (table, coordinator) -> assertCasNotApplied(coordinator.execute("UPDATE " + table + " SET v = 1 WHERE k = 0 IF v = 0", + ANY)); + consistencyAfterWriteTimeoutTest(operation, operation, false, THREE_NODES); + consistencyAfterWriteTimeoutTest(operation, operation, true, THREE_NODES); } /** @@ -312,13 +298,13 @@ public class CASTest extends TestBaseImpl public void mixedReadAndNonApplyingCasConsistencyAfterWriteTimeout() throws IOException { BiConsumer operation1 = - (table, coordinator) -> assertRows(coordinator.execute("SELECT * FROM " + table + " WHERE k=0", - ConsistencyLevel.SERIAL)); + (table, coordinator) -> assertRows(coordinator.execute("SELECT * FROM " + table + " WHERE k=0", + SERIAL)); BiConsumer operation2 = - (table, coordinator) -> assertCasNotApplied(coordinator.execute("UPDATE " + table + " SET v = 1 WHERE k = 0 IF v = 0", - ConsistencyLevel.QUORUM)); - consistencyAfterWriteTimeoutTest(operation1, operation2, false); - consistencyAfterWriteTimeoutTest(operation1, operation2, true); + (table, coordinator) -> assertCasNotApplied(coordinator.execute("UPDATE " + table + " SET v = 1 WHERE k = 0 IF v = 0", + QUORUM)); + consistencyAfterWriteTimeoutTest(operation1, operation2, false, THREE_NODES); + consistencyAfterWriteTimeoutTest(operation1, operation2, true, THREE_NODES); } /** @@ -334,104 +320,83 @@ public class CASTest extends TestBaseImpl // Note: we use CL.ANY so that the operation don't timeout in the case where we "lost" the operation1 commits. // The commit CL shouldn't have impact on this test anyway, so this doesn't diminishes the test. BiConsumer operation1 = - (table, coordinator) -> assertCasNotApplied(coordinator.execute("UPDATE " + table + " SET v = 1 WHERE k = 0 IF v = 0", - ConsistencyLevel.ANY)); + (table, coordinator) -> assertCasNotApplied(coordinator.execute("UPDATE " + table + " SET v = 1 WHERE k = 0 IF v = 0", + ANY)); BiConsumer operation2 = - (table, coordinator) -> assertRows(coordinator.execute("SELECT * FROM " + table + " WHERE k=0", - ConsistencyLevel.SERIAL)); - consistencyAfterWriteTimeoutTest(operation1, operation2, false); - consistencyAfterWriteTimeoutTest(operation1, operation2, true); + (table, coordinator) -> assertRows(coordinator.execute("SELECT * FROM " + table + " WHERE k=0", + SERIAL)); + consistencyAfterWriteTimeoutTest(operation1, operation2, false, THREE_NODES); + consistencyAfterWriteTimeoutTest(operation1, operation2, true, THREE_NODES); } // TODO: this shoud probably be moved into the dtest API. private void assertCasNotApplied(Object[][] resultSet) { assertFalse("Expected a CAS resultSet (with at least application result) but got an empty one.", - resultSet.length == 0); + resultSet.length == 0); assertFalse("Invalid empty first row in CAS resultSet.", resultSet[0].length == 0); Object wasApplied = resultSet[0][0]; assertTrue("Expected 1st column of CAS resultSet to be a boolean, but got a " + wasApplied.getClass(), - wasApplied instanceof Boolean); + wasApplied instanceof Boolean); assertFalse("Expected CAS to not be applied, but was applied.", (Boolean)wasApplied); } - /** - * Failed write (by node that did not yet witness a range movement via gossip) is witnessed later as successful - * conflicting with another successful write performed by a node that did witness the range movement - * Prepare, Propose and Commit A to {1, 2} - * Range moves to {2, 3, 4} - * Prepare and Propose B (=> !A) to {3, 4} - */ - @Ignore + // failed write (by node that did not yet witness a range movement via gossip) is witnessed later as successful + // conflicting with another successful write performed by a node that did witness the range movement + // A Promised, Accepted and Committed by {1, 2} + // Range moves to {2, 3, 4} + // B (=> !A) Promised and Proposed to {3, 4} @Test public void testSuccessfulWriteBeforeRangeMovement() throws Throwable { - try (Cluster cluster = Cluster.create(4, config -> config - .set("write_request_timeout", REQUEST_TIMEOUT) - .set("cas_contention_timeout", CONTENTION_TIMEOUT))) - { - cluster.schemaChange("CREATE KEYSPACE " + KEYSPACE + " WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 3};"); - cluster.schemaChange("CREATE TABLE " + KEYSPACE + ".tbl (pk int, ck int, v1 int, v2 int, PRIMARY KEY (pk, ck))"); + String tableName = tableName("tbl"); + FOUR_NODES.schemaChange("CREATE TABLE " + KEYSPACE + "." + tableName + " (pk int, ck int, v1 int, v2 int, PRIMARY KEY (pk, ck))"); - // make it so {1} is unaware (yet) that {4} is an owner of the token - cluster.get(1).acceptsOnInstance(UnsafeGossipHelper::removeFromRing).accept(cluster.get(4)); + // make it so {1} is unaware (yet) that {4} is an owner of the token + FOUR_NODES.get(1).acceptsOnInstance(CASTestBase::removeFromRing).accept(FOUR_NODES.get(4)); - int pk = pk(cluster, 1, 2); + int pk = pk(FOUR_NODES, 1, 2); - // {1} promises and accepts on !{3} => {1, 2}; commits on !{2,3} => {1} - cluster.filters().verbs(PAXOS_PREPARE_REQ.id, READ_REQ.id).from(1).to(3).drop(); - cluster.filters().verbs(PAXOS_PROPOSE_REQ.id).from(1).to(3).drop(); - cluster.filters().verbs(PAXOS_COMMIT_REQ.id).from(1).to(2, 3).drop(); - assertRows(cluster.coordinator(1).execute("INSERT INTO " + KEYSPACE + ".tbl (pk, ck, v1) VALUES (?, 1, 1) IF NOT EXISTS", ConsistencyLevel.ONE, pk), - row(true)); + // {1} promises and accepts on !{3} => {1, 2}; commits on !{2,3} => {1} + drop(FOUR_NODES, 1, to(3), to(3), to(2, 3)); + assertRows(FOUR_NODES.coordinator(1).execute("INSERT INTO " + KEYSPACE + "." + tableName + " (pk, ck, v1) VALUES (?, 1, 1) IF NOT EXISTS", ONE, pk), + row(true)); - for (int i = 1 ; i <= 3 ; ++i) - cluster.get(i).acceptsOnInstance(UnsafeGossipHelper::addToRingNormal).accept(cluster.get(4)); + for (int i = 1; i <= 3; ++i) + FOUR_NODES.get(i).acceptsOnInstance(CASTestBase::addToRingNormal).accept(FOUR_NODES.get(4)); - // {4} reads from !{2} => {3, 4} - cluster.filters().verbs(PAXOS_PREPARE_REQ.id, READ_REQ.id).from(4).to(2).drop(); - cluster.filters().verbs(PAXOS_PROPOSE_REQ.id).from(4).to(2).drop(); - assertRows(cluster.coordinator(4).execute("INSERT INTO " + KEYSPACE + ".tbl (pk, ck, v2) VALUES (?, 1, 2) IF NOT EXISTS", ConsistencyLevel.ONE, pk), - row(false, pk, 1, 1, null)); - } + // {4} reads from !{2} => {3, 4} + drop(FOUR_NODES, 4, to(2), to(2), to()); + assertRows(FOUR_NODES.coordinator(4).execute("INSERT INTO " + KEYSPACE + "." + tableName + " (pk, ck, v2) VALUES (?, 1, 2) IF NOT EXISTS", ONE, pk), + row(false, pk, 1, 1, null)); } /** * Failed write (by node that did not yet witness a range movement via gossip) is witnessed later as successful * conflicting with another successful write performed by a node that did witness the range movement * - Range moves from {1, 2, 3} to {2, 3, 4}, witnessed by X (not by !X) - * - X: Prepare, Propose and Commit A to {3, 4} - * - !X: Prepare and Propose B (=>!A) to {1, 2} + * - X: A Promised, Accepted and Committed by {3, 4} + * - !X: B (=>!A) Promised and Proposed to {1, 2} */ - @Ignore @Test public void testConflictingWritesWithStaleRingInformation() throws Throwable { - try (Cluster cluster = Cluster.create(4, config -> config - .set("write_request_timeout", REQUEST_TIMEOUT) - .set("cas_contention_timeout", CONTENTION_TIMEOUT))) - { - cluster.schemaChange("CREATE KEYSPACE " + KEYSPACE + " WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 3};"); - cluster.schemaChange("CREATE TABLE " + KEYSPACE + ".tbl (pk int, ck int, v1 int, v2 int, PRIMARY KEY (pk, ck))"); + String tableName = tableName("tbl"); + FOUR_NODES.schemaChange("CREATE TABLE " + KEYSPACE + "." + tableName + " (pk int, ck int, v1 int, v2 int, PRIMARY KEY (pk, ck))"); - // make it so {1} is unaware (yet) that {4} is an owner of the token - cluster.get(1).acceptsOnInstance(UnsafeGossipHelper::removeFromRing).accept(cluster.get(4)); + // make it so {1} is unaware (yet) that {4} is an owner of the token + FOUR_NODES.get(1).acceptOnInstance(CASTestBase::removeFromRing, FOUR_NODES.get(4)); - // {4} promises, accepts and commits on !{2} => {3, 4} - int pk = pk(cluster, 1, 2); - cluster.filters().verbs(PAXOS_PREPARE_REQ.id, READ_REQ.id).from(4).to(2).drop(); - cluster.filters().verbs(PAXOS_PROPOSE_REQ.id).from(4).to(2).drop(); - cluster.filters().verbs(PAXOS_COMMIT_REQ.id).from(4).to(2).drop(); - assertRows(cluster.coordinator(4).execute("INSERT INTO " + KEYSPACE + ".tbl (pk, ck, v1) VALUES (?, 1, 1) IF NOT EXISTS", ConsistencyLevel.ONE, pk), - row(true)); + // {4} promises, accepts and commits on !{2} => {3, 4} + int pk = pk(FOUR_NODES, 1, 2); + drop(FOUR_NODES, 4, to(2), to(2), to(2)); + assertRows(FOUR_NODES.coordinator(4).execute("INSERT INTO " + KEYSPACE + "." + tableName + " (pk, ck, v1) VALUES (?, 1, 1) IF NOT EXISTS", ONE, pk), + row(true)); - // {1} promises, accepts and commmits on !{3} => {1, 2} - cluster.filters().verbs(PAXOS_PREPARE_REQ.id, READ_REQ.id).from(1).to(3).drop(); - cluster.filters().verbs(PAXOS_PROPOSE_REQ.id).from(1).to(3).drop(); - cluster.filters().verbs(PAXOS_COMMIT_REQ.id).from(1).to(3).drop(); - assertRows(cluster.coordinator(1).execute("INSERT INTO " + KEYSPACE + ".tbl (pk, ck, v2) VALUES (?, 1, 2) IF NOT EXISTS", ConsistencyLevel.ONE, pk), - row(false, pk, 1, 1, null)); - } + // {1} promises, accepts and commmits on !{3} => {1, 2} + drop(FOUR_NODES, 1, to(3), to(3), to(3)); + assertRows(FOUR_NODES.coordinator(1).execute("INSERT INTO " + KEYSPACE + "." + tableName + " (pk, ck, v2) VALUES (?, 1, 2) IF NOT EXISTS", ONE, pk), + row(false, pk, 1, 1, null)); } /** @@ -439,90 +404,75 @@ public class CASTest extends TestBaseImpl * Very similar to {@link #testConflictingWritesWithStaleRingInformation}. * * - Range moves from {1, 2, 3} to {2, 3, 4}; witnessed by X (not by !X) - * - !X: Prepare and Propose to {1, 2} + * - !X: Promised and Accepted by {1, 2} * - Range movement witnessed by !X - * - Any: Prepare and Read from {3, 4} + * - Any: Promised and Read from {3, 4} */ - @Ignore @Test public void testSucccessfulWriteDuringRangeMovementFollowedByRead() throws Throwable { - try (Cluster cluster = Cluster.create(4, config -> config - .set("write_request_timeout", REQUEST_TIMEOUT) - .set("cas_contention_timeout", CONTENTION_TIMEOUT))) - { - cluster.schemaChange("CREATE KEYSPACE " + KEYSPACE + " WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 3};"); - cluster.schemaChange("CREATE TABLE " + KEYSPACE + ".tbl (pk int, ck int, v int, PRIMARY KEY (pk, ck))"); + String tableName = tableName("tbl"); + FOUR_NODES.schemaChange("CREATE TABLE " + KEYSPACE + "." + tableName + " (pk int, ck int, v int, PRIMARY KEY (pk, ck))"); - // make it so {4} is bootstrapping, and this has not propagated to other nodes yet - for (int i = 1 ; i <= 4 ; ++i) - cluster.get(1).acceptsOnInstance(UnsafeGossipHelper::removeFromRing).accept(cluster.get(4)); - cluster.get(4).acceptsOnInstance(UnsafeGossipHelper::addToRingBootstrapping).accept(cluster.get(4)); + // make it so {4} is bootstrapping, and this has propagated to only a quorum of other nodes + for (int i = 1 ; i <= 4 ; ++i) + FOUR_NODES.get(i).acceptsOnInstance(CASTestBase::removeFromRing).accept(FOUR_NODES.get(4)); + for (int i = 2 ; i <= 4 ; ++i) + FOUR_NODES.get(i).acceptsOnInstance(CASTestBase::addToRingBootstrapping).accept(FOUR_NODES.get(4)); - int pk = pk(cluster, 1, 2); + int pk = pk(FOUR_NODES, 1, 2); - // {1} promises and accepts on !{3} => {1, 2}; commmits on !{2, 3} => {1} - cluster.filters().verbs(PAXOS_PREPARE_REQ.id, READ_REQ.id).from(1).to(3).drop(); - cluster.filters().verbs(PAXOS_PROPOSE_REQ.id).from(1).to(3).drop(); - cluster.filters().verbs(PAXOS_COMMIT_REQ.id).from(1).to(2, 3).drop(); - assertRows(cluster.coordinator(1).execute("INSERT INTO " + KEYSPACE + ".tbl (pk, ck, v) VALUES (?, 1, 1) IF NOT EXISTS", ConsistencyLevel.ONE, pk), - row(true)); + // {1} promises and accepts on !{3} => {1, 2}; commmits on !{2, 3} => {1} + drop(FOUR_NODES, 1, to(3), to(3), to(2, 3)); + assertRows(FOUR_NODES.coordinator(1).execute("INSERT INTO " + KEYSPACE + "." + tableName + " (pk, ck, v) VALUES (?, 1, 1) IF NOT EXISTS", ONE, pk), + row(true)); - // finish topology change - for (int i = 1 ; i <= 4 ; ++i) - cluster.get(i).acceptsOnInstance(UnsafeGossipHelper::addToRingNormal).accept(cluster.get(4)); + // finish topology change + for (int i = 1 ; i <= 4 ; ++i) + FOUR_NODES.get(i).acceptsOnInstance(CASTestBase::addToRingNormal).accept(FOUR_NODES.get(4)); - // {3} reads from !{2} => {3, 4} - cluster.filters().verbs(PAXOS_PREPARE_REQ.id, READ_REQ.id).from(3).to(2).drop(); - assertRows(cluster.coordinator(3).execute("SELECT * FROM " + KEYSPACE + ".tbl WHERE pk = ?", ConsistencyLevel.SERIAL, pk), - row(pk, 1, 1)); - } + // {3} reads from !{2} => {3, 4} + drop(FOUR_NODES, 3, to(2), to(), to()); + assertRows(FOUR_NODES.coordinator(3).execute("SELECT * FROM " + KEYSPACE + "." + tableName + " WHERE pk = ?", SERIAL, pk), + row(pk, 1, 1)); } /** * Successful write during range movement not witnessed by write after range movement * * - Range moves from {1, 2, 3} to {2, 3, 4}; witnessed by X (not by !X) - * - !X: Prepare and Propose to {1, 2} + * - !X: Promised and Accepted by {1, 2} * - Range movement witnessed by !X - * - Any: Prepare and Propose to {3, 4} + * - Any: Promised and Propose to {3, 4} */ - @Ignore @Test public void testSuccessfulWriteDuringRangeMovementFollowedByConflicting() throws Throwable { - try (Cluster cluster = Cluster.create(4, config -> config - .set("write_request_timeout", REQUEST_TIMEOUT) - .set("cas_contention_timeout", CONTENTION_TIMEOUT))) - { - cluster.schemaChange("CREATE KEYSPACE " + KEYSPACE + " WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 3};"); - cluster.schemaChange("CREATE TABLE " + KEYSPACE + ".tbl (pk int, ck int, v1 int, v2 int, PRIMARY KEY (pk, ck))"); + FOUR_NODES.schemaChange("CREATE TABLE " + KEYSPACE + ".tbl (pk int, ck int, v1 int, v2 int, PRIMARY KEY (pk, ck))"); - // make it so {4} is bootstrapping, and this has not propagated to other nodes yet - for (int i = 1 ; i <= 4 ; ++i) - cluster.get(1).acceptsOnInstance(UnsafeGossipHelper::removeFromRing).accept(cluster.get(4)); - cluster.get(4).acceptsOnInstance(UnsafeGossipHelper::addToRingBootstrapping).accept(cluster.get(4)); + // make it so {4} is bootstrapping, and this has propagated to only a quorum of other nodes + for (int i = 1 ; i <= 4 ; ++i) + FOUR_NODES.get(i).acceptsOnInstance(CASTestBase::removeFromRing).accept(FOUR_NODES.get(4)); + for (int i = 2 ; i <= 4 ; ++i) + FOUR_NODES.get(i).acceptsOnInstance(CASTestBase::addToRingBootstrapping).accept(FOUR_NODES.get(4)); - int pk = pk(cluster, 1, 2); + int pk = pk(FOUR_NODES, 1, 2); - // {1} promises and accepts on !{3} => {1, 2}; commits on !{2, 3} => {1} - cluster.filters().verbs(PAXOS_PREPARE_REQ.id, READ_REQ.id).from(1).to(3).drop(); - cluster.filters().verbs(PAXOS_PROPOSE_REQ.id).from(1).to(3).drop(); - cluster.filters().verbs(PAXOS_COMMIT_REQ.id).from(1).to(2, 3).drop(); - assertRows(cluster.coordinator(1).execute("INSERT INTO " + KEYSPACE + ".tbl (pk, ck, v1) VALUES (?, 1, 1) IF NOT EXISTS", ConsistencyLevel.ONE, pk), - row(true)); + // {1} promises and accepts on !{3} => {1, 2}; commits on !{2, 3} => {1} + drop(FOUR_NODES, 1, to(3), to(3), to(2, 3)); + assertRows(FOUR_NODES.coordinator(1).execute("INSERT INTO " + KEYSPACE + ".tbl (pk, ck, v1) VALUES (?, 1, 1) IF NOT EXISTS", ONE, pk), + row(true)); - // finish topology change - for (int i = 1 ; i <= 4 ; ++i) - cluster.get(i).acceptsOnInstance(UnsafeGossipHelper::addToRingNormal).accept(cluster.get(4)); + // finish topology change + for (int i = 1 ; i <= 4 ; ++i) + FOUR_NODES.get(i).acceptsOnInstance(CASTestBase::addToRingNormal).accept(FOUR_NODES.get(4)); - // {3} reads from !{2} => {3, 4} - cluster.filters().verbs(PAXOS_PREPARE_REQ.id, READ_REQ.id).from(3).to(2).drop(); - assertRows(cluster.coordinator(3).execute("INSERT INTO " + KEYSPACE + ".tbl (pk, ck, v2) VALUES (?, 1, 2) IF NOT EXISTS", ConsistencyLevel.ONE, pk), - row(false, pk, 1, 1, null)); + // {3} reads from !{2} => {3, 4} + drop(FOUR_NODES, 3, to(2), to(), to()); + assertRows(FOUR_NODES.coordinator(3).execute("INSERT INTO " + KEYSPACE + ".tbl (pk, ck, v2) VALUES (?, 1, 2) IF NOT EXISTS", ONE, pk), + row(false, pk, 1, 1, null)); - // TODO: repair and verify base table state - } + // TODO: repair and verify base table state } /** @@ -533,60 +483,62 @@ public class CASTest extends TestBaseImpl * See CASSANDRA-15745 * * - Range moves from {1, 2, 3} to {2, 3, 4}; witnessed by X (not by !X) - * - X: Prepare to {2, 3, 4} - * - X: Propose to {4} - * - !X: Prepare and Propose to {1, 2} + * - X: Promised by {2, 3, 4} + * - X: Accepted by {4} + * - !X: Promised and Accepted by {1, 2} * - Range move visible by !X - * - Any: Prepare and Read from {3, 4} + * - Any: Promised and Propose to {3, 4} */ - @Ignore @Test public void testIncompleteWriteFollowedBySuccessfulWriteWithStaleRingDuringRangeMovementFollowedByRead() throws Throwable { - try (Cluster cluster = Cluster.create(4, config -> config - .set("write_request_timeout", REQUEST_TIMEOUT) - .set("cas_contention_timeout", CONTENTION_TIMEOUT))) + String tableName = tableName("tbl"); + FOUR_NODES.schemaChange("CREATE TABLE " + KEYSPACE + "." + tableName + " (pk int, ck int, v1 int, v2 int, PRIMARY KEY (pk, ck))"); + + // make it so {4} is bootstrapping, and this has propagated to only a quorum of other nodes + for (int i = 1 ; i <= 4 ; ++i) + FOUR_NODES.get(i).acceptsOnInstance(CASTestBase::removeFromRing).accept(FOUR_NODES.get(4)); + for (int i = 2 ; i <= 4 ; ++i) + FOUR_NODES.get(i).acceptsOnInstance(CASTestBase::addToRingBootstrapping).accept(FOUR_NODES.get(4)); + + int pk = pk(FOUR_NODES, 1, 2); + + // {4} promises !{1} => {2, 3, 4}, accepts on !{1, 2, 3} => {4} + try (AutoCloseable drop = drop(FOUR_NODES, 4, to(1), to(1, 2, 3), to())) { - cluster.schemaChange("CREATE KEYSPACE " + KEYSPACE + " WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 3};"); - cluster.schemaChange("CREATE TABLE " + KEYSPACE + ".tbl (pk int, ck int, v1 int, v2 int, PRIMARY KEY (pk, ck))"); - - // make it so {4} is bootstrapping, and this has not propagated to other nodes yet - for (int i = 1 ; i <= 4 ; ++i) - cluster.get(1).acceptsOnInstance(UnsafeGossipHelper::removeFromRing).accept(cluster.get(4)); - cluster.get(4).acceptsOnInstance(UnsafeGossipHelper::addToRingBootstrapping).accept(cluster.get(4)); - - int pk = pk(cluster, 1, 2); - - // {4} promises and accepts on !{1} => {2, 3, 4}; commits on !{1, 2, 3} => {4} - cluster.filters().verbs(PAXOS_PREPARE_REQ.id, READ_REQ.id).from(4).to(1).drop(); - cluster.filters().verbs(PAXOS_PROPOSE_REQ.id).from(4).to(1, 2, 3).drop(); - try - { - cluster.coordinator(4).execute("INSERT INTO " + KEYSPACE + ".tbl (pk, ck, v1) VALUES (?, 1, 1) IF NOT EXISTS", ConsistencyLevel.QUORUM, pk); - Assert.assertTrue(false); - } - catch (RuntimeException wrapped) - { - Assert.assertEquals("Operation timed out - received only 1 responses.", wrapped.getCause().getMessage()); - } - - // {1} promises and accepts on !{3} => {1, 2}; commits on !{2, 3} => {1} - cluster.filters().verbs(PAXOS_PREPARE_REQ.id, READ_REQ.id).from(1).to(3).drop(); - cluster.filters().verbs(PAXOS_PROPOSE_REQ.id).from(1).to(3).drop(); - cluster.filters().verbs(PAXOS_COMMIT_REQ.id).from(1).to(2, 3).drop(); - assertRows(cluster.coordinator(1).execute("INSERT INTO " + KEYSPACE + ".tbl (pk, ck, v2) VALUES (?, 1, 2) IF NOT EXISTS", ConsistencyLevel.ONE, pk), - row(true)); - - // finish topology change - for (int i = 1 ; i <= 4 ; ++i) - cluster.get(i).acceptsOnInstance(UnsafeGossipHelper::addToRingNormal).accept(cluster.get(4)); - - // {3} reads from !{2} => {3, 4} - cluster.filters().verbs(PAXOS_PREPARE_REQ.id, READ_REQ.id).from(3).to(2).drop(); - cluster.filters().verbs(PAXOS_PROPOSE_REQ.id).from(3).to(2).drop(); - assertRows(cluster.coordinator(3).execute("SELECT * FROM " + KEYSPACE + ".tbl WHERE pk = ?", ConsistencyLevel.SERIAL, pk), - row(pk, 1, null, 2)); + FOUR_NODES.coordinator(4).execute("INSERT INTO " + KEYSPACE + "." + tableName + " (pk, ck, v1) VALUES (?, 1, 1) IF NOT EXISTS", QUORUM, pk); + Assert.assertTrue(false); } + catch (Throwable t) + { + if (!t.getClass().getName().equals(CasWriteTimeoutException.class.getName())) + throw t; + } + + // {1} promises and accepts on !{3} => {1, 2}; commits on !{2, 3} => {1} + drop(FOUR_NODES, 1, to(3), to(3), to(2, 3)); + // two options: either we can invalidate the previous operation and succeed, or we can complete the previous operation + Object[][] result = FOUR_NODES.coordinator(1).execute("INSERT INTO " + KEYSPACE + "." + tableName + " (pk, ck, v2) VALUES (?, 1, 2) IF NOT EXISTS", ONE, pk); + Object[] expectRow; + if (result[0].length == 1) + { + assertRows(result, row(true)); + expectRow = row(pk, 1, null, 2); + } + else + { + assertRows(result, row(false, pk, 1, 1, null)); + expectRow = row(pk, 1, 1, null); + } + + // finish topology change + for (int i = 1 ; i <= 4 ; ++i) + FOUR_NODES.get(i).acceptsOnInstance(CASTestBase::addToRingNormal).accept(FOUR_NODES.get(4)); + + // {3} reads from !{2} => {3, 4} + drop(FOUR_NODES, 3, to(2), to(2), to()); + assertRows(FOUR_NODES.coordinator(3).execute("SELECT * FROM " + KEYSPACE + "." + tableName + " WHERE pk = ?", SERIAL, pk), + expectRow); } /** @@ -597,79 +549,141 @@ public class CASTest extends TestBaseImpl * See CASSANDRA-15745 * * - Range moves from {1, 2, 3} to {2, 3, 4}; witnessed by X (not by !X) - * - X: Prepare to {2, 3, 4} - * - X: Propose to {4} - * - !X: Prepare and Propose to {1, 2} + * - X: Promised by {2, 3, 4} + * - X: Accepted by {4} + * - !X: Promised and Accepted by {1, 2} * - Range move visible by !X - * - Any: Prepare and Propose to {3, 4} + * - Any: Promised and Propose to {3, 4} */ - @Ignore @Test public void testIncompleteWriteFollowedBySuccessfulWriteWithStaleRingDuringRangeMovementFollowedByWrite() throws Throwable { - try (Cluster cluster = Cluster.create(4, config -> config - .set("write_request_timeout", REQUEST_TIMEOUT) - .set("cas_contention_timeout", CONTENTION_TIMEOUT))) + String tableName = tableName("tbl"); + FOUR_NODES.schemaChange("CREATE TABLE " + KEYSPACE + "." + tableName + " (pk int, ck int, v1 int, v2 int, PRIMARY KEY (pk, ck))"); + + // make it so {4} is bootstrapping, and this has propagated to only a quorum of other nodes + for (int i = 1; i <= 4; ++i) + FOUR_NODES.get(i).acceptsOnInstance(CASTestBase::removeFromRing).accept(FOUR_NODES.get(4)); + for (int i = 2; i <= 4; ++i) + FOUR_NODES.get(i).acceptsOnInstance(CASTestBase::addToRingBootstrapping).accept(FOUR_NODES.get(4)); + + int pk = pk(FOUR_NODES, 1, 2); + + // {4} promises and accepts on !{1} => {2, 3, 4}; commits on !{1, 2, 3} => {4} + drop(FOUR_NODES, 4, to(1), to(1, 2, 3), to()); + try { - cluster.schemaChange("CREATE KEYSPACE " + KEYSPACE + " WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 3};"); - cluster.schemaChange("CREATE TABLE " + KEYSPACE + ".tbl (pk int, ck int, v1 int, v2 int, PRIMARY KEY (pk, ck))"); + FOUR_NODES.coordinator(4).execute("INSERT INTO " + KEYSPACE + "." + tableName + " (pk, ck, v1) VALUES (?, 1, 1) IF NOT EXISTS", QUORUM, pk); + Assert.assertTrue(false); + } + catch (Throwable t) + { + if (!t.getClass().getName().equals(CasWriteTimeoutException.class.getName())) + throw t; + } - // make it so {4} is bootstrapping, and this has not propagated to other nodes yet - for (int i = 1 ; i <= 4 ; ++i) - cluster.get(1).acceptsOnInstance(UnsafeGossipHelper::removeFromRing).accept(cluster.get(4)); - cluster.get(4).acceptsOnInstance(UnsafeGossipHelper::addToRingBootstrapping).accept(cluster.get(4)); + // {1} promises and accepts on !{3} => {1, 2}; commits on !{2, 3} => {1} + drop(FOUR_NODES, 1, to(3), to(3), to(2, 3)); + // two options: either we can invalidate the previous operation and succeed, or we can complete the previous operation + Object[][] result = FOUR_NODES.coordinator(1).execute("INSERT INTO " + KEYSPACE + "." + tableName + " (pk, ck, v2) VALUES (?, 1, 2) IF NOT EXISTS", ONE, pk); + Object[] expectRow; + if (result[0].length == 1) + { + assertRows(result, row(true)); + expectRow = row(false, pk, 1, null, 2); + } + else + { + assertRows(result, row(false, pk, 1, 1, null)); + expectRow = row(false, pk, 1, 1, null); + } - int pk = pk(cluster, 1, 2); + // finish topology change + for (int i = 1; i <= 4; ++i) + FOUR_NODES.get(i).acceptsOnInstance(CASTestBase::addToRingNormal).accept(FOUR_NODES.get(4)); - // {4} promises and accepts on !{1} => {2, 3, 4}; commits on !{1, 2, 3} => {4} - cluster.filters().verbs(PAXOS_PREPARE_REQ.id, READ_REQ.id).from(4).to(1).drop(); - cluster.filters().verbs(PAXOS_PROPOSE_REQ.id).from(4).to(1, 2, 3).drop(); - try - { - cluster.coordinator(4).execute("INSERT INTO " + KEYSPACE + ".tbl (pk, ck, v1) VALUES (?, 1, 1) IF NOT EXISTS", ConsistencyLevel.QUORUM, pk); - Assert.assertTrue(false); - } - catch (RuntimeException wrapped) - { - Assert.assertEquals("Operation timed out - received only 1 responses.", wrapped.getCause().getMessage()); - } + // {3} reads from !{2} => {3, 4} + FOUR_NODES.filters().verbs(PAXOS2_PREPARE_REQ.id, PAXOS_PREPARE_REQ.id, READ_REQ.id).from(3).to(2).drop(); + FOUR_NODES.filters().verbs(PAXOS2_PROPOSE_REQ.id, PAXOS_PROPOSE_REQ.id).from(3).to(2).drop(); + assertRows(FOUR_NODES.coordinator(3).execute("INSERT INTO " + KEYSPACE + "." + tableName + " (pk, ck, v2) VALUES (?, 1, 2) IF NOT EXISTS", ONE, pk), + expectRow); + } - // {1} promises and accepts on !{3} => {1, 2}; commits on !{2, 3} => {1} - cluster.filters().verbs(PAXOS_PREPARE_REQ.id, READ_REQ.id).from(1).to(3).drop(); - cluster.filters().verbs(PAXOS_PROPOSE_REQ.id).from(1).to(3).drop(); - cluster.filters().verbs(PAXOS_COMMIT_REQ.id).from(1).to(2, 3).drop(); - assertRows(cluster.coordinator(1).execute("INSERT INTO " + KEYSPACE + ".tbl (pk, ck, v2) VALUES (?, 1, 2) IF NOT EXISTS", ConsistencyLevel.ONE, pk), - row(true)); + // TODO: RF changes + // TODO: Aborted range movements + // TODO: Leaving ring - // finish topology change - for (int i = 1 ; i <= 4 ; ++i) - cluster.get(i).acceptsOnInstance(UnsafeGossipHelper::addToRingNormal).accept(cluster.get(4)); + static void consistencyAfterWriteTimeoutTest(BiConsumer postTimeoutOperation1, BiConsumer postTimeoutOperation2, boolean loseCommitOfOperation1, Cluster cluster) + { + String tableName = tableName("t"); + String table = KEYSPACE + "." + tableName; + cluster.schemaChange("CREATE TABLE " + table + " (k int PRIMARY KEY, v int)"); - // {3} reads from !{2} => {3, 4} - cluster.filters().verbs(PAXOS_PREPARE_REQ.id, READ_REQ.id).from(3).to(2).drop(); - cluster.filters().verbs(PAXOS_PROPOSE_REQ.id).from(3).to(2).drop(); - assertRows(cluster.coordinator(3).execute("INSERT INTO " + KEYSPACE + ".tbl (pk, ck, v2) VALUES (?, 1, 2) IF NOT EXISTS", ConsistencyLevel.ONE, pk), - row(false, 5, 1, null, 2)); + // We do a CAS insertion, but have with the PROPOSE message dropped on node 1 and 2. The CAS will not get + // through and should timeout. Importantly, node 3 does receive and answer the PROPOSE. + IMessageFilters.Filter dropProposeFilter = cluster.filters() + .verbs(PAXOS_PROPOSE_REQ.id, PAXOS2_PROPOSE_REQ.id) + .to(1, 2) + .drop(); + + // Prepare A to {1, 2, 3} + // Propose A to {3} + // Timeout + try + { + // NOTE: the consistency below is the "commit" one, so it doesn't matter at all here. + cluster.coordinator(1) + .execute("INSERT INTO " + table + "(k, v) VALUES (0, 0) IF NOT EXISTS", ONE); + Assert.fail("The insertion should have timed-out"); + } + catch (Throwable t) + { + if (!t.getClass().getName().equals(CasWriteTimeoutException.class.getName())) + throw t; + } + finally + { + dropProposeFilter.off(); + } + + // Prepare and Propose to {1, 2} + // Commit(?) to either {1, 2, 3} or {3} + // Isolates node 3 and executes the SERIAL operation. As neither node 1 or 2 got the initial insert proposal, + // there is nothing to "replay" and the operation should assert the table is still empty. + IMessageFilters.Filter ignoreNode3Filter = cluster.filters().verbs(paxosAndReadVerbs()).to(3).drop(); + IMessageFilters.Filter dropCommitFilter = null; + if (loseCommitOfOperation1) + { + dropCommitFilter = cluster.filters().verbs(PAXOS_COMMIT_REQ.id).to(1, 2).drop(); + } + try + { + postTimeoutOperation1.accept(table, cluster.coordinator(1)); + } + finally + { + ignoreNode3Filter.off(); + if (dropCommitFilter != null) + dropCommitFilter.off(); + } + + // Node 3 is now back and we isolate node 2 to ensure the next read hits node 1 and 3. + // What we want to ensure is that despite node 3 having the initial insert in its paxos state in a position of + // being replayed, that insert is _not_ replayed (it would contradict serializability since the previous + // operation asserted nothing was inserted). It is this execution that failed before CASSANDRA-12126. + IMessageFilters.Filter ignoreNode2Filter = cluster.filters().verbs(paxosAndReadVerbs()).to(2).drop(); + try + { + postTimeoutOperation2.accept(table, cluster.coordinator(1)); + } + finally + { + ignoreNode2Filter.off(); } } - private static int pk(Cluster cluster, int lb, int ub) + protected Cluster getCluster() { - return pk(cluster.get(lb), cluster.get(ub)); - } - - private static int pk(IInstance lb, IInstance ub) - { - return pk(Murmur3Partitioner.instance.getTokenFactory().fromString(lb.config().getString("initial_token")), - Murmur3Partitioner.instance.getTokenFactory().fromString(ub.config().getString("initial_token"))); - } - - private static int pk(Token lb, Token ub) - { - int pk = 0; - Token pkt; - while (lb.compareTo(pkt = Murmur3Partitioner.instance.getToken(Int32Type.instance.decompose(pk))) >= 0 || ub.compareTo(pkt) < 0) - ++pk; - return pk; + return THREE_NODES; } } diff --git a/test/distributed/org/apache/cassandra/distributed/test/CASTestBase.java b/test/distributed/org/apache/cassandra/distributed/test/CASTestBase.java new file mode 100644 index 0000000000..59bcb9e385 --- /dev/null +++ b/test/distributed/org/apache/cassandra/distributed/test/CASTestBase.java @@ -0,0 +1,221 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.distributed.test; + +import java.util.Collections; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicInteger; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.db.ConsistencyLevel; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.Keyspace; +import org.apache.cassandra.db.marshal.Int32Type; +import org.apache.cassandra.dht.IPartitioner; +import org.apache.cassandra.dht.Murmur3Partitioner; +import org.apache.cassandra.dht.Token; +import org.apache.cassandra.distributed.Cluster; +import org.apache.cassandra.distributed.api.IInstance; +import org.apache.cassandra.distributed.api.IInstanceConfig; +import org.apache.cassandra.distributed.api.IMessageFilters; +import org.apache.cassandra.gms.ApplicationState; +import org.apache.cassandra.gms.EndpointState; +import org.apache.cassandra.gms.Gossiper; +import org.apache.cassandra.gms.HeartBeatState; +import org.apache.cassandra.gms.VersionedValue; +import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.net.MessagingService; +import org.apache.cassandra.net.Verb; +import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.service.PendingRangeCalculatorService; +import org.apache.cassandra.service.StorageService; +import org.apache.cassandra.service.paxos.PaxosRepair; +import org.apache.cassandra.utils.FBUtilities; + +import static org.apache.cassandra.db.ConsistencyLevel.SERIAL; +import static org.apache.cassandra.net.Verb.PAXOS2_PREPARE_REQ; +import static org.apache.cassandra.net.Verb.PAXOS2_PROPOSE_REQ; +import static org.apache.cassandra.net.Verb.PAXOS2_REPAIR_REQ; +import static org.apache.cassandra.net.Verb.PAXOS_COMMIT_REQ; +import static org.apache.cassandra.net.Verb.PAXOS_PREPARE_REQ; +import static org.apache.cassandra.net.Verb.PAXOS_PROPOSE_REQ; +import static org.apache.cassandra.net.Verb.READ_REQ; + +public abstract class CASTestBase extends TestBaseImpl +{ + private static final Logger logger = LoggerFactory.getLogger(CASTestBase.class); + + static final AtomicInteger TABLE_COUNTER = new AtomicInteger(0); + + static String tableName() + { + return tableName("tbl"); + } + + static String tableName(String prefix) + { + return prefix + TABLE_COUNTER.getAndIncrement(); + } + + static void repair(Cluster cluster, String tableName, int pk, int repairWith, int repairWithout) + { + IMessageFilters.Filter filter = cluster.filters().verbs( + PAXOS2_REPAIR_REQ.id, + PAXOS2_PREPARE_REQ.id, PAXOS_PREPARE_REQ.id, READ_REQ.id).from(repairWith).to(repairWithout).drop(); + cluster.get(repairWith).runOnInstance(() -> { + TableMetadata schema = Keyspace.open(KEYSPACE).getColumnFamilyStore(tableName).metadata.get(); + DecoratedKey key = schema.partitioner.decorateKey(Int32Type.instance.decompose(pk)); + try + { + PaxosRepair.create(SERIAL, key, null, schema).start().await(); + } + catch (Throwable t) + { + throw new RuntimeException(t); + } + }); + filter.off(); + } + + static int pk(Cluster cluster, int lb, int ub) + { + return pk(cluster.get(lb), cluster.get(ub)); + } + + static int pk(IInstance lb, IInstance ub) + { + return pk(Murmur3Partitioner.instance.getTokenFactory().fromString(lb.config().getString("initial_token")), + Murmur3Partitioner.instance.getTokenFactory().fromString(ub.config().getString("initial_token"))); + } + + static int pk(Token lb, Token ub) + { + int pk = 0; + Token pkt; + while (lb.compareTo(pkt = Murmur3Partitioner.instance.getToken(Int32Type.instance.decompose(pk))) >= 0 || ub.compareTo(pkt) < 0) + ++pk; + return pk; + } + + int[] to(int ... nodes) + { + return nodes; + } + + + private static final IMessageFilters.Matcher LOG_DROPPED = (from, to, message) -> { logger.info("Dropping {} from {} to {}", Verb.fromId(message.verb()), from, to); return true; }; + AutoCloseable drop(Cluster cluster, int from, int[] toPrepareAndRead, int[] toPropose, int[] toCommit) + { + IMessageFilters.Filter filter1 = cluster.filters().verbs(PAXOS2_PREPARE_REQ.id, PAXOS_PREPARE_REQ.id, READ_REQ.id).from(from).to(toPrepareAndRead).messagesMatching(LOG_DROPPED).drop(); + IMessageFilters.Filter filter2 = cluster.filters().verbs(PAXOS2_PROPOSE_REQ.id, PAXOS_PROPOSE_REQ.id).from(from).to(toPropose).messagesMatching(LOG_DROPPED).drop(); + IMessageFilters.Filter filter3 = cluster.filters().verbs(PAXOS_COMMIT_REQ.id).from(from).to(toCommit).messagesMatching(LOG_DROPPED).drop(); + return () -> { + filter1.off(); + filter2.off(); + filter3.off(); + }; + } + + AutoCloseable drop(Cluster cluster, int from, int[] toPrepare, int[] toRead, int[] toPropose, int[] toCommit) + { + IMessageFilters.Filter filter1 = cluster.filters().verbs(PAXOS2_PREPARE_REQ.id, PAXOS_PREPARE_REQ.id).from(from).to(toPrepare).drop(); + IMessageFilters.Filter filter2 = cluster.filters().verbs(READ_REQ.id).from(from).to(toRead).drop(); + IMessageFilters.Filter filter3 = cluster.filters().verbs(PAXOS2_PROPOSE_REQ.id, PAXOS_PROPOSE_REQ.id).from(from).to(toPropose).drop(); + IMessageFilters.Filter filter4 = cluster.filters().verbs(PAXOS_COMMIT_REQ.id).from(from).to(toCommit).drop(); + return () -> { + filter1.off(); + filter2.off(); + filter3.off(); + filter4.off(); + }; + } + + public static void addToRing(boolean bootstrapping, IInstance peer) + { + try + { + IInstanceConfig config = peer.config(); + IPartitioner partitioner = FBUtilities.newPartitioner(config.getString("partitioner")); + Token token = partitioner.getTokenFactory().fromString(config.getString("initial_token")); + InetAddressAndPort address = InetAddressAndPort.getByAddress(peer.broadcastAddress()); + + UUID hostId = config.hostId(); + Gossiper.runInGossipStageBlocking(() -> { + Gossiper.instance.initializeNodeUnsafe(address, hostId, 1); + Gossiper.instance.injectApplicationState(address, + ApplicationState.TOKENS, + new VersionedValue.VersionedValueFactory(partitioner).tokens(Collections.singleton(token))); + VersionedValue status = bootstrapping + ? new VersionedValue.VersionedValueFactory(partitioner).bootstrapping(Collections.singleton(token)) + : new VersionedValue.VersionedValueFactory(partitioner).normal(Collections.singleton(token)); + Gossiper.instance.injectApplicationState(address, ApplicationState.STATUS, status); + StorageService.instance.onChange(address, ApplicationState.STATUS, status); + Gossiper.instance.realMarkAlive(address, Gossiper.instance.getEndpointStateForEndpoint(address)); + }); + int version = Math.min(MessagingService.current_version, peer.getMessagingVersion()); + MessagingService.instance().versions.set(address, version); + + if (!bootstrapping) + assert StorageService.instance.getTokenMetadata().isMember(address); + PendingRangeCalculatorService.instance.blockUntilFinished(); + } + catch (Throwable e) // UnknownHostException + { + throw new RuntimeException(e); + } + } + + // reset gossip state so we know of the node being alive only + public static void removeFromRing(IInstance peer) + { + try + { + IInstanceConfig config = peer.config(); + IPartitioner partitioner = FBUtilities.newPartitioner(config.getString("partitioner")); + Token token = partitioner.getTokenFactory().fromString(config.getString("initial_token")); + InetAddressAndPort address = InetAddressAndPort.getByAddress(config.broadcastAddress()); + + Gossiper.runInGossipStageBlocking(() -> { + StorageService.instance.onChange(address, + ApplicationState.STATUS, + new VersionedValue.VersionedValueFactory(partitioner).left(Collections.singleton(token), 0L, 0)); + Gossiper.instance.unsafeAnnulEndpoint(address); + Gossiper.instance.realMarkAlive(address, new EndpointState(new HeartBeatState(0, 0))); + }); + PendingRangeCalculatorService.instance.blockUntilFinished(); + } + catch (Throwable e) // UnknownHostException + { + throw new RuntimeException(e); + } + } + + public static void addToRingNormal(IInstance peer) + { + addToRing(false, peer); + assert StorageService.instance.getTokenMetadata().isMember(InetAddressAndPort.getByAddress(peer.broadcastAddress())); + } + + public static void addToRingBootstrapping(IInstance peer) + { + addToRing(true, peer); + } +} diff --git a/test/distributed/org/apache/cassandra/distributed/test/CasCriticalSectionTest.java b/test/distributed/org/apache/cassandra/distributed/test/CasCriticalSectionTest.java new file mode 100644 index 0000000000..d354ba0e8c --- /dev/null +++ b/test/distributed/org/apache/cassandra/distributed/test/CasCriticalSectionTest.java @@ -0,0 +1,218 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.distributed.test; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Random; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.BooleanSupplier; + +import com.google.common.util.concurrent.Uninterruptibles; +import org.junit.Assert; +import org.junit.Test; + +import org.apache.cassandra.distributed.Cluster; +import org.apache.cassandra.distributed.api.ConsistencyLevel; + +/** + * A simple sanity check that uses CAS as mutex: each lock tries to CAS a variable thread_id for a specific row, + * and set its own thread id. Then it sleeps for a short time, and releases the lock. + * + * Write timeouts are handled by simply re-reading the variable and checking if locking has actually succeeded. + */ +public class CasCriticalSectionTest extends TestBaseImpl +{ + private static Random rng = new Random(); + private static final int threadCount = 5; + private static final int rowCount = 1; + + @Test + public void criticalSectionTest() throws IOException, InterruptedException + { + try (Cluster cluster = init(Cluster.create(5, c -> c.set("paxos_variant", "v2") + .set("write_request_timeout_in_ms", 200L) + .set("cas_contention_timeout_in_ms", 200L) + .set("request_timeout_in_ms", 200L)))) + { + cluster.schemaChange("create table " + KEYSPACE + ".tbl (pk int, ck int, thread_id int, PRIMARY KEY(pk, ck))"); + + List threads = new ArrayList<>(); + + AtomicBoolean failed = new AtomicBoolean(false); + AtomicBoolean stop = new AtomicBoolean(false); + BooleanSupplier exitCondition = () -> failed.get() || stop.get(); + + for (int i = 0; i < rowCount; i++) + { + final int rowId = i; + AtomicInteger counter = new AtomicInteger(); + cluster.coordinator(1) + .execute("insert into " + KEYSPACE + ".tbl (pk, ck, thread_id) VALUES (?, ?, ?) IF NOT EXISTS", + ConsistencyLevel.QUORUM, + 1, rowId, 0); + + // threads should be numbered from 1 to allow 0 to be "unlocked" + for (int j = 1; j <= threadCount; j++) + { + int threadId = j; + AtomicInteger lockedTimes = new AtomicInteger(); + AtomicInteger unlockedTimes = new AtomicInteger(); + + Runnable sanityCheck = () -> { + Assert.assertEquals(lockedTimes.get(), unlockedTimes.get()); + }; + threads.add(new Thread(() -> { + while (!exitCondition.getAsBoolean()) + { + while (!tryLockOnce(cluster, threadId, rowId)) + { + if (exitCondition.getAsBoolean()) + { + sanityCheck.run(); + return; + } + } + int ctr = counter.getAndIncrement(); + if (ctr != 0) + { + failed.set(true); + Assert.fail(String.format("Thread %s encountered lock that is held by %d participants while trying to lock.", + Thread.currentThread().getName(), + ctr)); + } + + // hold lock for a bit + Uninterruptibles.sleepUninterruptibly(rng.nextInt(5), TimeUnit.MILLISECONDS); + ctr = counter.decrementAndGet(); + if (ctr != 0) + { + failed.set(true); + Assert.fail(String.format("Thread %s encountered lock that is held by %d participants while trying to unlock.", + Thread.currentThread().getName(), + ctr)); + } + while (!tryUnlockOnce(cluster, threadId, rowId)) + { + if (exitCondition.getAsBoolean()) + { + sanityCheck.run(); + return; + } + } + } + sanityCheck.run(); + }, String.format("CAS Thread %d-%d", rowId, threadId))); + } + } + + for (Thread thread : threads) + thread.start(); + + Thread.sleep(TimeUnit.SECONDS.toMillis(30)); + stop.set(true); + + for (Thread thread : threads) + thread.join(); + + Assert.assertFalse(failed.get()); + } + } + + public static boolean isCasSuccess(Object[][] res) + { + if (res == null || res.length != 1) + return false; + + return Arrays.equals(res[0], new Object[] {true}); + } + + public static boolean tryLockOnce(Cluster cluster, int threadId, int rowId) + { + Object[][] res = null; + + try + { + res = cluster.coordinator(rng.nextInt(cluster.size()) + 1) + .execute("update " + KEYSPACE + ".tbl SET thread_id = ? WHERE pk = ? AND ck = ? IF thread_id = 0", + ConsistencyLevel.QUORUM, + threadId, 1, rowId); + return isCasSuccess(res); + } + catch (Throwable writeTimeout) + { + while (true) + { + try + { + res = cluster.coordinator(rng.nextInt(cluster.size()) + 1) + .execute("SELECT thread_id FROM " + KEYSPACE + ".tbl WHERE pk = ? AND ck = ?", + ConsistencyLevel.SERIAL, + 1, rowId); + break; + } + catch (Throwable t) + { + // retry + } + } + + return (int) res[0][0] == threadId; + } + } + + public static boolean tryUnlockOnce(Cluster cluster, int threadId, int rowId) + { + Object[][] res = null; + + try + { + res = cluster.coordinator(rng.nextInt(cluster.size()) + 1) + .execute("update " + KEYSPACE + ".tbl SET thread_id = ? WHERE pk = ? AND ck = ? IF thread_id = ?", + ConsistencyLevel.QUORUM, + 0, 1, rowId, threadId); + return isCasSuccess(res); + } + catch (Throwable writeTimeout) + { + while (true) + { + try + { + res = cluster.coordinator(rng.nextInt(cluster.size()) + 1) + .execute("SELECT thread_id FROM " + KEYSPACE + ".tbl WHERE pk = ? AND ck = ?", + ConsistencyLevel.SERIAL, + 1, rowId); + break; + } + catch (Throwable t) + { + // retry + } + } + + return (int) res[0][0] != threadId; + } + } + +} \ No newline at end of file diff --git a/test/distributed/org/apache/cassandra/distributed/test/LegacyCASTest.java b/test/distributed/org/apache/cassandra/distributed/test/LegacyCASTest.java new file mode 100644 index 0000000000..0f764cf991 --- /dev/null +++ b/test/distributed/org/apache/cassandra/distributed/test/LegacyCASTest.java @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.distributed.test; + +import java.util.function.Consumer; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Ignore; +import org.junit.Test; + +import org.apache.cassandra.distributed.Cluster; +import org.apache.cassandra.distributed.api.IInstanceConfig; +import org.apache.cassandra.distributed.impl.UnsafeGossipHelper; + +import static org.apache.cassandra.distributed.api.ConsistencyLevel.ANY; +import static org.apache.cassandra.distributed.api.ConsistencyLevel.QUORUM; +import static org.apache.cassandra.distributed.shared.AssertUtils.assertRows; +import static org.apache.cassandra.distributed.shared.AssertUtils.row; + +public class LegacyCASTest extends CASCommonTestCases +{ + private static Cluster CLUSTER; + + @BeforeClass + public static void beforeClass() throws Throwable + { + TestBaseImpl.beforeClass(); + CLUSTER = init(Cluster.create(3, config())); + } + + @AfterClass + public static void afterClass() + { + if (CLUSTER != null) + CLUSTER.close(); + } + + private static Consumer config() + { + return config -> config + .set("paxos_variant", "v1") + .set("write_request_timeout_in_ms", 5000L) + .set("cas_contention_timeout_in_ms", 5000L) + .set("request_timeout_in_ms", 5000L); + } + + /** + * This particular variant is unique to legacy Paxos because of the differing quorums for consensus, read and commit. + * It is also unique to range movements with an even-numbered RF under legacy paxos. + * + * Range movements do not necessarily complete; they may be aborted. + * CAS consistency should not be affected by this. + * + * - Range moving from {1, 2} to {2, 3}; witnessed by all + * - Promised and Accepted on {2, 3}; Commits are delayed and arrive after next commit (or perhaps vanish) + * - Range move cancelled; a new one starts moving {1, 2} to {2, 4}; witnessed by all + * - Promised, Accepted and Committed on {1, 4} + */ + @Ignore // known to be unsafe, just documents issue + @Test + public void testAbortedRangeMovement() throws Throwable + { + try (Cluster cluster = Cluster.create(4, config())) + { + cluster.schemaChange("CREATE KEYSPACE " + KEYSPACE + " WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 3};"); + cluster.schemaChange("CREATE TABLE " + KEYSPACE + ".tbl (pk int, ck int, v1 int, v2 int, PRIMARY KEY (pk, ck))"); + int pk = pk(cluster, 1, 2); + + // set {3} bootstrapping, {4} not in ring + for (int i = 1 ; i <= 4 ; ++i) + cluster.get(i).acceptsOnInstance(UnsafeGossipHelper::removeFromRing).accept(cluster.get(3)); + for (int i = 1 ; i <= 4 ; ++i) + cluster.get(i).acceptsOnInstance(UnsafeGossipHelper::removeFromRing).accept(cluster.get(4)); + for (int i = 1 ; i <= 4 ; ++i) + cluster.get(i).acceptsOnInstance(UnsafeGossipHelper::addToRingBootstrapping).accept(cluster.get(3)); + + // {3} promises and accepts on !{1} => {2, 3} + // {3} commits do not YET arrive on either of {1, 2} (must be either due to read quorum differing on legacy Paxos) + drop(cluster, 3, to(1), to(), to(1), to(1, 2)); + assertRows(cluster.coordinator(3).execute("INSERT INTO " + KEYSPACE + ".tbl (pk, ck, v1) VALUES (?, 1, 1) IF NOT EXISTS", ANY, pk), + row(true)); + + // abort {3} bootstrap, start {4} bootstrap + for (int i = 1 ; i <= 4 ; ++i) + cluster.get(i).acceptsOnInstance(UnsafeGossipHelper::removeFromRing).accept(cluster.get(3)); + for (int i = 1 ; i <= 4 ; ++i) + cluster.get(i).acceptsOnInstance(UnsafeGossipHelper::addToRingBootstrapping).accept(cluster.get(4)); + + // {4} promises and accepts on !{2} => {1, 4} + // {4} commits on {1, 2, 4} + drop(cluster, 4, to(2), to(), to(2), to()); + assertRows(cluster.coordinator(4).execute("INSERT INTO " + KEYSPACE + ".tbl (pk, ck, v2) VALUES (?, 1, 2) IF NOT EXISTS", QUORUM, pk), + row(false, pk, 1, 1, null)); + } + } + + protected Cluster getCluster() + { + return CLUSTER; + } +} diff --git a/test/distributed/org/apache/cassandra/distributed/test/MoveTest.java b/test/distributed/org/apache/cassandra/distributed/test/MoveTest.java new file mode 100644 index 0000000000..07715d6e12 --- /dev/null +++ b/test/distributed/org/apache/cassandra/distributed/test/MoveTest.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.distributed.test; + +import java.util.ArrayList; +import java.util.List; + +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; +import org.junit.Assert; +import org.junit.Test; + +import org.apache.cassandra.distributed.Cluster; +import org.apache.cassandra.distributed.api.ConsistencyLevel; +import org.apache.cassandra.distributed.api.NodeToolResult; +import org.apache.cassandra.service.StorageService; + +import static org.apache.cassandra.distributed.api.Feature.GOSSIP; +import static org.apache.cassandra.distributed.api.Feature.NETWORK; + +public class MoveTest extends TestBaseImpl +{ + private static final String KEYSPACE = "move_test_ks"; + private static final String TABLE = "tbl"; + private static final String KS_TBL = KEYSPACE + '.' + TABLE; + + static + { + System.setProperty("cassandra.ring_delay_ms", "5000"); // down from 30s default + } + + private void move(boolean forwards) throws Throwable + { + try (Cluster cluster = Cluster.build(4) + .withConfig(config -> config.set("paxos_variant", "v2_without_linearizable_reads").with(NETWORK).with(GOSSIP)) + .start()) + { + cluster.schemaChange("CREATE KEYSPACE " + KEYSPACE + " WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 3};"); + cluster.schemaChange("CREATE TABLE " + KS_TBL + " (k int, c int, v int, primary key (k, c));"); + for (int i=0; i<30; i++) + { + cluster.coordinator(1).execute("INSERT INTO " + KS_TBL + " (k, c, v) VALUES (?, 1, 1) IF NOT EXISTS", + ConsistencyLevel.SERIAL, ConsistencyLevel.ALL, i); + } + + List initialTokens = new ArrayList<>(); + for (int i=0; i Iterables.getOnlyElement(StorageService.instance.getLocalTokens()).toString()).call(); + initialTokens.add(token); + } + Assert.assertEquals(Lists.newArrayList("-4611686018427387905", + "-3", + "4611686018427387899", + "9223372036854775801"), initialTokens); + + NodeToolResult result = cluster.get(forwards ? 2 : 3).nodetoolResult("move", "2305843009213693949"); + Assert.assertTrue(result.toString(), result.getRc() == 0); + } + } + + @Test + public void moveBack() throws Throwable + { + move(false); + } + + @Test + public void moveForwards() throws Throwable + { + move(true); + } + +} diff --git a/test/distributed/org/apache/cassandra/distributed/test/PaxosRepairTest.java b/test/distributed/org/apache/cassandra/distributed/test/PaxosRepairTest.java new file mode 100644 index 0000000000..842c6b2920 --- /dev/null +++ b/test/distributed/org/apache/cassandra/distributed/test/PaxosRepairTest.java @@ -0,0 +1,620 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.distributed.test; + +import java.net.InetSocketAddress; +import java.util.*; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicIntegerArray; +import java.util.function.Consumer; +import java.util.stream.Collectors; + +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterators; +import com.google.common.collect.Sets; +import com.google.common.util.concurrent.Uninterruptibles; +import org.junit.Assert; +import org.junit.Ignore; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.config.*; +import org.apache.cassandra.cql3.ColumnIdentifier; +import org.apache.cassandra.cql3.QueryOptions; +import org.apache.cassandra.cql3.QueryProcessor; +import org.apache.cassandra.cql3.statements.SelectStatement; +import org.apache.cassandra.db.*; +import org.apache.cassandra.db.compaction.CompactionManager; +import org.apache.cassandra.db.marshal.Int32Type; +import org.apache.cassandra.db.partitions.PartitionIterator; +import org.apache.cassandra.db.partitions.PartitionUpdate; +import org.apache.cassandra.db.rows.*; +import org.apache.cassandra.distributed.Cluster; +import org.apache.cassandra.distributed.api.ConsistencyLevel; +import org.apache.cassandra.distributed.api.Feature; +import org.apache.cassandra.distributed.api.IInstance; +import org.apache.cassandra.distributed.api.IInstanceConfig; +import org.apache.cassandra.distributed.api.IInvokableInstance; +import org.apache.cassandra.distributed.api.IMessageFilters; +import org.apache.cassandra.exceptions.CasWriteTimeoutException; +import org.apache.cassandra.gms.FailureDetector; +import org.apache.cassandra.gms.Gossiper; +import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.gms.ApplicationState; +import org.apache.cassandra.gms.EndpointState; +import org.apache.cassandra.gms.VersionedValue; +import org.apache.cassandra.repair.RepairParallelism; +import org.apache.cassandra.repair.messages.RepairOption; +import org.apache.cassandra.schema.ColumnMetadata; +import org.apache.cassandra.schema.Schema; +import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.service.ActiveRepairService; +import org.apache.cassandra.service.ClientState; +import org.apache.cassandra.service.StorageService; +import org.apache.cassandra.service.paxos.*; +import org.apache.cassandra.service.paxos.cleanup.PaxosCleanup; +import org.apache.cassandra.service.paxos.uncommitted.PaxosRows; +import org.apache.cassandra.streaming.PreviewKind; +import org.apache.cassandra.utils.*; + +import static java.util.concurrent.TimeUnit.SECONDS; +import static org.apache.cassandra.distributed.shared.AssertUtils.assertRows; +import static org.apache.cassandra.distributed.shared.AssertUtils.row; +import static org.apache.cassandra.net.Verb.PAXOS2_CLEANUP_FINISH_PREPARE_REQ; +import static org.apache.cassandra.net.Verb.PAXOS2_CLEANUP_REQ; +import static org.apache.cassandra.net.Verb.PAXOS2_COMMIT_AND_PREPARE_REQ; +import static org.apache.cassandra.net.Verb.PAXOS2_PREPARE_REQ; +import static org.apache.cassandra.net.Verb.PAXOS2_PREPARE_RSP; +import static org.apache.cassandra.net.Verb.PAXOS2_PROPOSE_REQ; +import static org.apache.cassandra.net.Verb.PAXOS2_PROPOSE_RSP; +import static org.apache.cassandra.net.Verb.PAXOS2_REPAIR_REQ; +import static org.apache.cassandra.net.Verb.PAXOS_COMMIT_REQ; +import static org.apache.cassandra.net.Verb.PAXOS_COMMIT_RSP; +import static org.apache.cassandra.schema.SchemaConstants.SYSTEM_KEYSPACE_NAME; + +public class PaxosRepairTest extends TestBaseImpl +{ + private static final Logger logger = LoggerFactory.getLogger(PaxosRepairTest.class); + private static final String TABLE = "tbl"; + + static + { + CassandraRelevantProperties.PAXOS_EXECUTE_ON_SELF.setBoolean(false); + DatabaseDescriptor.daemonInitialization(); + } + + private static int getUncommitted(IInvokableInstance instance, String keyspace, String table) + { + if (instance.isShutdown()) + return 0; + int uncommitted = instance.callsOnInstance(() -> { + TableMetadata meta = Schema.instance.getTableMetadata(keyspace, table); + return Iterators.size(PaxosState.uncommittedTracker().uncommittedKeyIterator(meta.id, null)); + }).call(); + logger.info("{} has {} uncommitted instances", instance, uncommitted); + return uncommitted; + } + + private static void assertAllAlive(Cluster cluster) + { + Set allEndpoints = cluster.stream().map(i -> InetAddressAndPort.getByAddress(i.broadcastAddress())).collect(Collectors.toSet()); + cluster.stream().forEach(instance -> { + instance.runOnInstance(() -> { + ImmutableSet endpoints = Gossiper.instance.getEndpoints(); + Assert.assertEquals(allEndpoints, endpoints); + for (InetAddressAndPort endpoint : endpoints) + Assert.assertTrue(FailureDetector.instance.isAlive(endpoint)); + }); + }); + } + + private static void assertUncommitted(IInvokableInstance instance, String ks, String table, int expected) + { + Assert.assertEquals(expected, getUncommitted(instance, ks, table)); + } + + private static boolean hasUncommitted(Cluster cluster, String ks, String table) + { + return cluster.stream().map(instance -> getUncommitted(instance, ks, table)).reduce((a, b) -> a + b).get() > 0; + } + + private static boolean hasUncommittedQuorum(Cluster cluster, String ks, String table) + { + int uncommitted = 0; + for (int i=0; i 0) + uncommitted++; + } + return uncommitted >= ((cluster.size() / 2) + 1); + } + + private static void repair(Cluster cluster, String keyspace, String table, boolean force) + { + Map options = new HashMap<>(); + options.put(RepairOption.PARALLELISM_KEY, RepairParallelism.SEQUENTIAL.getName()); + options.put(RepairOption.PRIMARY_RANGE_KEY, Boolean.toString(false)); + options.put(RepairOption.INCREMENTAL_KEY, Boolean.toString(false)); + options.put(RepairOption.JOB_THREADS_KEY, Integer.toString(1)); + options.put(RepairOption.TRACE_KEY, Boolean.toString(false)); + options.put(RepairOption.COLUMNFAMILIES_KEY, ""); + options.put(RepairOption.PULL_REPAIR_KEY, Boolean.toString(false)); + options.put(RepairOption.FORCE_REPAIR_KEY, Boolean.toString(force)); + options.put(RepairOption.PREVIEW, PreviewKind.NONE.toString()); + options.put(RepairOption.IGNORE_UNREPLICATED_KS, Boolean.toString(false)); + options.put(RepairOption.REPAIR_PAXOS_KEY, Boolean.toString(true)); + options.put(RepairOption.PAXOS_ONLY_KEY, Boolean.toString(true)); + + cluster.get(1).runOnInstance(() -> { + int cmd = StorageService.instance.repairAsync(keyspace, options); + + while (true) + { + try + { + Thread.sleep(500); + } + catch (InterruptedException e) + { + throw new AssertionError(e); + } + Pair> status = ActiveRepairService.instance.getRepairStatus(cmd); + if (status == null) + continue; + + switch (status.left) + { + case IN_PROGRESS: + continue; + case COMPLETED: + return; + default: + throw new AssertionError("Repair failed with errors: " + status.right); + } + } + }); + } + + private static void repair(Cluster cluster, String keyspace, String table) + { + repair(cluster, keyspace, table, false); + } + + private static final Consumer CONFIG_CONSUMER = cfg -> { + cfg.with(Feature.NETWORK); + cfg.with(Feature.GOSSIP); + cfg.set("paxos_purge_grace_period", "0"); + cfg.set("paxos_state_purging", Config.PaxosStatePurging.repaired.toString()); + cfg.set("paxos_variant", "v2_without_linearizable_reads"); + cfg.set("truncate_request_timeout_in_ms", 1000L); + cfg.set("partitioner", "ByteOrderedPartitioner"); + cfg.set("initial_token", ByteBufferUtil.bytesToHex(ByteBufferUtil.bytes(cfg.num() * 100))); + }; + + @Test + public void paxosRepairTest() throws Throwable + { + try (Cluster cluster = init(Cluster.create(3, CONFIG_CONSUMER))) + { + cluster.schemaChange("CREATE TABLE " + KEYSPACE + '.' + TABLE + " (pk int, ck int, v int, PRIMARY KEY (pk, ck))"); + cluster.coordinator(1).execute("INSERT INTO " + KEYSPACE + '.' + TABLE + " (pk, ck, v) VALUES (1, 1, 1) IF NOT EXISTS", ConsistencyLevel.QUORUM); + Assert.assertFalse(hasUncommittedQuorum(cluster, KEYSPACE, TABLE)); + + assertAllAlive(cluster); + cluster.verbs(PAXOS_COMMIT_REQ).drop(); + try + { + cluster.coordinator(1).execute("INSERT INTO " + KEYSPACE + '.' + TABLE + " (pk, ck, v) VALUES (400, 2, 2) IF NOT EXISTS", ConsistencyLevel.QUORUM); + Assert.fail("expected write timeout"); + } + catch (RuntimeException e) + { + // exception expected + } + + Assert.assertTrue(hasUncommitted(cluster, KEYSPACE, TABLE)); + + cluster.filters().reset(); + + assertAllAlive(cluster); + repair(cluster, KEYSPACE, TABLE); + + Assert.assertFalse(hasUncommitted(cluster, KEYSPACE, TABLE)); + + cluster.forEach(i -> i.runOnInstance(() -> { + compactPaxos(); + Map rows = getPaxosRows(); + assertLowBoundPurged(rows.values()); + Assert.assertEquals(Sets.newHashSet(400), rows.keySet()); + })); + + // check that operations occuring after the last repair are not purged + cluster.coordinator(1).execute("INSERT INTO " + KEYSPACE + '.' + TABLE + " (pk, ck, v) VALUES (500, 3, 3) IF NOT EXISTS", ConsistencyLevel.QUORUM); + cluster.forEach(i -> i.runOnInstance(() -> { + compactPaxos(); + Map rows = getPaxosRows(); + assertLowBoundPurged(rows.values()); + Assert.assertEquals(Sets.newHashSet(400, 500), rows.keySet()); + })); + } + } + + @Ignore + @Test + public void topologyChangePaxosTest() throws Throwable + { + try (Cluster cluster = Cluster.build(4).withConfig(CONFIG_CONSUMER).createWithoutStarting()) + { + for (int i=1; i<=3; i++) + cluster.get(i).startup(); + + init(cluster); + cluster.schemaChange("CREATE TABLE " + KEYSPACE + '.' + TABLE + " (pk int, ck int, v int, PRIMARY KEY (pk, ck))"); + cluster.coordinator(1).execute("INSERT INTO " + KEYSPACE + '.' + TABLE + " (pk, ck, v) VALUES (1, 1, 1) IF NOT EXISTS", ConsistencyLevel.QUORUM); + + cluster.verbs(PAXOS_COMMIT_REQ).drop(); + try + { + cluster.coordinator(1).execute("INSERT INTO " + KEYSPACE + '.' + TABLE + " (pk, ck, v) VALUES (350, 2, 2) IF NOT EXISTS", ConsistencyLevel.QUORUM); + Assert.fail("expected write timeout"); + } + catch (RuntimeException e) + { + // exception expected + } + Assert.assertTrue(hasUncommitted(cluster, KEYSPACE, TABLE)); + + cluster.filters().reset(); + + // node 4 starting should repair paxos and inform the other nodes of its gossip state + cluster.get(4).startup(); + Assert.assertFalse(hasUncommittedQuorum(cluster, KEYSPACE, TABLE)); + } + } + + @Test + public void paxosCleanupWithReproposal() throws Throwable + { + try (Cluster cluster = init(Cluster.create(3, cfg -> cfg + .set("paxos_variant", "v2") + .set("paxos_purge_grace_period", 0) + .set("paxos_state_purging", Config.PaxosStatePurging.repaired.toString()) + .set("truncate_request_timeout_in_ms", 1000L)))) + { + cluster.schemaChange("CREATE TABLE " + KEYSPACE + '.' + TABLE + " (pk int, ck int, v int, PRIMARY KEY (pk, ck))"); + + cluster.verbs(PAXOS_COMMIT_REQ).drop(); + try + { + cluster.coordinator(1).execute("INSERT INTO " + KEYSPACE + '.' + TABLE + " (pk, ck, v) VALUES (1, 1, 1) IF NOT EXISTS", ConsistencyLevel.QUORUM); + Assert.fail("expected write timeout"); + } + catch (RuntimeException e) + { + // exception expected + } + Assert.assertTrue(hasUncommitted(cluster, KEYSPACE, TABLE)); + cluster.forEach(i -> i.runOnInstance(() -> Keyspace.open("system").getColumnFamilyStore("paxos").forceBlockingFlush())); + + CountDownLatch haveFetchedLowBound = new CountDownLatch(1); + CountDownLatch haveReproposed = new CountDownLatch(1); + cluster.verbs(PAXOS2_CLEANUP_FINISH_PREPARE_REQ).inbound().messagesMatching((from, to, verb) -> { + haveFetchedLowBound.countDown(); + Uninterruptibles.awaitUninterruptibly(haveReproposed); + return false; + }).drop(); + + ExecutorService executor = Executors.newCachedThreadPool(); + List endpoints = cluster.stream().map(IInstance::broadcastAddress).map(InetAddressAndPort::getByAddress).collect(Collectors.toList()); + Future cleanup = cluster.get(1).appliesOnInstance((List es, ExecutorService exec)-> { + TableMetadata metadata = Keyspace.open(KEYSPACE).getMetadata().getTableOrViewNullable(TABLE); + return PaxosCleanup.cleanup(es.stream().map(InetAddressAndPort::getByAddress).collect(Collectors.toSet()), metadata, StorageService.instance.getLocalRanges(KEYSPACE), false, exec); + }).apply(endpoints, executor); + + Uninterruptibles.awaitUninterruptibly(haveFetchedLowBound); + IMessageFilters.Filter filter2 = cluster.verbs(PAXOS_COMMIT_REQ, PAXOS2_COMMIT_AND_PREPARE_REQ).drop(); + try + { + cluster.coordinator(1).execute("INSERT INTO " + KEYSPACE + '.' + TABLE + " (pk, ck, v) VALUES (1, 1, 1) IF NOT EXISTS", ConsistencyLevel.QUORUM); + Assert.fail("expected write timeout"); + } + catch (RuntimeException e) + { + // exception expected + } + filter2.off(); + haveReproposed.countDown(); + cluster.filters().reset(); + + cleanup.get(); + ExecutorUtils.shutdownNowAndWait(1L, TimeUnit.MINUTES, executor); + Assert.assertFalse(hasUncommitted(cluster, KEYSPACE, TABLE)); + cluster.forEach(i -> i.runOnInstance(PaxosRepairTest::compactPaxos)); + for (int i = 1 ; i <= 3 ; ++i) + assertRows(cluster.get(i).executeInternal("SELECT * FROM " + KEYSPACE + '.' + TABLE + " WHERE pk = 1"), row(1, 1, 1)); + + Assert.assertFalse(hasUncommittedQuorum(cluster, KEYSPACE, TABLE)); + assertLowBoundPurged(cluster); + } + } + + @SuppressWarnings("unused") + @Test + public void paxosCleanupWithReproposalClashingTimestamp() throws Throwable + { + try (Cluster cluster = init(Cluster.create(5, cfg -> cfg + .set("paxos_variant", "v2") + .set("paxos_purge_grace_period", 0) + .set("paxos_cache_size", "0") + .set("truncate_request_timeout_in_ms", 1000L)))) + { + cluster.schemaChange("CREATE TABLE " + KEYSPACE + '.' + TABLE + " (pk int, ck int, v int, PRIMARY KEY (pk, ck))"); + + // we ensure: + // - node 1 only witnesses a promise that conflicts with something we committed + // - node 2 does not witness the commit, so it has an in progress proposal + // - node 3 does not witness the proposal, so that we have an incomplete commit + // - node 1's response arrives first, so that it might retain its promise as latestWitnessed (without bugfix) + + CountDownLatch haveStartedCleanup = new CountDownLatch(1); + CountDownLatch haveInsertedClashingPromise = new CountDownLatch(1); + IMessageFilters.Filter pauseCleanupUntilCommitted = cluster.verbs(PAXOS2_CLEANUP_REQ).from(1).to(1).outbound().messagesMatching((from, to, verb) -> { + haveStartedCleanup.countDown(); + Uninterruptibles.awaitUninterruptibly(haveInsertedClashingPromise); + return false; + }).drop(); + + ExecutorService executor = Executors.newCachedThreadPool(); + List endpoints = cluster.stream().map(i -> InetAddressAndPort.getByAddress(i.broadcastAddress())).collect(Collectors.toList()); + Future cleanup = cluster.get(1).appliesOnInstance((List es, ExecutorService exec)-> { + TableMetadata metadata = Keyspace.open(KEYSPACE).getMetadata().getTableOrViewNullable(TABLE); + return PaxosCleanup.cleanup(es.stream().map(InetAddressAndPort::getByAddress).collect(Collectors.toSet()), metadata, StorageService.instance.getLocalRanges(KEYSPACE), false, exec); + }).apply(endpoints, executor); + + IMessageFilters.Filter dropAllTo1 = cluster.verbs(PAXOS2_PREPARE_REQ, PAXOS2_PROPOSE_REQ, PAXOS_COMMIT_REQ).from(2).to(1).outbound().drop(); + IMessageFilters.Filter dropCommitTo3 = cluster.verbs(PAXOS_COMMIT_REQ).from(2).to(3).outbound().drop(); + IMessageFilters.Filter dropAcceptTo4 = cluster.verbs(PAXOS2_PROPOSE_REQ).from(2).to(4).outbound().drop(); + + CountDownLatch haveFetchedClashingRepair = new CountDownLatch(1); + AtomicIntegerArray fetchResponseIds = new AtomicIntegerArray(new int[] { -1, -1, -1, -1, -1, -1 }); + cluster.verbs(PAXOS2_REPAIR_REQ).outbound().from(1).messagesMatching((from, to, msg) -> { + fetchResponseIds.set(to, msg.id()); + return false; + }).drop(); + cluster.verbs(PAXOS2_PREPARE_RSP, PAXOS2_PROPOSE_RSP, PAXOS_COMMIT_RSP).outbound().to(1).messagesMatching((from, to, msg) -> { + if (fetchResponseIds.get(from) == msg.id()) + { + if (from == 1) haveFetchedClashingRepair.countDown(); + else Uninterruptibles.awaitUninterruptibly(haveFetchedClashingRepair); + } + return false; + }).drop(); + + Uninterruptibles.awaitUninterruptibly(haveStartedCleanup); + cluster.coordinator(2).execute("INSERT INTO " + KEYSPACE + '.' + TABLE + " (pk, ck, v) VALUES (1, 1, 1) IF NOT EXISTS", ConsistencyLevel.ONE); + + UUID cfId = cluster.get(2).callOnInstance(() -> Keyspace.open(KEYSPACE).getColumnFamilyStore(TABLE).metadata.id.asUUID()); + TimeUUID uuid = (TimeUUID) cluster.get(2).executeInternal("select in_progress_ballot from system.paxos WHERE row_key = ? and cf_id = ?", Int32Type.instance.decompose(1), cfId)[0][0]; + TimeUUID clashingUuid = TimeUUID.fromBytes(uuid.msb(), 0); + cluster.get(1).executeInternal("update system.paxos set in_progress_ballot = ? WHERE row_key = ? and cf_id = ?", clashingUuid, Int32Type.instance.decompose(1), cfId); + Assert.assertEquals(clashingUuid, cluster.get(1).executeInternal("select in_progress_ballot from system.paxos WHERE row_key = ? and cf_id = ?", Int32Type.instance.decompose(1), cfId)[0][0]); + + Assert.assertTrue(hasUncommitted(cluster, KEYSPACE, TABLE)); + haveInsertedClashingPromise.countDown(); + + cleanup.get(); + ExecutorUtils.shutdownNowAndWait(1L, TimeUnit.MINUTES, executor); + } + } + + @Test + public void paxosCleanupWithDelayedProposal() throws Throwable + { + try (Cluster cluster = init(Cluster.create(3, cfg -> cfg + .set("paxos_variant", "v2") + .set("paxos_purge_grace_period", 0) + .set("paxos_state_purging", Config.PaxosStatePurging.repaired.toString()) + .set("truncate_request_timeout_in_ms", 1000L))) + ) + { + cluster.schemaChange("CREATE TABLE " + KEYSPACE + '.' + TABLE + " (pk int, ck int, v int, PRIMARY KEY (pk, ck))"); + + CountDownLatch haveFinishedRepair = new CountDownLatch(1); + cluster.verbs(PAXOS2_PREPARE_REQ).messagesMatching((from, to, verb) -> { + Uninterruptibles.awaitUninterruptibly(haveFinishedRepair); + return false; + }).drop(); + cluster.verbs(PAXOS_COMMIT_REQ).drop(); + Future insert = cluster.get(1).async(() -> { + cluster.coordinator(1).execute("INSERT INTO " + KEYSPACE + '.' + TABLE + " (pk, ck, v) VALUES (1, 1, 1) IF NOT EXISTS", ConsistencyLevel.QUORUM); + Assert.fail("expected write timeout"); + }).call(); + cluster.verbs(PAXOS2_CLEANUP_FINISH_PREPARE_REQ).messagesMatching((from, to, verb) -> { + haveFinishedRepair.countDown(); + try { insert.get(); } catch (Throwable t) {} + cluster.filters().reset(); + return false; + }).drop(); + + ExecutorService executor = Executors.newCachedThreadPool(); + + Uninterruptibles.sleepUninterruptibly(10L, TimeUnit.MILLISECONDS); + + List endpoints = cluster.stream().map(i -> InetAddressAndPort.getByAddress(i.broadcastAddress())).collect(Collectors.toList()); + Future cleanup = cluster.get(1).appliesOnInstance((List es, ExecutorService exec)-> { + TableMetadata metadata = Keyspace.open(KEYSPACE).getMetadata().getTableOrViewNullable(TABLE); + return PaxosCleanup.cleanup(es.stream().map(InetAddressAndPort::getByAddress).collect(Collectors.toSet()), metadata, StorageService.instance.getLocalRanges(KEYSPACE), false, exec); + }).apply(endpoints, executor); + + cleanup.get(); + try + { + insert.get(); + } + catch (Throwable t) + { + } + ExecutorUtils.shutdownNowAndWait(1L, TimeUnit.MINUTES, executor); + Assert.assertFalse(hasUncommittedQuorum(cluster, KEYSPACE, TABLE)); + + assertLowBoundPurged(cluster); + } + } + + private static void setVersion(IInvokableInstance instance, InetSocketAddress peer, String version) + { + instance.runOnInstance(() -> { + Gossiper.runInGossipStageBlocking(() -> { + EndpointState epState = Gossiper.instance.getEndpointStateForEndpoint(InetAddressAndPort.getByAddress(peer.getAddress())); + VersionedValue value = version != null ? StorageService.instance.valueFactory.rack(version) : null; + epState.addApplicationState(ApplicationState.RELEASE_VERSION, value); + }); + }); + } + + private static void assertRepairFailsWithVersion(Cluster cluster, String version) + { + setVersion(cluster.get(1), cluster.get(2).broadcastAddress(), version); + try + { + repair(cluster, KEYSPACE, TABLE); + } + catch (AssertionError e) + { + return; + } + Assert.fail("Repair should have failed on unsupported version"); + } + + private static void assertRepairSucceedsWithVersion(Cluster cluster, String version) + { + setVersion(cluster.get(1), cluster.get(2).broadcastAddress(), version); + repair(cluster, KEYSPACE, TABLE); + } + + @Test + public void paxosRepairVersionGate() throws Throwable + { + try (Cluster cluster = init(Cluster.create(3, CONFIG_CONSUMER))) + { + cluster.schemaChange("CREATE TABLE " + KEYSPACE + '.' + TABLE + " (pk int, ck int, v int, PRIMARY KEY (pk, ck))"); + cluster.coordinator(1).execute("INSERT INTO " + KEYSPACE + '.' + TABLE + " (pk, ck, v) VALUES (1, 1, 1) IF NOT EXISTS", ConsistencyLevel.QUORUM); + Assert.assertFalse(hasUncommittedQuorum(cluster, KEYSPACE, TABLE)); + + assertAllAlive(cluster); + cluster.verbs(PAXOS_COMMIT_REQ).drop(); + try + { + cluster.coordinator(1).execute("INSERT INTO " + KEYSPACE + '.' + TABLE + " (pk, ck, v) VALUES (400, 2, 2) IF NOT EXISTS", ConsistencyLevel.QUORUM); + Assert.fail("expected write timeout"); + } + catch (RuntimeException e) + { + // exception expected + } + + Assert.assertTrue(hasUncommitted(cluster, KEYSPACE, TABLE)); + + cluster.filters().reset(); + + assertAllAlive(cluster); + + assertRepairFailsWithVersion(cluster, "3.0.24"); + assertRepairFailsWithVersion(cluster, "4.0.0"); + + // test valid versions + assertRepairSucceedsWithVersion(cluster, "4.1.0"); + } + } + + private static class PaxosRow + { + final DecoratedKey key; + final Row row; + + PaxosRow(DecoratedKey key, Row row) + { + this.key = key; + this.row = row; + } + + public String toString() + { + TableMetadata table = Schema.instance.getTableMetadata(SYSTEM_KEYSPACE_NAME, SystemKeyspace.PAXOS); + return ByteBufferUtil.bytesToHex(key.getKey()) + " -> " + row.toString(table, true); + } + } + + private static void compactPaxos() + { + ColumnFamilyStore paxos = Keyspace.open(SYSTEM_KEYSPACE_NAME).getColumnFamilyStore(SystemKeyspace.PAXOS); + FBUtilities.waitOnFuture(paxos.forceFlush()); + FBUtilities.waitOnFutures(CompactionManager.instance.submitMaximal(paxos, 0, false)); + } + + private static Map getPaxosRows() + { + Map rows = new HashMap<>(); + String queryStr = "SELECT * FROM " + SYSTEM_KEYSPACE_NAME + '.' + SystemKeyspace.PAXOS; + SelectStatement stmt = (SelectStatement) QueryProcessor.parseStatement(queryStr).prepare(ClientState.forInternalCalls()); + ReadQuery query = stmt.getQuery(QueryOptions.DEFAULT, FBUtilities.nowInSeconds()); + try (ReadExecutionController controller = query.executionController(); PartitionIterator partitions = query.executeInternal(controller)) + { + while (partitions.hasNext()) + { + try (RowIterator partition = partitions.next()) + { + while (partition.hasNext()) + { + rows.put(Int32Type.instance.compose(partition.partitionKey().getKey()), + new PaxosRow(partition.partitionKey(), partition.next())); + } + } + } + } + return rows; + } + + private static void assertLowBoundPurged(Collection rows) + { + Assert.assertEquals(0, DatabaseDescriptor.getPaxosPurgeGrace(SECONDS)); + String ip = FBUtilities.getBroadcastAddressAndPort().toString(); + for (PaxosRow row : rows) + { + Ballot keyLowBound = Keyspace.open(KEYSPACE).getColumnFamilyStore(TABLE).getPaxosRepairLowBound(row.key); + Assert.assertTrue(ip, Commit.isAfter(keyLowBound, Ballot.none())); + Assert.assertFalse(ip, PaxosRows.hasBallotBeforeOrEqualTo(row.row, keyLowBound)); + } + } + + private static void assertLowBoundPurged(IInvokableInstance instance) + { + instance.runOnInstance(() -> assertLowBoundPurged(getPaxosRows().values())); + } + + private static void assertLowBoundPurged(Cluster cluster) + { + cluster.forEach(PaxosRepairTest::assertLowBoundPurged); + } +} diff --git a/test/distributed/org/apache/cassandra/distributed/test/PaxosRepairTest2.java b/test/distributed/org/apache/cassandra/distributed/test/PaxosRepairTest2.java new file mode 100644 index 0000000000..e4ea6f2f7c --- /dev/null +++ b/test/distributed/org/apache/cassandra/distributed/test/PaxosRepairTest2.java @@ -0,0 +1,612 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.distributed.test; + +import java.net.InetAddress; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterators; +import com.google.common.collect.Sets; +import org.junit.Assert; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.config.CassandraRelevantProperties; +import org.apache.cassandra.config.Config; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.cql3.ColumnIdentifier; +import org.apache.cassandra.cql3.QueryOptions; +import org.apache.cassandra.cql3.QueryProcessor; +import org.apache.cassandra.cql3.statements.SelectStatement; +import org.apache.cassandra.db.Clustering; +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.Keyspace; +import org.apache.cassandra.db.ReadExecutionController; +import org.apache.cassandra.db.Memtable; +import org.apache.cassandra.db.ReadQuery; +import org.apache.cassandra.db.SystemKeyspace; +import org.apache.cassandra.db.compaction.CompactionManager; +import org.apache.cassandra.db.marshal.Int32Type; +import org.apache.cassandra.db.partitions.PartitionIterator; +import org.apache.cassandra.db.partitions.PartitionUpdate; +import org.apache.cassandra.db.rows.BTreeRow; +import org.apache.cassandra.db.rows.BufferCell; +import org.apache.cassandra.db.rows.Cell; +import org.apache.cassandra.db.rows.Row; +import org.apache.cassandra.db.rows.RowIterator; +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.Token; +import org.apache.cassandra.distributed.Cluster; +import org.apache.cassandra.distributed.api.ConsistencyLevel; +import org.apache.cassandra.distributed.api.Feature; +import org.apache.cassandra.distributed.api.IInvokableInstance; +import org.apache.cassandra.exceptions.CasWriteTimeoutException; +import org.apache.cassandra.gms.FailureDetector; +import org.apache.cassandra.gms.Gossiper; +import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.net.Verb; +import org.apache.cassandra.repair.RepairParallelism; +import org.apache.cassandra.repair.messages.RepairOption; +import org.apache.cassandra.schema.ColumnMetadata; +import org.apache.cassandra.schema.Schema; +import org.apache.cassandra.schema.TableId; +import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.service.ActiveRepairService; +import org.apache.cassandra.service.ClientState; +import org.apache.cassandra.service.StorageService; +import org.apache.cassandra.service.paxos.Ballot; +import org.apache.cassandra.service.paxos.Commit; +import org.apache.cassandra.service.paxos.Paxos; +import org.apache.cassandra.service.paxos.PaxosState; +import org.apache.cassandra.service.paxos.uncommitted.PaxosKeyState; +import org.apache.cassandra.service.paxos.uncommitted.PaxosRows; +import org.apache.cassandra.service.paxos.uncommitted.PaxosUncommittedTracker; +import org.apache.cassandra.service.paxos.uncommitted.PaxosUncommittedTracker.UpdateSupplier; +import org.apache.cassandra.streaming.PreviewKind; +import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.utils.Clock; +import org.apache.cassandra.utils.FBUtilities; +import org.apache.cassandra.utils.Pair; + +import static java.util.concurrent.TimeUnit.SECONDS; +import static org.apache.cassandra.schema.SchemaConstants.SYSTEM_KEYSPACE_NAME; +import static org.apache.cassandra.service.paxos.Ballot.Flag.GLOBAL; +import static org.apache.cassandra.service.paxos.BallotGenerator.Global.nextBallot; +import static org.apache.cassandra.service.paxos.BallotGenerator.Global.staleBallot; + +import org.apache.cassandra.utils.CloseableIterator; + +// quick workaround for metaspace ooms, will properly reuse clusters later +public class PaxosRepairTest2 extends TestBaseImpl +{ + private static final Logger logger = LoggerFactory.getLogger(PaxosRepairTest2.class); + private static final String TABLE = "tbl"; + + static + { + CassandraRelevantProperties.PAXOS_EXECUTE_ON_SELF.setBoolean(false); + DatabaseDescriptor.daemonInitialization(); + } + + private static int getUncommitted(IInvokableInstance instance, String keyspace, String table) + { + if (instance.isShutdown()) + return 0; + int uncommitted = instance.callsOnInstance(() -> { + TableMetadata cfm = Schema.instance.getTableMetadata(keyspace, table); + return Iterators.size(PaxosState.uncommittedTracker().uncommittedKeyIterator(cfm.id, null)); + }).call(); + logger.info("{} has {} uncommitted instances", instance, uncommitted); + return uncommitted; + } + + private static void assertAllAlive(Cluster cluster) + { + Set allEndpoints = cluster.stream().map(i -> i.broadcastAddress().getAddress()).collect(Collectors.toSet()); + cluster.stream().forEach(instance -> { + instance.runOnInstance(() -> { + ImmutableSet endpoints = Gossiper.instance.getEndpoints(); + Assert.assertEquals(allEndpoints, endpoints); + for (InetAddressAndPort endpoint : endpoints) + Assert.assertTrue(FailureDetector.instance.isAlive(endpoint)); + }); + }); + } + + private static void assertUncommitted(IInvokableInstance instance, String ks, String table, int expected) + { + Assert.assertEquals(expected, getUncommitted(instance, ks, table)); + } + + private static void repair(Cluster cluster, String keyspace, String table, boolean force) + { + Map options = new HashMap<>(); + options.put(RepairOption.PARALLELISM_KEY, RepairParallelism.SEQUENTIAL.getName()); + options.put(RepairOption.PRIMARY_RANGE_KEY, Boolean.toString(false)); + options.put(RepairOption.INCREMENTAL_KEY, Boolean.toString(false)); + options.put(RepairOption.JOB_THREADS_KEY, Integer.toString(1)); + options.put(RepairOption.TRACE_KEY, Boolean.toString(false)); + options.put(RepairOption.COLUMNFAMILIES_KEY, ""); + options.put(RepairOption.PULL_REPAIR_KEY, Boolean.toString(false)); + options.put(RepairOption.FORCE_REPAIR_KEY, Boolean.toString(force)); + options.put(RepairOption.PREVIEW, PreviewKind.NONE.toString()); + options.put(RepairOption.IGNORE_UNREPLICATED_KS, Boolean.toString(false)); + options.put(RepairOption.REPAIR_PAXOS_KEY, Boolean.toString(true)); + options.put(RepairOption.PAXOS_ONLY_KEY, Boolean.toString(true)); + + cluster.get(1).runOnInstance(() -> { + int cmd = StorageService.instance.repairAsync(keyspace, options); + + while (true) + { + try + { + Thread.sleep(500); + } + catch (InterruptedException e) + { + throw new AssertionError(e); + } + Pair> status = ActiveRepairService.instance.getRepairStatus(cmd); + if (status == null) + continue; + + switch (status.left) + { + case IN_PROGRESS: + continue; + case COMPLETED: + return; + default: + throw new AssertionError("Repair failed with errors: " + status.right); + } + } + }); + } + + private static void repair(Cluster cluster, String keyspace, String table) + { + repair(cluster, keyspace, table, false); + } + + @Test + public void paxosRepairPreventsStaleReproposal() throws Throwable + { + Ballot staleBallot = Paxos.newBallot(Ballot.none(), org.apache.cassandra.db.ConsistencyLevel.SERIAL); + try (Cluster cluster = init(Cluster.create(3, cfg -> cfg + .set("paxos_variant", "v2") + .set("paxos_purge_grace_period", 0) + .set("truncate_request_timeout_in_ms", 1000L))) + ) + { + cluster.schemaChange("CREATE TABLE " + KEYSPACE + '.' + TABLE + " (k int primary key, v int)"); + repair(cluster, KEYSPACE, TABLE); + + // stop and start node 2 to test loading paxos repair history from disk + cluster.get(2).shutdown(); + cluster.get(2).startup(); + + for (int i=0; i { + ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(TABLE); + DecoratedKey key = cfs.decorateKey(ByteBufferUtil.bytes(1)); + Assert.assertFalse(FBUtilities.getBroadcastAddressAndPort().toString(), Commit.isAfter(staleBallot, cfs.getPaxosRepairLowBound(key))); + }); + } + + // add in the stale proposal + cluster.get(1).runOnInstance(() -> { + TableMetadata cfm = Schema.instance.getTableMetadata(KEYSPACE, TABLE); + DecoratedKey key = DatabaseDescriptor.getPartitioner().decorateKey(ByteBufferUtil.bytes(1)); + ColumnMetadata cdef = cfm.getColumn(new ColumnIdentifier("v", false)); + Cell cell = BufferCell.live(cdef, staleBallot.unixMicros(), ByteBufferUtil.bytes(1)); + Row row = BTreeRow.singleCellRow(Clustering.EMPTY, cell); + PartitionUpdate update = PartitionUpdate.singleRowUpdate(cfm, key, row); + Commit.Proposal proposal = new Commit.Proposal(staleBallot, update); + SystemKeyspace.savePaxosProposal(proposal); + }); + + // shutdown node 3 so we're guaranteed to see the stale proposal + cluster.get(3).shutdown(); + + // the stale inflight proposal should be ignored and the query should succeed + String query = "INSERT INTO " + KEYSPACE + '.' + TABLE + " (k, v) VALUES (1, 2) IF NOT EXISTS"; + Object[][] result = cluster.coordinator(1).execute(query, ConsistencyLevel.QUORUM); + Assert.assertEquals(new Object[][]{new Object[]{ true }}, result); + + assertLowBoundPurged(cluster.get(1)); + assertLowBoundPurged(cluster.get(2)); + } + } + + @Test + public void paxosRepairHistoryIsntUpdatedInForcedRepair() throws Throwable + { + Ballot staleBallot = staleBallot(System.currentTimeMillis() - 1000000, System.currentTimeMillis() - 100000, GLOBAL); + try (Cluster cluster = init(Cluster.create(3, cfg -> cfg.with(Feature.GOSSIP, Feature.NETWORK) + .set("paxos_variant", "v2") + .set("truncate_request_timeout_in_ms", 1000L))) + ) + { + cluster.schemaChange("CREATE TABLE " + KEYSPACE + '.' + TABLE + " (k int primary key, v int)"); + cluster.get(3).shutdown(); + InetAddressAndPort node3 = InetAddressAndPort.getByAddress(cluster.get(3).broadcastAddress()); + + for (int i = 0; i < 10; i++) + { + if (!cluster.get(1).callOnInstance(() -> FailureDetector.instance.isAlive(node3))) + break; + } + + repair(cluster, KEYSPACE, TABLE, true); + for (int i = 0; i < cluster.size() - 1; i++) + { + cluster.get(i + 1).runOnInstance(() -> { + ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(TABLE); + DecoratedKey key = cfs.decorateKey(ByteBufferUtil.bytes(1)); + Assert.assertTrue(FBUtilities.getBroadcastAddressAndPort().toString(), Commit.isAfter(staleBallot, cfs.getPaxosRepairLowBound(key))); + }); + } + } + } + + private static class PaxosRow + { + final DecoratedKey key; + final Row row; + + PaxosRow(DecoratedKey key, Row row) + { + this.key = key; + this.row = row; + } + + public String toString() + { + TableMetadata cfm = Schema.instance.getTableMetadata(SYSTEM_KEYSPACE_NAME, SystemKeyspace.PAXOS); + return ByteBufferUtil.bytesToHex(key.getKey()) + " -> " + row.toString(cfm, true); + } + } + + private static void compactPaxos() + { + ColumnFamilyStore paxos = Keyspace.open(SYSTEM_KEYSPACE_NAME).getColumnFamilyStore(SystemKeyspace.PAXOS); + FBUtilities.waitOnFuture(paxos.forceFlush()); + FBUtilities.waitOnFutures(CompactionManager.instance.submitMaximal(paxos, 0, false)); + } + + private static Map getPaxosRows() + { + Map rows = new HashMap<>(); + String queryStr = "SELECT * FROM " + SYSTEM_KEYSPACE_NAME + '.' + SystemKeyspace.PAXOS; + SelectStatement stmt = (SelectStatement) QueryProcessor.parseStatement(queryStr).prepare(ClientState.forInternalCalls()); + ReadQuery query = stmt.getQuery(QueryOptions.DEFAULT, FBUtilities.nowInSeconds()); + try (ReadExecutionController controller = query.executionController(); PartitionIterator partitions = query.executeInternal(controller)) + { + while (partitions.hasNext()) + { + RowIterator partition = partitions.next(); + while (partition.hasNext()) + { + rows.put(Int32Type.instance.compose(partition.partitionKey().getKey()), + new PaxosRow(partition.partitionKey(), partition.next())); + } + } + } + return rows; + } + + private static void assertLowBoundPurged(Collection rows) + { + Assert.assertEquals(0, DatabaseDescriptor.getPaxosPurgeGrace(SECONDS)); + String ip = FBUtilities.getBroadcastAddressAndPort().toString(); + for (PaxosRow row : rows) + { + Ballot keyLowBound = Keyspace.open(KEYSPACE).getColumnFamilyStore(TABLE).getPaxosRepairLowBound(row.key); + Assert.assertTrue(ip, Commit.isAfter(keyLowBound, Ballot.none())); + Assert.assertFalse(ip, PaxosRows.hasBallotBeforeOrEqualTo(row.row, keyLowBound)); + } + } + + private static void assertLowBoundPurged(IInvokableInstance instance) + { + instance.runOnInstance(() -> assertLowBoundPurged(getPaxosRows().values())); + } + + private static void assertLowBoundPurged(Cluster cluster) + { + cluster.forEach(PaxosRepairTest2::assertLowBoundPurged); + } + + @Test + public void paxosAutoRepair() throws Throwable + { + System.setProperty("cassandra.auto_repair_frequency_seconds", "1"); + System.setProperty("cassandra.disable_paxos_auto_repairs", "true"); + try (Cluster cluster = init(Cluster.create(3, cfg -> cfg + .set("paxos_variant", "v2") + .set("disable_paxos_repair", false) + .set("truncate_request_timeout_in_ms", 1000L))) + ) + { + cluster.schemaChange("CREATE TABLE " + KEYSPACE + '.' + TABLE + " (pk int, ck int, v int, PRIMARY KEY (pk, ck))"); + cluster.get(3).shutdown(); + cluster.verbs(Verb.PAXOS_COMMIT_REQ).drop(); + try + { + cluster.coordinator(1).execute("INSERT INTO " + KEYSPACE + '.' + TABLE + " (pk, ck, v) VALUES (1, 1, 1) IF NOT EXISTS", ConsistencyLevel.QUORUM); + Assert.fail("expected write timeout"); + } + catch (Throwable t) + { + // expected + } + assertUncommitted(cluster.get(1), KEYSPACE, TABLE, 1); + assertUncommitted(cluster.get(2), KEYSPACE, TABLE, 1); + + cluster.filters().reset(); + // paxos table needs at least 1 flush to be picked up by auto-repairs + cluster.get(1).flush("system"); + cluster.get(2).flush("system"); + // re-enable repairs + cluster.get(1).runOnInstance(() -> StorageService.instance.setPaxosAutoRepairsEnabled(true)); + cluster.get(2).runOnInstance(() -> StorageService.instance.setPaxosAutoRepairsEnabled(true)); + Thread.sleep(2000); + for (int i=0; i<20; i++) + { + if (!cluster.get(1).callsOnInstance(() -> PaxosState.uncommittedTracker().hasInflightAutoRepairs()).call() + && !cluster.get(2).callsOnInstance(() -> PaxosState.uncommittedTracker().hasInflightAutoRepairs()).call()) + break; + logger.info("Waiting for auto repairs to finish..."); + Thread.sleep(1000); + } + assertUncommitted(cluster.get(1), KEYSPACE, TABLE, 0); + assertUncommitted(cluster.get(2), KEYSPACE, TABLE, 0); + } + finally + { + System.clearProperty("cassandra.auto_repair_frequency_seconds"); + System.clearProperty("cassandra.disable_paxos_auto_repairs"); + } + } + + @Test + public void paxosPurgeGraceSeconds() throws Exception + { + int graceSeconds = 5; + try (Cluster cluster = init(Cluster.create(3, cfg -> cfg + .set("paxos_variant", "v2") + .set("paxos_purge_grace_period", graceSeconds + "s") + .set("paxos_state_purging", Config.PaxosStatePurging.repaired.toString()) + .set("truncate_request_timeout_in_ms", 1000L))) + ) + { + cluster.schemaChange("CREATE TABLE " + KEYSPACE + '.' + TABLE + " (pk int, ck int, v int, PRIMARY KEY (pk, ck))"); + cluster.coordinator(1).execute("INSERT INTO " + KEYSPACE + '.' + TABLE + " (pk, ck, v) VALUES (1, 1, 1) IF NOT EXISTS", ConsistencyLevel.QUORUM); + + repair(cluster, KEYSPACE, TABLE); + cluster.forEach(i -> i.runOnInstance(() -> { + compactPaxos(); + Map rows = getPaxosRows(); + Assert.assertEquals(Sets.newHashSet(1), rows.keySet()); + })); + + // wait for the grace period to pass, repair again, and the rows should be removed + Thread.sleep((graceSeconds + 1) * 1000); + repair(cluster, KEYSPACE, TABLE); + cluster.forEach(i -> i.runOnInstance(() -> { + compactPaxos(); + Map rows = getPaxosRows(); + Assert.assertEquals(Sets.newHashSet(), rows.keySet()); + })); + } + } + + static void assertTimeout(Runnable runnable) + { + try + { + runnable.run(); + Assert.fail("timeout expected"); + } + catch (RuntimeException e) + { + Assert.assertEquals(CasWriteTimeoutException.class.getName(), e.getClass().getName()); + } + } + + private static int ballotDeletion(Commit commit) + { + return (int) TimeUnit.MICROSECONDS.toSeconds(commit.ballot.unixMicros()) + SystemKeyspace.legacyPaxosTtlSec(commit.update.metadata()); + } + + private static void backdateTimestamps(int seconds) + { + long offsetMillis = SECONDS.toMillis(seconds); + ClientState.resetLastTimestamp(System.currentTimeMillis() - offsetMillis); + OffsettableClock.offsetMillis = -offsetMillis; + } + + public static class OffsettableClock implements Clock + { + private static volatile long offsetMillis = 0; + public long nanoTime() + { + return System.nanoTime(); // checkstyle: permit system clock + } + + public long currentTimeMillis() + { + return System.currentTimeMillis() + offsetMillis; // checkstyle: permit system clock + } + } + + @Test + public void legacyPurgeRepairLoop() throws Exception + { + CassandraRelevantProperties.CLOCK_GLOBAL.setString(OffsettableClock.class.getName()); + try (Cluster cluster = init(Cluster.create(3, cfg -> cfg + .set("paxos_variant", "v2") + .set("paxos_state_purging", "legacy") + .set("paxos_purge_grace_period", 0) + .set("truncate_request_timeout_in_ms", 1000L))) + ) + { + int ttl = 3*3600; + cluster.schemaChange("CREATE TABLE " + KEYSPACE + '.' + TABLE + " (pk int, ck int, v int, PRIMARY KEY (pk, ck)) WITH gc_grace_seconds=" + ttl); + + // prepare an operation ttl + 1 hour into the past on a single node + cluster.forEach(instance -> { + instance.runOnInstance(() -> { + backdateTimestamps(ttl + 3600); + }); + }); + cluster.filters().inbound().to(1, 2).drop(); + assertTimeout(() -> cluster.coordinator(3).execute("INSERT INTO " + KEYSPACE + '.' + TABLE + " (pk, ck, v) VALUES (400, 2, 2) IF NOT EXISTS", ConsistencyLevel.QUORUM)); + Ballot oldBallot = Ballot.fromUuid(cluster.get(3).callOnInstance(() -> { + TableMetadata cfm = Schema.instance.getTableMetadata(KEYSPACE, TABLE); + DecoratedKey dk = cfm.partitioner.decorateKey(ByteBufferUtil.bytes(400)); + try (PaxosState state = PaxosState.get(dk, cfm)) + { + return state.currentSnapshot().promised.asUUID(); + } + })); + + assertUncommitted(cluster.get(1), KEYSPACE, TABLE, 0); + assertUncommitted(cluster.get(2), KEYSPACE, TABLE, 0); + assertUncommitted(cluster.get(3), KEYSPACE, TABLE, 1); + + // commit an operation just over ttl in the past on the other nodes + cluster.filters().reset(); + cluster.filters().inbound().to(2).drop(); + cluster.forEach(instance -> { + instance.runOnInstance(() -> { + backdateTimestamps(ttl + 2); + }); + }); + cluster.coordinator(1).execute("INSERT INTO " + KEYSPACE + '.' + TABLE + " (pk, ck, v) VALUES (400, 2, 2) IF NOT EXISTS", ConsistencyLevel.QUORUM); + + // expire the cache entries + int nowInSec = FBUtilities.nowInSeconds(); + cluster.get(1).runOnInstance(() -> { + TableMetadata table = Schema.instance.getTableMetadata(KEYSPACE, TABLE); + DecoratedKey dk = table.partitioner.decorateKey(ByteBufferUtil.bytes(400)); + try (PaxosState state = PaxosState.get(dk, table)) + { + state.updateStateUnsafe(s -> { + Assert.assertNull(s.accepted); + Assert.assertTrue(Commit.isAfter(s.committed.ballot, oldBallot)); + Commit.CommittedWithTTL committed = new Commit.CommittedWithTTL(s.committed.ballot, + s.committed.update, + ballotDeletion(s.committed)); + Assert.assertTrue(committed.localDeletionTime < nowInSec); + return new PaxosState.Snapshot(Ballot.none(), Ballot.none(), null, committed); + }); + } + }); + + cluster.get(3).runOnInstance(() -> { + TableMetadata table = Schema.instance.getTableMetadata(KEYSPACE, TABLE); + DecoratedKey dk = table.partitioner.decorateKey(ByteBufferUtil.bytes(400)); + try (PaxosState state = PaxosState.get(dk, table)) + { + state.updateStateUnsafe(s -> { + Assert.assertNull(s.accepted); + Assert.assertTrue(Commit.isAfter(s.committed.ballot, oldBallot)); + Commit.CommittedWithTTL committed = new Commit.CommittedWithTTL(s.committed.ballot, + s.committed.update, + ballotDeletion(s.committed)); + Assert.assertTrue(committed.localDeletionTime < nowInSec); + return new PaxosState.Snapshot(oldBallot, oldBallot, null, committed); + }); + } + }); + + cluster.forEach(instance -> { + instance.runOnInstance(() -> { + backdateTimestamps(0); + }); + }); + + cluster.filters().reset(); + cluster.filters().inbound().to(2).drop(); + cluster.get(3).runOnInstance(() -> { + + TableMetadata table = Schema.instance.getTableMetadata(KEYSPACE, TABLE); + DecoratedKey dk = table.partitioner.decorateKey(ByteBufferUtil.bytes(400)); + + UpdateSupplier supplier = PaxosState.uncommittedTracker().unsafGetUpdateSupplier(); + try + { + PaxosState.uncommittedTracker().unsafSetUpdateSupplier(new SingleUpdateSupplier(table, dk, oldBallot)); + StorageService.instance.autoRepairPaxos(table.id).get(); + } + catch (Exception e) + { + throw new RuntimeException(e); + } + finally + { + PaxosState.uncommittedTracker().unsafSetUpdateSupplier(supplier); + } + }); + + assertUncommitted(cluster.get(1), KEYSPACE, TABLE, 0); + assertUncommitted(cluster.get(2), KEYSPACE, TABLE, 0); + assertUncommitted(cluster.get(3), KEYSPACE, TABLE, 0); + } + } + + private static class SingleUpdateSupplier implements UpdateSupplier + { + private final TableMetadata cfm; + private final DecoratedKey dk; + private final Ballot ballot; + + public SingleUpdateSupplier(TableMetadata cfm, DecoratedKey dk, Ballot ballot) + { + this.cfm = cfm; + this.dk = dk; + this.ballot = ballot; + } + + public CloseableIterator repairIterator(TableId cfId, Collection> ranges) + { + if (!cfId.equals(cfm.id)) + return CloseableIterator.empty(); + return CloseableIterator.wrap(Collections.singleton(new PaxosKeyState(cfId, dk, ballot, false)).iterator()); + } + + public CloseableIterator flushIterator(Memtable paxos) + { + throw new UnsupportedOperationException(); + } + } +} diff --git a/test/distributed/org/apache/cassandra/distributed/test/PaxosUncommittedIndexTest.java b/test/distributed/org/apache/cassandra/distributed/test/PaxosUncommittedIndexTest.java new file mode 100644 index 0000000000..0ef52e7d3d --- /dev/null +++ b/test/distributed/org/apache/cassandra/distributed/test/PaxosUncommittedIndexTest.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.distributed.test; + +import org.junit.Assert; +import org.junit.Test; + +import org.apache.cassandra.distributed.Cluster; +import org.apache.cassandra.distributed.api.Feature; + +public class PaxosUncommittedIndexTest extends TestBaseImpl +{ + @Test + public void indexCqlIsExportableAndParsableTest() throws Throwable + { + String expectedCreateCustomIndex = "CREATE CUSTOM INDEX \"PaxosUncommittedIndex\" ON system.paxos () USING 'org.apache.cassandra.service.paxos.uncommitted.PaxosUncommittedIndex'"; + try (Cluster dtestCluster = init(Cluster.build(1).withConfig(c -> c.with(Feature.NATIVE_PROTOCOL)).start())) + { + try (com.datastax.driver.core.Cluster clientCluster = com.datastax.driver.core.Cluster.builder().addContactPoint("127.0.0.1").build()) + { + Assert.assertTrue(clientCluster.getMetadata().exportSchemaAsString() + .contains(expectedCreateCustomIndex)); + Throwable thrown = null; + try + { + dtestCluster.schemaChange(expectedCreateCustomIndex); + } + catch (Throwable tr) + { + thrown = tr; + } + + // Check parsing succeeds and index creation fails + Assert.assertTrue(thrown.getMessage().contains("System keyspace 'system' is not user-modifiable")); + } + } + } +} diff --git a/test/distributed/org/apache/cassandra/distributed/test/ReadDigestConsistencyTest.java b/test/distributed/org/apache/cassandra/distributed/test/ReadDigestConsistencyTest.java index 05e705bc9b..85c2783161 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/ReadDigestConsistencyTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/ReadDigestConsistencyTest.java @@ -31,6 +31,7 @@ import org.apache.cassandra.distributed.api.ConsistencyLevel; import org.apache.cassandra.distributed.api.ICoordinator; import org.apache.cassandra.exceptions.SyntaxException; import org.apache.cassandra.utils.Throwables; +import org.apache.cassandra.utils.TimeUUID; public class ReadDigestConsistencyTest extends TestBaseImpl { @@ -65,7 +66,7 @@ public class ReadDigestConsistencyTest extends TestBaseImpl public static void checkTraceForDigestMismatch(ICoordinator coordinator, String query, Object... boundValues) { - UUID sessionId = UUID.randomUUID(); + UUID sessionId = TimeUUID.Generator.nextTimeUUID().asUUID(); try { coordinator.executeWithTracing(sessionId, query, ConsistencyLevel.ALL, boundValues); diff --git a/test/distributed/org/apache/cassandra/distributed/test/ReadRepairTest.java b/test/distributed/org/apache/cassandra/distributed/test/ReadRepairTest.java index 7ce55bcd63..77218ac7ef 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/ReadRepairTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/ReadRepairTest.java @@ -493,7 +493,7 @@ public class ReadRepairTest extends TestBaseImpl // on timestamp tie of RT and partition deletion: we should not generate RT bounds in such case, // since monotonicity is already ensured by the partition deletion, and RT is unnecessary there. // For details, see CASSANDRA-16453. - public static Object repairPartition(DecoratedKey partitionKey, Map mutations, ReplicaPlan.ForTokenWrite writePlan, @SuperCall Callable r) throws Exception + public static Object repairPartition(DecoratedKey partitionKey, Map mutations, ReplicaPlan.ForWrite writePlan, @SuperCall Callable r) throws Exception { Assert.assertEquals(2, mutations.size()); for (Mutation value : mutations.values()) diff --git a/test/distributed/org/apache/cassandra/distributed/test/SecondaryIndexTest.java b/test/distributed/org/apache/cassandra/distributed/test/SecondaryIndexTest.java index b530dcc4d0..3b55dcf27d 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/SecondaryIndexTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/SecondaryIndexTest.java @@ -39,6 +39,7 @@ import org.junit.Test; import org.apache.cassandra.distributed.Cluster; import org.apache.cassandra.distributed.api.ConsistencyLevel; +import org.apache.cassandra.utils.TimeUUID; public class SecondaryIndexTest extends TestBaseImpl { @@ -91,7 +92,7 @@ public class SecondaryIndexTest extends TestBaseImpl for (int i = 0 ; i < 33; ++i) { - UUID trace = UUID.randomUUID(); + UUID trace = TimeUUID.Generator.nextTimeUUID().asUUID(); Object[][] result = cluster.coordinator(1).executeWithTracing(trace, String.format("SELECT * FROM %s WHERE v = ?", tableName), ConsistencyLevel.ALL, i); Assert.assertEquals("Failed on iteration " + i, 3, result.length); diff --git a/test/microbench/org/apache/cassandra/test/microbench/MessageOutBench.java b/test/microbench/org/apache/cassandra/test/microbench/MessageOutBench.java index 4bfc2706fa..907047364a 100644 --- a/test/microbench/org/apache/cassandra/test/microbench/MessageOutBench.java +++ b/test/microbench/org/apache/cassandra/test/microbench/MessageOutBench.java @@ -21,7 +21,6 @@ package org.apache.cassandra.test.microbench; import java.io.IOException; import java.util.EnumMap; import java.util.Map; -import java.util.UUID; import java.util.concurrent.TimeUnit; import com.google.common.net.InetAddresses; @@ -37,7 +36,6 @@ import org.apache.cassandra.net.MessagingService; import org.apache.cassandra.net.NoPayload; import org.apache.cassandra.net.ParamType; import org.apache.cassandra.utils.TimeUUID; -import org.apache.cassandra.utils.UUIDGen; import org.openjdk.jmh.annotations.Benchmark; import org.openjdk.jmh.annotations.BenchmarkMode; import org.openjdk.jmh.annotations.Fork; @@ -74,12 +72,12 @@ public class MessageOutBench { DatabaseDescriptor.daemonInitialization(); - TimeUUID uuid = nextTimeUUID(); + TimeUUID timeUuid = nextTimeUUID(); Map parameters = new EnumMap<>(ParamType.class); if (withParams) { - parameters.put(ParamType.TRACE_SESSION, uuid); + parameters.put(ParamType.TRACE_SESSION, timeUuid); } addr = InetAddressAndPort.getByAddress(InetAddresses.forString("127.0.73.101")); diff --git a/test/microbench/org/apache/cassandra/test/microbench/TimedMonitorBench.java b/test/microbench/org/apache/cassandra/test/microbench/TimedMonitorBench.java new file mode 100644 index 0000000000..2420b71847 --- /dev/null +++ b/test/microbench/org/apache/cassandra/test/microbench/TimedMonitorBench.java @@ -0,0 +1,198 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.test.microbench; + +import org.openjdk.jmh.annotations.*; + +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; + +@BenchmarkMode(Mode.SampleTime) +@OutputTimeUnit(TimeUnit.NANOSECONDS) +@Warmup(iterations = 1, time = 1, timeUnit = TimeUnit.SECONDS) +@Measurement(iterations = 2, time = 10, timeUnit = TimeUnit.SECONDS) +@Fork(value = 2) +@Threads(4) +@State(Scope.Benchmark) +public class TimedMonitorBench +{ + @Param({"A", "B"}) + private String type; + + private Lock lock; + + @State(Scope.Thread) + public static class ThreadState + { + Lock lock; + + @Setup(Level.Iteration) + public void setup(TimedMonitorBench benchState) throws Throwable + { + if (benchState.type.equals("A")) lock = new A(); + else if (benchState.type.equals("B")) lock = new B(); + else throw new IllegalStateException(); + } + } + + @Setup(Level.Trial) + public void setup() throws Throwable + { + if (type.equals("A")) lock = new A(); + else if (type.equals("B")) lock = new B(); + else throw new IllegalStateException(); + } + + interface Lock + { + boolean lock(long deadline); + void maybeUnlock(); + } + + static class A implements Lock + { + private volatile Thread lockedBy; + private volatile int waiting; + + private static final AtomicReferenceFieldUpdater lockedByUpdater = AtomicReferenceFieldUpdater.newUpdater(A.class, Thread.class, "lockedBy"); + + public boolean lock(long deadline) + { + try + { + Thread thread = Thread.currentThread(); + if (lockedByUpdater.compareAndSet(this, null, thread)) + return true; + + synchronized (this) + { + waiting++; + + try + { + while (true) + { + if (lockedByUpdater.compareAndSet(this, null, thread)) + return true; + + while (lockedBy != null) + { + long now = System.nanoTime(); + if (now >= deadline) + return false; + + wait(1 + ((deadline - now) - 1) / 1000000); + } + } + } + finally + { + waiting--; + } + } + } + catch (InterruptedException e) + { + Thread.currentThread().interrupt(); + return false; + } + } + + public void maybeUnlock() + { + // no visibility requirements, as if we hold the lock it was last updated by us + if (lockedBy == null) + return; + + Thread thread = Thread.currentThread(); + + if (lockedBy == thread) + { + lockedBy = null; + if (waiting > 0) + { + synchronized (this) + { + notify(); + } + } + } + } + } + + static class B implements Lock + { + private Thread lockedBy; + + public synchronized boolean lock(long deadline) + { + try + { + Thread thread = Thread.currentThread(); + while (lockedBy != null) + { + long now = System.nanoTime(); + if (now >= deadline) + return false; + + wait(1 + ((deadline - now) - 1) / 1000000); + } + lockedBy = thread; + return true; + } + catch (InterruptedException e) + { + Thread.currentThread().interrupt(); + return false; + } + } + + public void maybeUnlock() + { + // no visibility requirements, as if we hold the lock it was last updated by us + if (lockedBy == null) + return; + + Thread thread = Thread.currentThread(); + + if (lockedBy == thread) + { + synchronized (this) + { + lockedBy = null; + notify(); + } + } + } + } + + @Benchmark + public void unshared(ThreadState state) + { + state.lock.lock(Long.MAX_VALUE); + state.lock.maybeUnlock(); + } + + @Benchmark + public void shared() + { + lock.lock(Long.MAX_VALUE); + lock.maybeUnlock(); + } +} diff --git a/test/simulator/asm/org/apache/cassandra/simulator/asm/ClassTransformer.java b/test/simulator/asm/org/apache/cassandra/simulator/asm/ClassTransformer.java index 0b9ee32008..6e6b0d30de 100644 --- a/test/simulator/asm/org/apache/cassandra/simulator/asm/ClassTransformer.java +++ b/test/simulator/asm/org/apache/cassandra/simulator/asm/ClassTransformer.java @@ -20,11 +20,15 @@ package org.apache.cassandra.simulator.asm; import java.util.EnumSet; import java.util.List; +import java.util.function.Consumer; import org.objectweb.asm.AnnotationVisitor; import org.objectweb.asm.ClassReader; import org.objectweb.asm.ClassVisitor; import org.objectweb.asm.ClassWriter; +import org.objectweb.asm.FieldVisitor; +import org.objectweb.asm.Handle; +import org.objectweb.asm.Label; import org.objectweb.asm.MethodVisitor; import org.objectweb.asm.Opcodes; import org.objectweb.asm.tree.AbstractInsnNode; @@ -40,8 +44,10 @@ import static org.apache.cassandra.simulator.asm.Flag.NO_PROXY_METHODS; import static org.apache.cassandra.simulator.asm.TransformationKind.HASHCODE; import static org.apache.cassandra.simulator.asm.TransformationKind.SYNCHRONIZED; import static org.apache.cassandra.simulator.asm.Utils.deterministicToString; +import static org.apache.cassandra.simulator.asm.Utils.visitEachRefType; import static org.apache.cassandra.simulator.asm.Utils.generateTryFinallyProxyCall; import static org.objectweb.asm.Opcodes.ACC_PRIVATE; +import static org.objectweb.asm.Opcodes.ACC_STATIC; import static org.objectweb.asm.Opcodes.ACC_SYNTHETIC; import static org.objectweb.asm.Opcodes.INVOKESTATIC; @@ -50,6 +56,49 @@ class ClassTransformer extends ClassVisitor implements MethodWriterSink private static final List DETERMINISM_SETUP = singletonList(new MethodInsnNode(INVOKESTATIC, "org/apache/cassandra/simulator/systems/InterceptibleThread", "enterDeterministicMethod", "()V", false)); private static final List DETERMINISM_CLEANUP = singletonList(new MethodInsnNode(INVOKESTATIC, "org/apache/cassandra/simulator/systems/InterceptibleThread", "exitDeterministicMethod", "()V", false)); + class DependentTypeVisitor extends MethodVisitor + { + public DependentTypeVisitor(int api, MethodVisitor methodVisitor) + { + super(api, methodVisitor); + } + + @Override + public void visitTypeInsn(int opcode, String type) + { + super.visitTypeInsn(opcode, type); + Utils.visitIfRefType(type, dependentTypes); + } + + @Override + public void visitFieldInsn(int opcode, String owner, String name, String descriptor) + { + super.visitFieldInsn(opcode, owner, name, descriptor); + Utils.visitIfRefType(descriptor, dependentTypes); + } + + @Override + public void visitMethodInsn(int opcode, String owner, String name, String descriptor, boolean isInterface) + { + super.visitMethodInsn(opcode, owner, name, descriptor, isInterface); + Utils.visitEachRefType(descriptor, dependentTypes); + } + + @Override + public void visitInvokeDynamicInsn(String name, String descriptor, Handle bootstrapMethodHandle, Object... bootstrapMethodArguments) + { + super.visitInvokeDynamicInsn(name, descriptor, bootstrapMethodHandle, bootstrapMethodArguments); + Utils.visitEachRefType(descriptor, dependentTypes); + } + + @Override + public void visitLocalVariable(String name, String descriptor, String signature, Label start, Label end, int index) + { + super.visitLocalVariable(name, descriptor, signature, start, end, index); + Utils.visitIfRefType(descriptor, dependentTypes); + } + } + private final String className; private final ChanceSupplier monitorDelayChance; private final NemesisGenerator nemesis; @@ -59,24 +108,26 @@ class ClassTransformer extends ClassVisitor implements MethodWriterSink private boolean isTransformed; private boolean isCacheablyTransformed = true; private final EnumSet flags; + private final Consumer dependentTypes; - ClassTransformer(int api, String className, EnumSet flags) + ClassTransformer(int api, String className, EnumSet flags, Consumer dependentTypes) { - this(api, new ClassWriter(0), className, flags, null, null, null, null); + this(api, new ClassWriter(0), className, flags, null, null, null, null, dependentTypes); } - ClassTransformer(int api, String className, EnumSet flags, ChanceSupplier monitorDelayChance, NemesisGenerator nemesis, NemesisFieldKind.Selector nemesisFieldSelector, Hashcode insertHashcode) + ClassTransformer(int api, String className, EnumSet flags, ChanceSupplier monitorDelayChance, NemesisGenerator nemesis, NemesisFieldKind.Selector nemesisFieldSelector, Hashcode insertHashcode, Consumer dependentTypes) { - this(api, new ClassWriter(0), className, flags, monitorDelayChance, nemesis, nemesisFieldSelector, insertHashcode); + this(api, new ClassWriter(0), className, flags, monitorDelayChance, nemesis, nemesisFieldSelector, insertHashcode, dependentTypes); } - private ClassTransformer(int api, ClassWriter classWriter, String className, EnumSet flags, ChanceSupplier monitorDelayChance, NemesisGenerator nemesis, NemesisFieldKind.Selector nemesisFieldSelector, Hashcode insertHashcode) + private ClassTransformer(int api, ClassWriter classWriter, String className, EnumSet flags, ChanceSupplier monitorDelayChance, NemesisGenerator nemesis, NemesisFieldKind.Selector nemesisFieldSelector, Hashcode insertHashcode, Consumer dependentTypes) { super(api, classWriter); if (flags.contains(NEMESIS) && (nemesis == null || nemesisFieldSelector == null)) throw new IllegalArgumentException(); if (flags.contains(MONITORS) && monitorDelayChance == null) throw new IllegalArgumentException(); + this.dependentTypes = dependentTypes; this.className = className; this.flags = flags; this.monitorDelayChance = monitorDelayChance; @@ -86,12 +137,28 @@ class ClassTransformer extends ClassVisitor implements MethodWriterSink this.methodLogger = MethodLogger.log(api, className); } + @Override + public FieldVisitor visitField(int access, String name, String descriptor, String signature, Object value) + { + if (dependentTypes != null) + Utils.visitIfRefType(descriptor, dependentTypes); + return super.visitField(access, name, descriptor, signature, value); + } + @Override public MethodVisitor visitMethod(int access, String name, String descriptor, String signature, String[] exceptions) { + if (dependentTypes != null) + visitEachRefType(descriptor, dependentTypes); + EnumSet flags = this.flags; if (flags.isEmpty() || ((access & ACC_SYNTHETIC) != 0 && (name.endsWith("$unsync") || name.endsWith("$catch") || name.endsWith("$nemesis")))) - return super.visitMethod(access, name, descriptor, signature, exceptions); + { + MethodVisitor visitor = super.visitMethod(access, name, descriptor, signature, exceptions); + if (dependentTypes != null && (access & (ACC_STATIC | ACC_SYNTHETIC)) != 0 && (name.equals("") || name.startsWith("lambda$"))) + visitor = new DependentTypeVisitor(api, visitor); + return visitor; + } boolean isToString = false; if (access == Opcodes.ACC_PUBLIC && name.equals("toString") && descriptor.equals("()Ljava/lang/String;") && !flags.contains(NO_PROXY_METHODS)) @@ -129,6 +196,8 @@ class ClassTransformer extends ClassVisitor implements MethodWriterSink visitor = new GlobalMethodTransformer(flags, this, api, name, visitor); if (flags.contains(NEMESIS)) visitor = new NemesisTransformer(this, api, name, visitor, nemesis, nemesisFieldSelector); + if (dependentTypes != null && (access & (ACC_STATIC | ACC_SYNTHETIC)) != 0 && (name.equals("") || name.startsWith("lambda$"))) + visitor = new DependentTypeVisitor(api, visitor); return visitor; } diff --git a/test/simulator/asm/org/apache/cassandra/simulator/asm/Flag.java b/test/simulator/asm/org/apache/cassandra/simulator/asm/Flag.java index 00b88ab085..d127064ebd 100644 --- a/test/simulator/asm/org/apache/cassandra/simulator/asm/Flag.java +++ b/test/simulator/asm/org/apache/cassandra/simulator/asm/Flag.java @@ -20,5 +20,5 @@ package org.apache.cassandra.simulator.asm; public enum Flag { - GLOBAL_CLOCK, MONITORS, LOCK_SUPPORT, GLOBAL_METHODS, DETERMINISTIC, NO_PROXY_METHODS, NEMESIS + GLOBAL_CLOCK, SYSTEM_CLOCK, MONITORS, LOCK_SUPPORT, GLOBAL_METHODS, DETERMINISTIC, NO_PROXY_METHODS, NEMESIS } diff --git a/test/simulator/asm/org/apache/cassandra/simulator/asm/GlobalMethodTransformer.java b/test/simulator/asm/org/apache/cassandra/simulator/asm/GlobalMethodTransformer.java index 868833a126..fbea223339 100644 --- a/test/simulator/asm/org/apache/cassandra/simulator/asm/GlobalMethodTransformer.java +++ b/test/simulator/asm/org/apache/cassandra/simulator/asm/GlobalMethodTransformer.java @@ -39,6 +39,7 @@ class GlobalMethodTransformer extends MethodVisitor private final String methodName; private boolean globalMethods; private boolean globalClock; + private boolean systemClock; private boolean lockSupport; private boolean deterministic; boolean hasSeenAnyMethodInsn; @@ -48,6 +49,7 @@ class GlobalMethodTransformer extends MethodVisitor super(api, parent); this.globalMethods = flags.contains(GLOBAL_METHODS); this.globalClock = flags.contains(Flag.GLOBAL_CLOCK); + this.systemClock = flags.contains(Flag.SYSTEM_CLOCK); this.lockSupport = flags.contains(Flag.LOCK_SUPPORT); this.deterministic = flags.contains(Flag.DETERMINISTIC); this.transformer = transformer; @@ -115,6 +117,11 @@ class GlobalMethodTransformer extends MethodVisitor transformer.witness(GLOBAL_METHOD); super.visitMethodInsn(Opcodes.INVOKESTATIC, "org/apache/cassandra/simulator/systems/SimulatedTime$Global", "nextGlobalMonotonicMicros", descriptor, false); } + else if (systemClock && opcode == Opcodes.INVOKESTATIC && owner.equals("java/lang/System") && (name.equals("nanoTime") || name.equals("currentTimeMillis"))) + { + transformer.witness(GLOBAL_METHOD); + super.visitMethodInsn(Opcodes.INVOKESTATIC, "org/apache/cassandra/simulator/systems/InterceptorOfSystemMethods$Global", name, descriptor, false); + } else { super.visitMethodInsn(opcode, owner, name, descriptor, isInterface); @@ -147,6 +154,7 @@ class GlobalMethodTransformer extends MethodVisitor default: throw new AssertionError(); case GLOBAL_METHODS: globalMethods = add; break; case GLOBAL_CLOCK: globalClock = add; break; + case SYSTEM_CLOCK: systemClock = add; break; case LOCK_SUPPORT: lockSupport = add; break; case DETERMINISTIC: deterministic = add; break; case MONITORS: throw new UnsupportedOperationException("Cannot currently toggle MONITORS at the method level"); diff --git a/test/simulator/asm/org/apache/cassandra/simulator/asm/InterceptAgent.java b/test/simulator/asm/org/apache/cassandra/simulator/asm/InterceptAgent.java index 019ea3d340..c58c5d26da 100644 --- a/test/simulator/asm/org/apache/cassandra/simulator/asm/InterceptAgent.java +++ b/test/simulator/asm/org/apache/cassandra/simulator/asm/InterceptAgent.java @@ -43,10 +43,13 @@ import org.objectweb.asm.MethodVisitor; import static org.apache.cassandra.simulator.asm.Flag.DETERMINISTIC; import static org.apache.cassandra.simulator.asm.Flag.LOCK_SUPPORT; import static org.apache.cassandra.simulator.asm.Flag.NO_PROXY_METHODS; +import static org.apache.cassandra.simulator.asm.Flag.SYSTEM_CLOCK; import static org.apache.cassandra.simulator.asm.InterceptClasses.BYTECODE_VERSION; +import static org.objectweb.asm.Opcodes.ARETURN; import static org.objectweb.asm.Opcodes.ALOAD; import static org.objectweb.asm.Opcodes.GETFIELD; import static org.objectweb.asm.Opcodes.GETSTATIC; +import static org.objectweb.asm.Opcodes.INVOKEINTERFACE; import static org.objectweb.asm.Opcodes.INVOKESPECIAL; import static org.objectweb.asm.Opcodes.INVOKESTATIC; import static org.objectweb.asm.Opcodes.INVOKEVIRTUAL; @@ -103,7 +106,7 @@ public class InterceptAgent return transformConcurrent(className, bytecode, DETERMINISTIC, NO_PROXY_METHODS); if (className.startsWith("java/util/concurrent/locks")) - return transformConcurrent(className, bytecode, LOCK_SUPPORT, NO_PROXY_METHODS); + return transformConcurrent(className, bytecode, SYSTEM_CLOCK, LOCK_SUPPORT, NO_PROXY_METHODS); return null; } @@ -253,6 +256,9 @@ public class InterceptAgent { class ThreadLocalRandomVisitor extends ClassVisitor { + // CassandraRelevantProperties is not available to us here + final boolean determinismCheck = System.getProperty("cassandra.test.simulator.determinismcheck", "none").matches("relaxed|strict"); + public ThreadLocalRandomVisitor(int api, ClassVisitor classVisitor) { super(api, classVisitor); @@ -307,7 +313,10 @@ public class InterceptAgent } else { - return super.visitMethod(access, name, descriptor, signature, exceptions); + MethodVisitor mv = super.visitMethod(access, name, descriptor, signature, exceptions); + if (determinismCheck && (name.equals("nextSeed") || name.equals("nextSecondarySeed"))) + mv = new ThreadLocalRandomCheckTransformer(api, mv); + return mv; } } } @@ -325,7 +334,7 @@ public class InterceptAgent private static byte[] transformConcurrent(String className, byte[] bytes, Flag flag, Flag ... flags) { - ClassTransformer transformer = new ClassTransformer(BYTECODE_VERSION, className, EnumSet.of(flag, flags)); + ClassTransformer transformer = new ClassTransformer(BYTECODE_VERSION, className, EnumSet.of(flag, flags), null); transformer.readAndTransform(bytes); if (!transformer.isTransformed()) return null; diff --git a/test/simulator/asm/org/apache/cassandra/simulator/asm/InterceptClasses.java b/test/simulator/asm/org/apache/cassandra/simulator/asm/InterceptClasses.java index f57fb77efa..11e7f89dda 100644 --- a/test/simulator/asm/org/apache/cassandra/simulator/asm/InterceptClasses.java +++ b/test/simulator/asm/org/apache/cassandra/simulator/asm/InterceptClasses.java @@ -18,22 +18,34 @@ package org.apache.cassandra.simulator.asm; +import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.io.InputStream; import java.io.Serializable; import java.io.UncheckedIOException; import java.lang.reflect.Method; import java.util.Collections; import java.util.EnumSet; +import java.util.HashSet; import java.util.Map; +import java.util.NavigableSet; import java.util.Set; +import java.util.TreeSet; import java.util.concurrent.ConcurrentHashMap; import java.util.function.BiFunction; +import java.util.function.Consumer; +import java.util.function.Predicate; import java.util.regex.Pattern; import org.objectweb.asm.Opcodes; +import static org.apache.cassandra.simulator.asm.InterceptClasses.Cached.Kind.MODIFIED; +import static org.apache.cassandra.simulator.asm.InterceptClasses.Cached.Kind.UNMODIFIED; +import static org.apache.cassandra.simulator.asm.InterceptClasses.Cached.Kind.UNSHAREABLE; + // TODO (completeness): confirm that those classes we weave monitor-access for only extend other classes we also weave monitor access for // TODO (completeness): confirm that those classes we weave monitor access for only take monitors on types we also weave monitor access for (and vice versa) +// WARNING: does not implement IClassTransformer directly as must be accessible to bootstrap class loader public class InterceptClasses implements BiFunction { public static final int BYTECODE_VERSION = Opcodes.ASM7; @@ -58,13 +70,35 @@ public class InterceptClasses implements BiFunction private static final Pattern NEMESIS = GLOBAL_METHODS; private static final Set WARNED = Collections.newSetFromMap(new ConcurrentHashMap<>()); - static final Cached SENTINEL = new Cached(null); + static final byte[] SENTINEL = new byte[0]; static class Cached { - final byte[] cached; - private Cached(byte[] cached) + enum Kind { MODIFIED, UNMODIFIED, UNSHAREABLE } + final Kind kind; + final byte[] bytes; + final Set uncacheablePeers; + private Cached(Kind kind, byte[] bytes, Set uncacheablePeers) { - this.cached = cached; + this.kind = kind; + this.bytes = bytes; + this.uncacheablePeers = uncacheablePeers; + } + } + + static class PeerGroup + { + final Set uncacheablePeers = new TreeSet<>(); + final Cached unmodified = new Cached(UNMODIFIED, null, uncacheablePeers); + } + + class SubTransformer implements BiFunction + { + private final Map isolatedCache = new ConcurrentHashMap<>(); + + @Override + public byte[] apply(String name, byte[] bytes) + { + return transformTransitiveClosure(name, bytes, isolatedCache); } } @@ -75,70 +109,168 @@ public class InterceptClasses implements BiFunction private final ChanceSupplier monitorDelayChance; private final Hashcode insertHashcode; private final NemesisFieldKind.Selector nemesisFieldSelector; + private final ClassLoader prewarmClassLoader; + private final Predicate prewarm; + private final byte[] bufIn = new byte[4096]; + private final ByteArrayOutputStream bufOut = new ByteArrayOutputStream(); - public InterceptClasses(ChanceSupplier monitorDelayChance, ChanceSupplier nemesisChance, NemesisFieldKind.Selector nemesisFieldSelector) + public InterceptClasses(ChanceSupplier monitorDelayChance, ChanceSupplier nemesisChance, NemesisFieldKind.Selector nemesisFieldSelector, ClassLoader prewarmClassLoader, Predicate prewarm) { - this(BYTECODE_VERSION, monitorDelayChance, nemesisChance, nemesisFieldSelector); + this(BYTECODE_VERSION, monitorDelayChance, nemesisChance, nemesisFieldSelector, prewarmClassLoader, prewarm); } - public InterceptClasses(int api, ChanceSupplier monitorDelayChance, ChanceSupplier nemesisChance, NemesisFieldKind.Selector nemesisFieldSelector) + public InterceptClasses(int api, ChanceSupplier monitorDelayChance, ChanceSupplier nemesisChance, NemesisFieldKind.Selector nemesisFieldSelector, ClassLoader prewarmClassLoader, Predicate prewarm) { this.api = api; this.nemesisChance = nemesisChance; this.monitorDelayChance = monitorDelayChance; this.insertHashcode = new Hashcode(api); this.nemesisFieldSelector = nemesisFieldSelector; + this.prewarmClassLoader = prewarmClassLoader; + this.prewarm = prewarm; } @Override public synchronized byte[] apply(String name, byte[] bytes) { - if (bytes == null) - return maybeSynthetic(name); + return transformTransitiveClosure(name, bytes, null); + } - Hashcode hashcode = insertHashCode(name); + private byte[] transformTransitiveClosure(String externalName, byte[] input, Map isolatedCache) + { + if (input == null) + return maybeSynthetic(externalName); + + String internalName = dotsToSlashes(externalName); + if (isolatedCache != null) + { + byte[] isolatedCached = isolatedCache.get(internalName); + if (isolatedCached != null) + return isolatedCached == SENTINEL ? input : isolatedCached; + } + + Cached cached = cache.get(internalName); + if (cached != null) + { + if (isolatedCache == null) + { + switch (cached.kind) + { + default: throw new AssertionError(); + case MODIFIED: + return cached.bytes; + case UNMODIFIED: + return input; + case UNSHAREABLE: + return transform(internalName, externalName, null, input, null, null); + } + } + + for (String peer : cached.uncacheablePeers) + transform(peer, slashesToDots(peer), null, cache.get(peer).bytes, isolatedCache, null); + + switch (cached.kind) + { + default: throw new AssertionError(); + case MODIFIED: + return cached.bytes; + case UNMODIFIED: + return input; + case UNSHAREABLE: + return isolatedCache.get(internalName); + } + } + + Set visited = new HashSet<>(); + visited.add(internalName); + NavigableSet load = new TreeSet<>(); + Consumer dependentTypeConsumer = type -> { + if (prewarm.test(type) && visited.add(type)) + load.add(type); + }; + + final PeerGroup peerGroup = new PeerGroup(); + byte[] result = transform(internalName, externalName, peerGroup, input, isolatedCache, dependentTypeConsumer); + for (String next = load.pollFirst(); next != null; next = load.pollFirst()) + { + // TODO (now): otherwise merge peer groups + Cached existing = cache.get(next); + if (existing == null) + transform(next, slashesToDots(next), peerGroup, read(next), isolatedCache, dependentTypeConsumer); + } + + return result; + } + + private byte[] read(String name) + { + try (InputStream in = prewarmClassLoader.getResourceAsStream(dotsToSlashes(name) + ".class")) + { + if (in == null) + throw new NoClassDefFoundError(dotsToSlashes(name) + ".class"); + + bufOut.reset(); + for (int c = in.read(bufIn) ; c >= 0 ; c = in.read(bufIn)) + bufOut.write(bufIn, 0, c); + return bufOut.toByteArray(); + } + catch (IOException e) + { + throw new NoClassDefFoundError(name); + } + } + + private byte[] transform(String internalName, String externalName, PeerGroup peerGroup, byte[] input, Map isolatedCache, Consumer dependentTypes) + { + Hashcode hashcode = insertHashCode(externalName); - name = dotsToSlashes(name); EnumSet flags = EnumSet.noneOf(Flag.class); - if (MONITORS.matcher(name).matches()) + if (MONITORS.matcher(internalName).matches()) { flags.add(Flag.MONITORS); } - if (GLOBAL_METHODS.matcher(name).matches()) + if (GLOBAL_METHODS.matcher(internalName).matches()) { flags.add(Flag.GLOBAL_METHODS); flags.add(Flag.LOCK_SUPPORT); } - if (NEMESIS.matcher(name).matches()) + if (NEMESIS.matcher(internalName).matches()) { flags.add(Flag.NEMESIS); } if (flags.isEmpty() && hashcode == null) - return bytes; - - Cached prev = cache.get(name); - if (prev != null) { - if (prev == SENTINEL) - return bytes; - return prev.cached; + cache.put(internalName, peerGroup.unmodified); + return input; } - ClassTransformer transformer = new ClassTransformer(api, name, flags, monitorDelayChance, new NemesisGenerator(api, name, nemesisChance), nemesisFieldSelector, hashcode); - transformer.readAndTransform(bytes); + ClassTransformer transformer = new ClassTransformer(api, internalName, flags, monitorDelayChance, new NemesisGenerator(api, internalName, nemesisChance), nemesisFieldSelector, hashcode, dependentTypes); + transformer.readAndTransform(input); if (!transformer.isTransformed()) { - cache.put(name, SENTINEL); - return bytes; + cache.put(internalName, peerGroup.unmodified); + return input; } - bytes = transformer.toBytes(); + byte[] output = transformer.toBytes(); if (transformer.isCacheablyTransformed()) - cache.put(name, new Cached(bytes)); + { + cache.put(internalName, new Cached(MODIFIED, output, peerGroup.uncacheablePeers)); + } + else + { + if (peerGroup != null) + { + cache.put(internalName, new Cached(UNSHAREABLE, input, peerGroup.uncacheablePeers)); + peerGroup.uncacheablePeers.add(internalName); + } + if (isolatedCache != null) + isolatedCache.put(internalName, output); + } - return bytes; + return output; } static String dotsToSlashes(String className) @@ -151,6 +283,11 @@ public class InterceptClasses implements BiFunction return dotsToSlashes(clazz.getName()); } + static String slashesToDots(String className) + { + return className.replace('/', '.'); + } + /** * Decide if we should insert our own hashCode() implementation that assigns deterministic hashes, i.e. * - If it's one of our classes @@ -160,14 +297,14 @@ public class InterceptClasses implements BiFunction * * Otherwise we either probably do not need it, or may break serialization between classloaders */ - private Hashcode insertHashCode(String name) + private Hashcode insertHashCode(String externalName) { try { - if (!name.startsWith("org.apache.cassandra")) + if (!externalName.startsWith("org.apache.cassandra")) return null; - Class sharedClass = getClass().getClassLoader().loadClass(name); + Class sharedClass = getClass().getClassLoader().loadClass(externalName); if (sharedClass.isInterface() || sharedClass.isEnum() || sharedClass.isArray() || sharedClass.isSynthetic()) return null; @@ -197,14 +334,14 @@ public class InterceptClasses implements BiFunction } catch (NoSuchFieldException e) { - if (!Throwable.class.isAssignableFrom(sharedClass) && WARNED.add(name)) + if (!Throwable.class.isAssignableFrom(sharedClass) && WARNED.add(externalName)) System.err.println("No serialVersionUID on Serializable " + sharedClass); return null; } } catch (ClassNotFoundException e) { - System.err.println("Unable to determine if should insert hashCode() for " + name); + System.err.println("Unable to determine if should insert hashCode() for " + externalName); e.printStackTrace(); } return null; @@ -216,22 +353,22 @@ public class InterceptClasses implements BiFunction static final String shadowOuterTypePrefix = shadowRootType + '$'; static final String originalOuterTypePrefix = originalRootType + '$'; - protected byte[] maybeSynthetic(String name) + protected byte[] maybeSynthetic(String externalName) { - if (!name.startsWith(shadowRootExternalType)) + if (!externalName.startsWith(shadowRootExternalType)) return null; try { - String originalType, shadowType = Utils.toInternalName(name); + String originalType, shadowType = Utils.toInternalName(externalName); if (!shadowType.startsWith(shadowOuterTypePrefix)) originalType = originalRootType; else - originalType = originalOuterTypePrefix + name.substring(shadowOuterTypePrefix.length()); + originalType = originalOuterTypePrefix + externalName.substring(shadowOuterTypePrefix.length()); EnumSet flags = EnumSet.of(Flag.GLOBAL_METHODS, Flag.MONITORS, Flag.LOCK_SUPPORT); - if (NEMESIS.matcher(name).matches()) flags.add(Flag.NEMESIS); - NemesisGenerator nemesis = new NemesisGenerator(api, name, nemesisChance); + if (NEMESIS.matcher(externalName).matches()) flags.add(Flag.NEMESIS); + NemesisGenerator nemesis = new NemesisGenerator(api, externalName, nemesisChance); ShadowingTransformer transformer; transformer = new ShadowingTransformer(InterceptClasses.BYTECODE_VERSION, @@ -245,7 +382,6 @@ public class InterceptClasses implements BiFunction { throw new UncheckedIOException(e); } - } } \ No newline at end of file diff --git a/test/simulator/asm/org/apache/cassandra/simulator/asm/MethodLogger.java b/test/simulator/asm/org/apache/cassandra/simulator/asm/MethodLogger.java index 99252d3c81..45f33dd90e 100644 --- a/test/simulator/asm/org/apache/cassandra/simulator/asm/MethodLogger.java +++ b/test/simulator/asm/org/apache/cassandra/simulator/asm/MethodLogger.java @@ -41,15 +41,15 @@ import static org.apache.cassandra.simulator.asm.MethodLogger.Level.valueOf; // TODO (config): support logging only for packages/classes matching a pattern interface MethodLogger { - static final Level LOG = valueOf(System.getProperty("cassandra.simulator.print_asm", "none").toUpperCase()); - static final Set KINDS = System.getProperty("cassandra.simulator.print_asm_opts", "").isEmpty() + static final Level LOG = valueOf(System.getProperty("cassandra.test.simulator.print_asm", "none").toUpperCase()); + static final Set KINDS = System.getProperty("cassandra.test.simulator.print_asm_opts", "").isEmpty() ? EnumSet.allOf(TransformationKind.class) - : stream(System.getProperty("cassandra.simulator.print_asm_opts", "").split(",")) + : stream(System.getProperty("cassandra.test.simulator.print_asm_opts", "").split(",")) .map(TransformationKind::valueOf) .collect(() -> EnumSet.noneOf(TransformationKind.class), Collection::add, Collection::addAll); - static final Pattern LOG_CLASSES = System.getProperty("cassandra.simulator.print_asm_classes", "").isEmpty() + static final Pattern LOG_CLASSES = System.getProperty("cassandra.test.simulator.print_asm_classes", "").isEmpty() ? null - : Pattern.compile(System.getProperty("cassandra.simulator.print_asm_classes", "")); + : Pattern.compile(System.getProperty("cassandra.test.simulator.print_asm_classes", "")); // debug the output of each class at most once static final Set LOGGED_CLASS = LOG != NONE ? Collections.newSetFromMap(new ConcurrentHashMap<>()) : null; diff --git a/test/simulator/asm/org/apache/cassandra/simulator/asm/ShadowingTransformer.java b/test/simulator/asm/org/apache/cassandra/simulator/asm/ShadowingTransformer.java index f75bdee21c..ca42f245c4 100644 --- a/test/simulator/asm/org/apache/cassandra/simulator/asm/ShadowingTransformer.java +++ b/test/simulator/asm/org/apache/cassandra/simulator/asm/ShadowingTransformer.java @@ -56,7 +56,7 @@ public class ShadowingTransformer extends ClassTransformer ShadowingTransformer(int api, String originalType, String shadowType, String originalRootType, String shadowRootType, String originalOuterTypePrefix, String shadowOuterTypePrefix, EnumSet flags, ChanceSupplier monitorDelayChance, NemesisGenerator nemesis, NemesisFieldKind.Selector nemesisFieldSelector, Hashcode insertHashcode) { - super(api, shadowType, flags, monitorDelayChance, nemesis, nemesisFieldSelector, insertHashcode); + super(api, shadowType, flags, monitorDelayChance, nemesis, nemesisFieldSelector, insertHashcode, null); this.originalType = originalType; this.originalRootType = originalRootType; this.shadowRootType = shadowRootType; diff --git a/test/simulator/asm/org/apache/cassandra/simulator/asm/ThreadLocalRandomCheckTransformer.java b/test/simulator/asm/org/apache/cassandra/simulator/asm/ThreadLocalRandomCheckTransformer.java new file mode 100644 index 0000000000..ba5cefb477 --- /dev/null +++ b/test/simulator/asm/org/apache/cassandra/simulator/asm/ThreadLocalRandomCheckTransformer.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.simulator.asm; + +import org.objectweb.asm.MethodVisitor; +import org.objectweb.asm.Opcodes; + +/** + * Handle simply thread signalling behaviours, namely monitorenter/monitorexit bytecodes to + * {@link org.apache.cassandra.simulator.systems.InterceptorOfGlobalMethods}, and LockSupport invocations to + * {@link org.apache.cassandra.simulator.systems.InterceptibleThread}. + * + * The global static methods we redirect monitors to take only one parameter (the monitor) and also return it, + * so that they have net zero effect on the stack, permitting the existing monitorenter/monitorexit instructions + * to remain where they are. LockSupport on the other hand is redirected entirely to the new method. + */ +class ThreadLocalRandomCheckTransformer extends MethodVisitor +{ + public ThreadLocalRandomCheckTransformer(int api, MethodVisitor parent) + { + super(api, parent); + } + + @Override + public void visitInsn(int opcode) + { + switch (opcode) + { + case Opcodes.IRETURN: + super.visitMethodInsn(Opcodes.INVOKESTATIC, "org/apache/cassandra/simulator/systems/InterceptorOfSystemMethods$Global", + "threadLocalRandomCheck", "(I)I", false); + break; + case Opcodes.LRETURN: + super.visitMethodInsn(Opcodes.INVOKESTATIC, "org/apache/cassandra/simulator/systems/InterceptorOfSystemMethods$Global", + "threadLocalRandomCheck", "(J)J", false); + break; + } + super.visitInsn(opcode); + } +} diff --git a/test/simulator/asm/org/apache/cassandra/simulator/asm/Utils.java b/test/simulator/asm/org/apache/cassandra/simulator/asm/Utils.java index 0422bf031c..be2ef6c5d8 100644 --- a/test/simulator/asm/org/apache/cassandra/simulator/asm/Utils.java +++ b/test/simulator/asm/org/apache/cassandra/simulator/asm/Utils.java @@ -23,6 +23,7 @@ import java.io.InputStream; import java.util.Arrays; import java.util.List; import java.util.function.BiConsumer; +import java.util.function.Consumer; import org.objectweb.asm.AnnotationVisitor; import org.objectweb.asm.Label; @@ -262,4 +263,28 @@ public class Utils }; } + public static void visitEachRefType(String descriptor, Consumer forEach) + { + Type[] argTypes = Type.getArgumentTypes(descriptor); + Type retType = Type.getReturnType(descriptor); + for (Type argType : argTypes) + visitIfRefType(argType.getDescriptor(), forEach); + visitIfRefType(retType.getDescriptor(), forEach); + } + + public static void visitIfRefType(String descriptor, Consumer forEach) + { + if (descriptor.charAt(0) != '[' && descriptor.charAt(descriptor.length() - 1) != ';') + { + if (descriptor.length() > 1) + forEach.accept(descriptor); + } + else + { + int i = 1; + while (descriptor.charAt(i) == '[') ++i; + if (descriptor.charAt(i) == 'L') + forEach.accept(descriptor.substring(i + 1, descriptor.length() - 1)); + } + } } diff --git a/test/simulator/bootstrap/org/apache/cassandra/simulator/systems/InterceptorOfSystemMethods.java b/test/simulator/bootstrap/org/apache/cassandra/simulator/systems/InterceptorOfSystemMethods.java index f66a44f25e..645bbb82cb 100644 --- a/test/simulator/bootstrap/org/apache/cassandra/simulator/systems/InterceptorOfSystemMethods.java +++ b/test/simulator/bootstrap/org/apache/cassandra/simulator/systems/InterceptorOfSystemMethods.java @@ -19,9 +19,7 @@ package org.apache.cassandra.simulator.systems; import java.lang.reflect.Field; -import java.security.SecureRandom; import java.util.UUID; -import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.LockSupport; import java.util.function.ToIntFunction; @@ -63,6 +61,11 @@ public interface InterceptorOfSystemMethods long randomSeed(); UUID randomUUID(); + void threadLocalRandomCheck(long seed); + + long nanoTime(); + long currentTimeMillis(); + @SuppressWarnings("unused") public static class Global { @@ -184,6 +187,28 @@ public interface InterceptorOfSystemMethods return methods.randomUUID(); } + public static int threadLocalRandomCheck(int seed) + { + methods.threadLocalRandomCheck(seed); + return seed; + } + + public static long threadLocalRandomCheck(long seed) + { + methods.threadLocalRandomCheck(seed); + return seed; + } + + public static long nanoTime() + { + return methods.nanoTime(); + } + + public static long currentTimeMillis() + { + return methods.currentTimeMillis(); + } + public static int identityHashCode(Object object) { return identityHashCode.applyAsInt(object); @@ -368,6 +393,23 @@ public interface InterceptorOfSystemMethods { return UUID.randomUUID(); } + + @Override + public long nanoTime() + { + return System.nanoTime(); + } + + @Override + public long currentTimeMillis() + { + return System.currentTimeMillis(); + } + + @Override + public void threadLocalRandomCheck(long seed) + { + } } } diff --git a/test/simulator/main/org/apache/cassandra/simulator/Action.java b/test/simulator/main/org/apache/cassandra/simulator/Action.java index 17cb628a47..edb661fae1 100644 --- a/test/simulator/main/org/apache/cassandra/simulator/Action.java +++ b/test/simulator/main/org/apache/cassandra/simulator/Action.java @@ -19,6 +19,8 @@ package org.apache.cassandra.simulator; import java.io.Serializable; +import java.util.ArrayDeque; +import java.util.Deque; import java.util.EnumSet; import java.util.IdentityHashMap; import java.util.List; @@ -86,7 +88,7 @@ public abstract class Action implements PriorityQueueNode WITHHOLD((char)0, false), // Mark operations as a THREAD_TIMEOUT, and parent operations as forbidding such timeouts (unless all else has failed) - NO_TIMEOUTS('n', true, null, true), TIMEOUT('t', false, NO_TIMEOUTS), + NO_THREAD_TIMEOUTS('n', true, null, true), THREAD_TIMEOUT('t', false, NO_THREAD_TIMEOUTS), /** * All children of this action should be performed in strict order wrt the parent's consequences @@ -110,10 +112,18 @@ public abstract class Action implements PriorityQueueNode /** * Must be combined with ORPHAN. Unlinks an Action from its direct parent, attaching it as a child of its - * grandparent. This is used to support streams of streams. + * grandparent. This is used to support streams of streams */ ORPHAN_TO_GRANDPARENT((char)0, false), + /** + * When we both deliver a message and timeout, the timeout may be scheduled for much later. We do not want to + * apply restrictions on later operations starting because we are waiting for a timeout to fire in this case, + * so we detach the timeout from its parent's accounting - but re-attach its children to the parent if + * still alive. Must be coincident with ORPHAN. + */ + PSEUDO_ORPHAN('p', false), + /** * Recurring tasks, that the schedule may discount when determining if has terminated */ @@ -186,6 +196,7 @@ public abstract class Action implements PriorityQueueNode public static final Modifiers INFO = Modifier.INFO.asSet(); public static final Modifiers RELIABLE = Modifier.RELIABLE.asSet(); public static final Modifiers DROP = Modifier.DROP.asSet(); + public static final Modifiers PSEUDO_ORPHAN = of(Modifier.PSEUDO_ORPHAN); public static final Modifiers STREAM = of(Modifier.STREAM); public static final Modifiers INFINITE_STREAM = of(Modifier.STREAM, DAEMON); public static final Modifiers STREAM_ITEM = of(Modifier.STREAM, ORPHAN, ORPHAN_TO_GRANDPARENT); @@ -195,15 +206,15 @@ public abstract class Action implements PriorityQueueNode public static final Modifiers START_THREAD = of(THREAD_SIGNAL); public static final Modifiers START_INFINITE_LOOP = of(ORPHAN, THREAD_SIGNAL); public static final Modifiers START_SCHEDULED_TASK = of(THREAD_SIGNAL); - public static final Modifiers START_TIMEOUT_TASK = of(Modifier.TIMEOUT, THREAD_SIGNAL); + public static final Modifiers START_TIMEOUT_TASK = of(Modifier.THREAD_TIMEOUT, THREAD_SIGNAL); public static final Modifiers START_DAEMON_TASK = of(ORPHAN, Modifier.DAEMON, THREAD_SIGNAL); public static final Modifiers WAKE_UP_THREAD = of(THREAD_SIGNAL, WAKEUP); public static final Modifiers STRICT = of(STRICT_CHILD_ORDER); - public static final Modifiers NO_TIMEOUTS = Modifier.NO_TIMEOUTS.asSet(); + public static final Modifiers NO_TIMEOUTS = Modifier.NO_THREAD_TIMEOUTS.asSet(); - public static final Modifiers RELIABLE_NO_TIMEOUTS = of(Modifier.NO_TIMEOUTS, Modifier.RELIABLE); + public static final Modifiers RELIABLE_NO_TIMEOUTS = of(Modifier.NO_THREAD_TIMEOUTS, Modifier.RELIABLE); public static final Modifiers DISPLAY_ORIGIN = of(Modifier.DISPLAY_ORIGIN); public static Modifiers of() @@ -339,7 +350,7 @@ public abstract class Action implements PriorityQueueNode private List listeners; /** The immediate parent, and furthest ancestor of this Action */ - protected Action parent, origin = this; + protected Action parent, origin = this, pseudoParent; /** The number of direct consequences of this action that have not transitively terminated */ private int childCount; @@ -415,6 +426,10 @@ public abstract class Action implements PriorityQueueNode { return phase.compareTo(FINISHED) >= 0; } + public boolean isCancelled() + { + return phase.compareTo(CANCELLED) >= 0; + } public boolean isInvalidated() { return phase.compareTo(INVALIDATED) >= 0; @@ -575,41 +590,49 @@ public abstract class Action implements PriorityQueueNode Throwables.maybeFail(fail); } - boolean withhold = false; - int orphanCount = 0; + boolean isParentPseudoOrphan = is(PSEUDO_ORPHAN); + boolean withheld = false; for (int i = 0 ; i < consequences.size() ; ++i) { Action child = consequences.get(i); if (child.is(ORPHAN)) { - Preconditions.checkState(!child.is(WITHHOLD)); - ++orphanCount; if (parent != null && child.is(ORPHAN_TO_GRANDPARENT)) { ++parent.childCount; parent.registerChild(child); } + else if (child.is(PSEUDO_ORPHAN)) + { + child.inherit(transitive); + registerPseudoOrphan(child); + assert !child.is(WITHHOLD); + } } else { - child.inherit(transitive); + Action parent; + if (isParentPseudoOrphan && pseudoParent != null && pseudoParent.childCount > 0) + parent = pseudoParent; + else + parent = this; + + child.inherit(parent.transitive); if (child.is(WITHHOLD)) { // this could be supported in principle by applying the ordering here, but it would be // some work to ensure it doesn't lead to deadlocks so for now just assert we don't use it - Preconditions.checkState(!is(STRICT_CHILD_ORDER) && !is(STRICT_CHILD_OF_PARENT_ORDER)); - withhold = true; - addWithheld(child); + Preconditions.checkState(!parent.is(STRICT_CHILD_ORDER) && !parent.is(STRICT_CHILD_OF_PARENT_ORDER)); + withheld = true; + parent.addWithheld(child); } - registerChild(child); + parent.registerChild(child); + parent.childCount++; } } - int addChildCount = consequences.size() - orphanCount; - childCount += addChildCount; - - if (!withhold) + if (!withheld) return consequences; return consequences.filter(child -> !child.is(WITHHOLD)); @@ -620,9 +643,22 @@ public abstract class Action implements PriorityQueueNode { assert child.parent == null; child.parent = this; + registerChildOrigin(child); + if (DEBUG && !register(child, CHILD)) throw new AssertionError(); + } + + private void registerPseudoOrphan(Action child) + { + assert child.parent == null; + assert child.pseudoParent == null; + child.pseudoParent = this; + registerChildOrigin(child); + } + + private void registerChildOrigin(Action child) + { if (is(Modifier.DISPLAY_ORIGIN)) child.origin = this; else if (origin != this) child.origin = origin; - if (DEBUG && !register(child, CHILD)) throw new AssertionError(); } private boolean register(Object object, RegisteredType type) @@ -791,22 +827,28 @@ public abstract class Action implements PriorityQueueNode void schedule(SimulatedTime time, FutureActionScheduler future) { - setPriority(scheduler.priority()); - if (deadline == 0) deadline = time.nanoTime(); - if (is(THREAD_SIGNAL)) - deadline += future.schedulerDelayNanos(); + setPriority(time, scheduler.priority()); + if (is(THREAD_SIGNAL) || deadline == 0) + { + long newDeadline = deadline == 0 ? time.nanoTime() : deadline; + newDeadline += future.schedulerDelayNanos(); + deadline = newDeadline; + time.onTimeEvent("ResetDeadline", newDeadline); + } } - public void setDeadline(long deadlineNanos) + public void setDeadline(SimulatedTime time, long deadlineNanos) { Preconditions.checkState(deadline == 0); Preconditions.checkArgument(deadlineNanos >= deadline); deadline = deadlineNanos; + time.onTimeEvent("SetDeadline", deadlineNanos); } - void setPriority(double priority) + public void setPriority(SimulatedTime time, double priority) { this.priority = priority; + time.onTimeEvent("SetPriority", Double.doubleToLongBits(priority)); } public long deadline() @@ -890,17 +932,62 @@ public abstract class Action implements PriorityQueueNode return describeModifiers() + description() + (origin != this ? " for " + origin : ""); } - public String describeCurrentState() + public String toReconcileString() { - return describeCurrentState(new StringBuilder(), "").toString(); + return this + " at [" + deadline + ',' + priority + ']'; } - private StringBuilder describeCurrentState(StringBuilder sb, String prefix) + private static class StackElement { - if (!prefix.isEmpty()) + final Action action; + final Deque children; + + private StackElement(Action action) { - sb.append(prefix); + this.action = action; + this.children = new ArrayDeque<>(action.childCount); + for (Map.Entry e : action.registered.entrySet()) + { + if (e.getValue() == CHILD) + children.add((Action) e.getKey()); + } } + } + + public String describeCurrentState() + { + StringBuilder sb = new StringBuilder(); + Deque stack = new ArrayDeque<>(); + appendCurrentState(sb); + + stack.push(new StackElement(this)); + while (!stack.isEmpty()) + { + StackElement last = stack.peek(); + if (last.children.isEmpty()) + { + stack.pop(); + } + else + { + Action child = last.children.pop(); + sb.append('\n'); + appendPrefix(stack.size(), sb); + child.appendCurrentState(sb); + stack.push(new StackElement(child)); + } + } + return sb.toString(); + } + + private static void appendPrefix(int count, StringBuilder sb) + { + while (--count >= 0) + sb.append(" |"); + } + + private void appendCurrentState(StringBuilder sb) + { if (!isStarted()) sb.append("NOT_STARTED "); else if (!isFinished()) sb.append("NOT_FINISHED "); if (childCount > 0) @@ -915,15 +1002,6 @@ public abstract class Action implements PriorityQueueNode sb.append(": "); } sb.append(description()); - registered.entrySet().stream() - .filter(e -> e.getValue() == CHILD) - .map(e -> (Action) e.getKey()) - .forEach(a -> { - sb.append('\n'); - a.describeCurrentState(sb, prefix + " |"); - }); - - return sb; } } \ No newline at end of file diff --git a/test/simulator/main/org/apache/cassandra/simulator/ActionList.java b/test/simulator/main/org/apache/cassandra/simulator/ActionList.java index 5052e4356a..64474e6184 100644 --- a/test/simulator/main/org/apache/cassandra/simulator/ActionList.java +++ b/test/simulator/main/org/apache/cassandra/simulator/ActionList.java @@ -25,6 +25,7 @@ import java.util.Iterator; import java.util.function.Consumer; import java.util.function.Function; import java.util.function.Predicate; +import java.util.stream.Collectors; import java.util.stream.Stream; import com.google.common.collect.Iterators; @@ -138,5 +139,10 @@ public class ActionList extends AbstractCollection { return Arrays.toString(actions); } + + public String toReconcileString() + { + return Arrays.stream(actions).map(Action::toReconcileString).collect(Collectors.joining(",", "[", "]")); + } } diff --git a/test/simulator/main/org/apache/cassandra/simulator/ActionPlan.java b/test/simulator/main/org/apache/cassandra/simulator/ActionPlan.java index b9e9d2dd43..1813de00f2 100644 --- a/test/simulator/main/org/apache/cassandra/simulator/ActionPlan.java +++ b/test/simulator/main/org/apache/cassandra/simulator/ActionPlan.java @@ -29,8 +29,7 @@ import org.apache.cassandra.utils.CloseableIterator; import static java.util.Collections.emptyList; import static java.util.Collections.singletonList; -import static org.apache.cassandra.simulator.ActionSchedule.Mode.STREAM_LIMITED; -import static org.apache.cassandra.simulator.ActionSchedule.Mode.TIME_LIMITED; +import static org.apache.cassandra.simulator.ActionSchedule.Mode.FINITE; import static org.apache.cassandra.simulator.ActionSchedule.Mode.UNLIMITED; public class ActionPlan @@ -62,12 +61,12 @@ public class ActionPlan this.post = post; } - public CloseableIterator iterator(long runForNanos, LongSupplier schedulerJitter, SimulatedTime time, RunnableActionScheduler preAndPostScheduler, RunnableActionScheduler mainScheduler, FutureActionScheduler futureScheduler) + public CloseableIterator iterator(ActionSchedule.Mode mode, long runForNanos, LongSupplier schedulerJitter, SimulatedTime time, RunnableActionScheduler runnableScheduler, FutureActionScheduler futureScheduler) { - return new ActionSchedule(time, futureScheduler, schedulerJitter, - new Work(UNLIMITED, preAndPostScheduler, singletonList(pre.setStrictlySequential())), - new Work(runForNanos > 0 ? TIME_LIMITED : STREAM_LIMITED, runForNanos, mainScheduler, interleave), - new Work(UNLIMITED, preAndPostScheduler, singletonList(post.setStrictlySequential()))); + return new ActionSchedule(time, futureScheduler, schedulerJitter, runnableScheduler, + new Work(UNLIMITED, singletonList(pre.setStrictlySequential())), + new Work(mode, runForNanos, interleave), + new Work(FINITE, singletonList(post.setStrictlySequential()))); } public static ActionPlan interleave(List interleave) diff --git a/test/simulator/main/org/apache/cassandra/simulator/ActionSchedule.java b/test/simulator/main/org/apache/cassandra/simulator/ActionSchedule.java index 59d018082d..18fc877824 100644 --- a/test/simulator/main/org/apache/cassandra/simulator/ActionSchedule.java +++ b/test/simulator/main/org/apache/cassandra/simulator/ActionSchedule.java @@ -26,11 +26,11 @@ import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.NoSuchElementException; +import java.util.function.LongConsumer; import java.util.function.LongSupplier; import java.util.stream.Stream; import com.google.common.base.Preconditions; -import org.apache.commons.lang3.tuple.Pair; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -68,43 +68,65 @@ import static org.apache.cassandra.simulator.SimulatorUtils.dumpStackTraces; * all descendants have executed (with the aim of it ordinarily being invalidated before this happens), and this * is not imposed here because it would be more complicated to manage. */ -public class ActionSchedule implements CloseableIterator +public class ActionSchedule implements CloseableIterator, LongConsumer { private static final Logger logger = LoggerFactory.getLogger(ActionList.class); - public enum Mode { TIME_LIMITED, STREAM_LIMITED, UNLIMITED } + public enum Mode { TIME_LIMITED, STREAM_LIMITED, TIME_AND_STREAM_LIMITED, FINITE, UNLIMITED } public static class Work { final Mode mode; final long runForNanos; - final RunnableActionScheduler runnableScheduler; final List actors; - public Work(Mode mode, RunnableActionScheduler runnableScheduler, List actors) + public Work(Mode mode, List actors) { - this(mode, -1, runnableScheduler, actors); + this(mode, -1, actors); Preconditions.checkArgument(mode != TIME_LIMITED); } - public Work(long runForNanos, RunnableActionScheduler runnableScheduler, List actors) + public Work(long runForNanos, List actors) { - this(TIME_LIMITED, runForNanos, runnableScheduler, actors); + this(TIME_LIMITED, runForNanos, actors); Preconditions.checkArgument(runForNanos > 0); } - public Work(Mode mode, long runForNanos, RunnableActionScheduler runnableScheduler, List actors) + public Work(Mode mode, long runForNanos, List actors) { this.mode = mode; this.runForNanos = runForNanos; this.actors = actors; - this.runnableScheduler = runnableScheduler; + } + } + + public static class ReconcileItem + { + final long start, end; + final Action performed; + final ActionList result; + + public ReconcileItem(long start, long end, Action performed, ActionList result) + { + this.start = start; + this.end = end; + this.performed = performed; + this.result = result; + } + + public String toString() + { + return "run:" + performed.toReconcileString() + "; next:" + result.toReconcileString() + + "; between [" + start + ',' + end + ']'; } } final SimulatedTime time; final FutureActionScheduler scheduler; + final RunnableActionScheduler runnableScheduler; final LongSupplier schedulerJitter; // we will prioritise all actions scheduled to run within this period of the current oldest action + long currentJitter, currentJitterUntil; + // Action flow is: // perform() -> [withheld] // -> consequences @@ -139,14 +161,16 @@ public class ActionSchedule implements CloseableIterator private final Iterator moreWork; - public ActionSchedule(SimulatedTime time, FutureActionScheduler futureScheduler, LongSupplier schedulerJitter, Work ... moreWork) + public ActionSchedule(SimulatedTime time, FutureActionScheduler futureScheduler, LongSupplier schedulerJitter, RunnableActionScheduler runnableScheduler, Work... moreWork) { - this(time, futureScheduler, schedulerJitter, Arrays.asList(moreWork).iterator()); + this(time, futureScheduler, runnableScheduler, schedulerJitter, Arrays.asList(moreWork).iterator()); } - public ActionSchedule(SimulatedTime time, FutureActionScheduler futureScheduler, LongSupplier schedulerJitter, Iterator moreWork) + public ActionSchedule(SimulatedTime time, FutureActionScheduler futureScheduler, RunnableActionScheduler runnableScheduler, LongSupplier schedulerJitter, Iterator moreWork) { this.time = time; + this.runnableScheduler = runnableScheduler; + this.time.onDiscontinuity(this); this.scheduler = futureScheduler; this.schedulerJitter = schedulerJitter; this.moreWork = moreWork; @@ -164,6 +188,13 @@ public class ActionSchedule implements CloseableIterator switch (mode) { default: throw new AssertionError(); + case TIME_AND_STREAM_LIMITED: + if ((activeFiniteStreamCount == 0 || time.nanoTime() >= runUntilNanos) && action.is(DAEMON)) + { + action.cancel(); + return; + } + break; case TIME_LIMITED: if (time.nanoTime() >= runUntilNanos && (action.is(DAEMON) || action.is(STREAM))) { @@ -187,6 +218,9 @@ public class ActionSchedule implements CloseableIterator return; } break; + case FINITE: + if (action.is(STREAM)) throw new IllegalStateException(); + break; } action.advanceTo(READY_TO_SCHEDULE); advance(action); @@ -266,13 +300,12 @@ public class ActionSchedule implements CloseableIterator } else { - logger.error("Simulation failed to make progress. Run with assertions enabled to see the blocked task graph. Blocked tasks:"); + logger.error("Simulation failed to make progress. Run with -Dcassandra.test.simulator.debug=true to see the blocked task graph. Blocked tasks:"); actions = sequences.values() .stream() .filter(s -> s.on instanceof OrderOnId) .map(s -> ((OrderOnId) s.on).id) .flatMap(s -> s instanceof ActionList ? ((ActionList) s).stream() : Stream.empty()); - logger.error("Run with assertions enabled to see the blocked task graph."); } actions.filter(Action::isStarted) @@ -310,7 +343,7 @@ public class ActionSchedule implements CloseableIterator pendingDaemonWave = null; } } - work.actors.forEach(work.runnableScheduler::attachTo); + work.actors.forEach(runnableScheduler::attachTo); work.actors.forEach(a -> a.forEach(Action::setConsequence)); work.actors.forEach(this::add); return true; @@ -318,10 +351,16 @@ public class ActionSchedule implements CloseableIterator public Object next() { + long now = time.nanoTime(); + if (now >= currentJitterUntil) + { + currentJitter = schedulerJitter.getAsLong(); + currentJitterUntil = now + currentJitter + schedulerJitter.getAsLong(); + } if (!scheduled.isEmpty()) { - long scheduleUntil = (runnableByDeadline.isEmpty() ? time.nanoTime() : runnableByDeadline.peek().deadline()) - + schedulerJitter.getAsLong(); + long scheduleUntil = Math.min((runnableByDeadline.isEmpty() ? now : runnableByDeadline.peek().deadline()) + + currentJitter, currentJitterUntil); while (!scheduled.isEmpty() && (runnable.isEmpty() || scheduled.peek().deadline() <= scheduleUntil)) advance(scheduled.poll()); @@ -341,7 +380,8 @@ public class ActionSchedule implements CloseableIterator if (perform.is(STREAM) && !perform.is(DAEMON)) --activeFiniteStreamCount; - return Pair.of(perform, consequences); + long end = time.nanoTime(); + return new ReconcileItem(now, end, perform, consequences); } private void maybeScheduleDaemons(Action perform) @@ -387,4 +427,12 @@ public class ActionSchedule implements CloseableIterator sequences.clear(); Throwables.maybeFail(fail); } + + @Override + public void accept(long discontinuity) + { + if (runUntilNanos > 0) + runUntilNanos += discontinuity; + } + } diff --git a/test/simulator/main/org/apache/cassandra/simulator/ClusterSimulation.java b/test/simulator/main/org/apache/cassandra/simulator/ClusterSimulation.java index 6882390c80..a1f68621df 100644 --- a/test/simulator/main/org/apache/cassandra/simulator/ClusterSimulation.java +++ b/test/simulator/main/org/apache/cassandra/simulator/ClusterSimulation.java @@ -22,11 +22,17 @@ import java.io.IOException; import java.lang.reflect.Field; import java.lang.reflect.Modifier; import java.nio.file.FileSystem; +import java.util.ArrayList; import java.util.EnumMap; +import java.util.List; import java.util.Map; import java.util.TreeMap; +import java.util.concurrent.Callable; +import java.util.concurrent.CopyOnWriteArrayList; import java.util.function.Consumer; import java.util.function.IntSupplier; +import java.util.function.LongConsumer; +import java.util.function.Predicate; import java.util.function.Supplier; import com.google.common.jimfs.Configuration; @@ -42,22 +48,28 @@ import org.apache.cassandra.distributed.api.IInstance; import org.apache.cassandra.distributed.api.IInstanceConfig; import org.apache.cassandra.distributed.api.IInstanceInitializer; import org.apache.cassandra.distributed.api.IInvokableInstance; +import org.apache.cassandra.distributed.api.IIsolatedExecutor; import org.apache.cassandra.distributed.api.IIsolatedExecutor.SerializableBiConsumer; import org.apache.cassandra.distributed.api.IIsolatedExecutor.SerializableConsumer; +import org.apache.cassandra.distributed.api.IIsolatedExecutor.SerializableRunnable; import org.apache.cassandra.distributed.impl.DirectStreamingConnectionFactory; import org.apache.cassandra.distributed.impl.IsolatedExecutor; import org.apache.cassandra.io.compress.LZ4Compressor; import org.apache.cassandra.service.paxos.BallotGenerator; +import org.apache.cassandra.service.paxos.PaxosPrepare; import org.apache.cassandra.simulator.RandomSource.Choices; +import org.apache.cassandra.simulator.asm.InterceptAsClassTransformer; import org.apache.cassandra.simulator.asm.NemesisFieldSelectors; import org.apache.cassandra.simulator.cluster.ClusterActions; import org.apache.cassandra.simulator.cluster.ClusterActions.TopologyChange; -import org.apache.cassandra.simulator.debug.Capture; -import org.apache.cassandra.simulator.asm.InterceptClasses; import org.apache.cassandra.simulator.systems.Failures; +import org.apache.cassandra.simulator.systems.InterceptedWait.CaptureSites.Capture; import org.apache.cassandra.simulator.systems.InterceptibleThread; +import org.apache.cassandra.simulator.systems.InterceptingGlobalMethods; +import org.apache.cassandra.simulator.systems.InterceptingGlobalMethods.ThreadLocalRandomCheck; import org.apache.cassandra.simulator.systems.InterceptorOfGlobalMethods; import org.apache.cassandra.simulator.systems.InterceptingExecutorFactory; +import org.apache.cassandra.simulator.systems.InterceptorOfGlobalMethods.IfInterceptibleThread; import org.apache.cassandra.simulator.systems.NetworkConfig; import org.apache.cassandra.simulator.systems.NetworkConfig.PhaseConfig; import org.apache.cassandra.simulator.systems.SchedulerConfig; @@ -69,12 +81,12 @@ import org.apache.cassandra.simulator.systems.SimulatedFailureDetector; import org.apache.cassandra.simulator.systems.SimulatedMessageDelivery; import org.apache.cassandra.simulator.systems.SimulatedSnitch; import org.apache.cassandra.simulator.systems.SimulatedTime; -import org.apache.cassandra.simulator.systems.SimulatedWaits; import org.apache.cassandra.simulator.utils.ChanceRange; import org.apache.cassandra.simulator.utils.IntRange; import org.apache.cassandra.simulator.utils.KindOfSequence; import org.apache.cassandra.simulator.utils.LongRange; import org.apache.cassandra.utils.Clock; +import org.apache.cassandra.utils.Closeable; import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.Throwables; import org.apache.cassandra.utils.concurrent.Ref; @@ -90,6 +102,7 @@ import static java.util.concurrent.TimeUnit.NANOSECONDS; import static java.util.concurrent.TimeUnit.SECONDS; import static org.apache.cassandra.distributed.impl.AbstractCluster.getSharedClassPredicate; import static org.apache.cassandra.simulator.SimulatorUtils.failWithOOM; +import static org.apache.cassandra.simulator.systems.NonInterceptible.Permit.REQUIRED; import static org.apache.cassandra.utils.Shared.Scope.ANY; import static org.apache.cassandra.utils.Shared.Scope.SIMULATION; @@ -131,6 +144,7 @@ public class ClusterSimulation implements AutoCloseable protected IntRange nodeCount = new IntRange(4, 16), dcCount = new IntRange(1, 2), primaryKeySeconds = new IntRange(5, 30), withinKeyConcurrency = new IntRange(2, 5); protected TopologyChange[] topologyChanges = TopologyChange.values(); + protected int topologyChangeLimit = -1; protected int primaryKeyCount; protected int secondsToSimulate; @@ -143,8 +157,8 @@ public class ClusterSimulation implements AutoCloseable networkFlakyChance = new ChanceRange(randomSource -> randomSource.qlog2uniformFloat(4), 0.0f, 0.1f), monitorDelayChance = new ChanceRange(randomSource -> randomSource.qlog2uniformFloat(4), 0.01f, 0.1f), schedulerDelayChance = new ChanceRange(randomSource -> randomSource.qlog2uniformFloat(4), 0.01f, 0.1f), - timeoutChance = new ChanceRange(randomSource -> randomSource.qlog2uniformFloat(4), 0.01f, 0.1f), - readChance = new ChanceRange(RandomSource::uniformFloat, 0.05f, 0.95f), + timeoutChance = new ChanceRange(randomSource -> randomSource.qlog2uniformFloat(4), 0.01f, 0.1f), + readChance = new ChanceRange(RandomSource::uniformFloat, 0.05f, 0.95f), nemesisChance = new ChanceRange(randomSource -> randomSource.qlog2uniformFloat(4), 0.001f, 0.01f); protected LongRange normalNetworkLatencyNanos = new LongRange(1, 2, MILLISECONDS, NANOSECONDS), @@ -171,6 +185,8 @@ public class ClusterSimulation implements AutoCloseable protected Debug debug = new Debug(); protected Capture capture = new Capture(false, false, false); protected HeapPool.Logged.Listener memoryListener; + protected SimulatedTime.Listener timeListener = (i1, i2) -> {}; + protected LongConsumer onThreadLocalRandomCheck; public Debug debug() { @@ -260,6 +276,12 @@ public class ClusterSimulation implements AutoCloseable return this; } + public Builder topologyChangeLimit(int topologyChangeLimit) + { + this.topologyChangeLimit = topologyChangeLimit; + return this; + } + public int primaryKeyCount() { return primaryKeyCount; @@ -470,6 +492,12 @@ public class ClusterSimulation implements AutoCloseable return this; } + public Builder timeListener(SimulatedTime.Listener timeListener) + { + this.timeListener = timeListener; + return this; + } + public Builder capture(Capture capture) { this.capture = capture; @@ -481,6 +509,12 @@ public class ClusterSimulation implements AutoCloseable return capture; } + public Builder onThreadLocalRandomCheck(LongConsumer runnable) + { + this.onThreadLocalRandomCheck = runnable; + return this; + } + public abstract ClusterSimulation create(long seed) throws IOException; } @@ -575,7 +609,9 @@ public class ClusterSimulation implements AutoCloseable public final Cluster cluster; public final S simulation; private final FileSystem jimfs; - protected final Map factories = new TreeMap<>(); + protected final Map> onUnexpectedShutdown = new TreeMap<>(); + protected final List> onShutdown = new CopyOnWriteArrayList<>(); + protected final ThreadLocalRandomCheck threadLocalRandomCheck; public ClusterSimulation(RandomSource random, long seed, int uniqueNum, Builder builder, @@ -588,7 +624,6 @@ public class ClusterSimulation implements AutoCloseable .build()); final SimulatedMessageDelivery delivery; - final SimulatedWaits waits; final SimulatedExecution execution; final SimulatedBallots ballots; final SimulatedSnitch snitch; @@ -596,7 +631,7 @@ public class ClusterSimulation implements AutoCloseable final SimulatedFailureDetector failureDetector; int numOfNodes = builder.nodeCount.select(random); - int numOfDcs = builder.dcCount.select(random, 0, numOfNodes / 3); + int numOfDcs = builder.dcCount.select(random, 0, numOfNodes / 4); int[] numInDcs = new int[numOfDcs]; int[] nodeToDc = new int[numOfNodes]; @@ -618,24 +653,27 @@ public class ClusterSimulation implements AutoCloseable } snitch = new SimulatedSnitch(nodeToDc, numInDcs); - waits = new SimulatedWaits(random); - waits.captureStackTraces(builder.capture.waitSites, builder.capture.wakeSites, builder.capture.nowSites); execution = new SimulatedExecution(); KindOfSequence kindOfDriftSequence = Choices.uniform(KindOfSequence.values()).choose(random); KindOfSequence kindOfDiscontinuitySequence = Choices.uniform(KindOfSequence.values()).choose(random); - time = new SimulatedTime(random, 1577836800000L /*Jan 1st UTC*/, builder.clockDriftNanos, kindOfDriftSequence, - kindOfDiscontinuitySequence.period(builder.clockDiscontinuitIntervalNanos, random)); + time = new SimulatedTime(numOfNodes, random, 1577836800000L /*Jan 1st UTC*/, builder.clockDriftNanos, kindOfDriftSequence, + kindOfDiscontinuitySequence.period(builder.clockDiscontinuitIntervalNanos, random), + builder.timeListener); ballots = new SimulatedBallots(random, () -> { long max = random.uniform(2, 16); return () -> random.uniform(1, max); }); + Predicate sharedClassPredicate = getSharedClassPredicate(ISOLATE, SHARE, ANY, SIMULATION); + InterceptAsClassTransformer interceptClasses = new InterceptAsClassTransformer(builder.monitorDelayChance.asSupplier(random), builder.nemesisChance.asSupplier(random), NemesisFieldSelectors.get(), ClassLoader.getSystemClassLoader(), sharedClassPredicate.negate()); + threadLocalRandomCheck = new ThreadLocalRandomCheck(builder.onThreadLocalRandomCheck); + Failures failures = new Failures(); ThreadAllocator threadAllocator = new ThreadAllocator(random, builder.threadCount, numOfNodes); cluster = snitch.setup(Cluster.build(numOfNodes) .withRoot(jimfs.getPath("/cassandra")) - .withSharedClasses(getSharedClassPredicate(ISOLATE, SHARE, ANY, SIMULATION)) + .withSharedClasses(sharedClassPredicate) .withConfig(config -> configUpdater.accept(threadAllocator.update(config .with(Feature.BLANK_GOSSIP) .set("read_request_timeout", String.format("%dms", NANOSECONDS.toMillis(builder.readTimeoutNanos))) @@ -655,56 +693,85 @@ public class ClusterSimulation implements AutoCloseable @Override public void initialise(ClassLoader classLoader, ThreadGroup threadGroup, int num, int generation) { - InterceptorOfGlobalMethods interceptorOfGlobalMethods = waits.interceptGlobalMethods(classLoader); + List onShutdown = new ArrayList<>(); + InterceptorOfGlobalMethods interceptorOfGlobalMethods = IsolatedExecutor.transferAdhoc((IIsolatedExecutor.SerializableQuadFunction, RandomSource, InterceptorOfGlobalMethods>) InterceptingGlobalMethods::new, classLoader) + .apply(builder.capture, builder.onThreadLocalRandomCheck, failures, random); + onShutdown.add(interceptorOfGlobalMethods); + InterceptingExecutorFactory factory = execution.factory(interceptorOfGlobalMethods, classLoader, threadGroup); IsolatedExecutor.transferAdhoc((SerializableConsumer) ExecutorFactory.Global::unsafeSet, classLoader) .accept(factory); + onShutdown.add(factory); + IsolatedExecutor.transferAdhoc((SerializableBiConsumer) InterceptorOfGlobalMethods.Global::unsafeSet, classLoader) .accept(interceptorOfGlobalMethods, () -> { if (InterceptibleThread.isDeterministic()) throw failWithOOM(); return random.uniform(Integer.MIN_VALUE, Integer.MAX_VALUE); }); - time.setup(classLoader); - factories.put(num, factory); + onShutdown.add(IsolatedExecutor.transferAdhoc((SerializableRunnable)InterceptorOfGlobalMethods.Global::unsafeReset, classLoader)::run); + onShutdown.add(time.setup(num, classLoader)); + + onUnexpectedShutdown.put(num, onShutdown); } @Override public void beforeStartup(IInstance i) { - if (builder.memoryListener != null) - ((IInvokableInstance) i).unsafeAcceptOnThisThread(HeapPool.Logged::setListener, builder.memoryListener); ((IInvokableInstance) i).unsafeAcceptOnThisThread(FBUtilities::setAvailableProcessors, i.config().getInt("available_processors")); + ((IInvokableInstance) i).unsafeAcceptOnThisThread(IfInterceptibleThread::setThreadLocalRandomCheck, (LongConsumer) threadLocalRandomCheck); + + int num = i.config().num(); + if (builder.memoryListener != null) + { + ((IInvokableInstance) i).unsafeAcceptOnThisThread(HeapPool.Logged::setListener, builder.memoryListener); + onUnexpectedShutdown.get(num).add(() -> ((IInvokableInstance) i).unsafeAcceptOnThisThread(HeapPool.Logged::setListener, (ignore1, ignore2) -> {})); + } + + ((IInvokableInstance) i).unsafeAcceptOnThisThread(PaxosPrepare::setOnLinearizabilityViolation, SimulatorUtils::failWithOOM); + onUnexpectedShutdown.get(num).add(() -> ((IInvokableInstance) i).unsafeRunOnThisThread(() -> PaxosPrepare.setOnLinearizabilityViolation(null))); } @Override public void afterStartup(IInstance i) { + int num = i.config().num(); ((IInvokableInstance) i).unsafeAcceptOnThisThread(BallotGenerator.Global::unsafeSet, (BallotGenerator) ballots.get()); - ((IInvokableInstance) i).unsafeAcceptOnThisThread((SerializableConsumer) debug -> BufferPools.forChunkCache().debug(null, debug), failures); - ((IInvokableInstance) i).unsafeAcceptOnThisThread((SerializableConsumer) debug -> BufferPools.forNetworking().debug(null, debug), failures); - ((IInvokableInstance) i).unsafeAcceptOnThisThread((SerializableConsumer) Ref::setOnLeak, failures); - } + onUnexpectedShutdown.get(num).add(() -> ((IInvokableInstance) i).unsafeRunOnThisThread(() -> BallotGenerator.Global.unsafeSet(new BallotGenerator.Default()))); - }).withClassTransformer(new InterceptClasses(builder.monitorDelayChance.asSupplier(random), builder.nemesisChance.asSupplier(random), NemesisFieldSelectors.get())::apply) + ((IInvokableInstance) i).unsafeAcceptOnThisThread((SerializableConsumer) debug -> BufferPools.forChunkCache().debug(null, debug), failures); + onUnexpectedShutdown.get(num).add(() -> ((IInvokableInstance) i).unsafeRunOnThisThread(() -> BufferPools.forChunkCache().debug(null, null))); + + ((IInvokableInstance) i).unsafeAcceptOnThisThread((SerializableConsumer) debug -> BufferPools.forNetworking().debug(null, debug), failures); + onUnexpectedShutdown.get(num).add(() -> ((IInvokableInstance) i).unsafeRunOnThisThread(() -> BufferPools.forNetworking().debug(null, null))); + + ((IInvokableInstance) i).unsafeAcceptOnThisThread((SerializableConsumer) Ref::setOnLeak, failures); + onUnexpectedShutdown.get(num).add(() -> ((IInvokableInstance) i).unsafeRunOnThisThread(() -> Ref.setOnLeak(null))); + } + }).withClassTransformer(interceptClasses) + .withShutdownExecutor((name, classLoader, shuttingDown, call) -> { + onShutdown.add(call); + return null; + }) ).createWithoutStarting(); + IfInterceptibleThread.setThreadLocalRandomCheck(threadLocalRandomCheck); snitch.setup(cluster); DirectStreamingConnectionFactory.setup(cluster); delivery = new SimulatedMessageDelivery(cluster); failureDetector = new SimulatedFailureDetector(cluster); SimulatedFutureActionScheduler futureActionScheduler = builder.futureActionScheduler(numOfNodes, time, random); - simulated = new SimulatedSystems(random, time, waits, delivery, execution, ballots, failureDetector, snitch, futureActionScheduler, builder.debug, failures); + simulated = new SimulatedSystems(random, time, delivery, execution, ballots, failureDetector, snitch, futureActionScheduler, builder.debug, failures); simulated.register(futureActionScheduler); RunnableActionScheduler scheduler = builder.schedulerFactory.create(random); - ClusterActions.Options options = new ClusterActions.Options(Choices.uniform(KindOfSequence.values()).choose(random).period(builder.topologyChangeIntervalNanos, random), + ClusterActions.Options options = new ClusterActions.Options(builder.topologyChangeLimit, Choices.uniform(KindOfSequence.values()).choose(random).period(builder.topologyChangeIntervalNanos, random), Choices.random(random, builder.topologyChanges), minRf, initialRf, maxRf, null); simulation = factory.create(simulated, scheduler, cluster, options); } - public void close() throws IOException + public synchronized void close() throws IOException { // Re-enable time on shutdown try @@ -723,10 +790,19 @@ public class ClusterSimulation implements AutoCloseable throw new RuntimeException(e); } - simulated.waits.stop(); + threadLocalRandomCheck.stop(); simulated.execution.forceStop(); - factories.values().forEach(InterceptingExecutorFactory::close); + SimulatedTime.Global.disable(); + Throwable fail = null; + for (int num = 1 ; num <= cluster.size() ; ++num) + { + if (!cluster.get(num).isShutdown()) + { + fail = Throwables.close(fail, onUnexpectedShutdown.get(num)); + } + } + try { simulation.close(); @@ -735,6 +811,7 @@ public class ClusterSimulation implements AutoCloseable { fail = t; } + try { cluster.close(); @@ -743,6 +820,17 @@ public class ClusterSimulation implements AutoCloseable { fail = Throwables.merge(fail, t); } + for (Callable call : onShutdown) + { + try + { + call.call(); + } + catch (Throwable t) + { + fail = Throwables.merge(fail, t); + } + } Throwables.maybeFail(fail, IOException.class); } } diff --git a/test/simulator/main/org/apache/cassandra/simulator/Debug.java b/test/simulator/main/org/apache/cassandra/simulator/Debug.java index a03d3040c7..8afdd3ed36 100644 --- a/test/simulator/main/org/apache/cassandra/simulator/Debug.java +++ b/test/simulator/main/org/apache/cassandra/simulator/Debug.java @@ -163,7 +163,7 @@ public class Debug case CONSEQUENCES: case ALL: listener = adapt.apply(recursive(new LogOne(time, true))); break; } } - else + else if (keyspace != null) { Consumer debug; switch (info) @@ -183,6 +183,7 @@ public class Debug case ALL: listener = recursive(runAfter(ignoreLogEvents(debug))); break; } } + else continue; listeners.add(listener); } diff --git a/test/simulator/main/org/apache/cassandra/simulator/FutureActionScheduler.java b/test/simulator/main/org/apache/cassandra/simulator/FutureActionScheduler.java index e0ccc563e0..67d601d63b 100644 --- a/test/simulator/main/org/apache/cassandra/simulator/FutureActionScheduler.java +++ b/test/simulator/main/org/apache/cassandra/simulator/FutureActionScheduler.java @@ -24,7 +24,7 @@ package org.apache.cassandra.simulator; */ public interface FutureActionScheduler { - enum Deliver { DELIVER, TIMEOUT, FAILURE } + enum Deliver { DELIVER, TIMEOUT, DELIVER_AND_TIMEOUT, FAILURE } /** * Make a decision about the result of some attempt to deliver a message. @@ -42,7 +42,7 @@ public interface FutureActionScheduler * The simulated global nanoTime at which a timeout should be reported for a message * with {@code expiresAfterNanos} timeout */ - long messageTimeoutNanos(long expiresAfterNanos); + long messageTimeoutNanos(long expiresAfterNanos, long expirationIntervalNanos); /** * The simulated global nanoTime at which a failure should be reported for a message diff --git a/test/simulator/main/org/apache/cassandra/simulator/OrderOns.java b/test/simulator/main/org/apache/cassandra/simulator/OrderOns.java index 19a2f4b89a..d4a07c1667 100644 --- a/test/simulator/main/org/apache/cassandra/simulator/OrderOns.java +++ b/test/simulator/main/org/apache/cassandra/simulator/OrderOns.java @@ -22,9 +22,14 @@ import java.util.ArrayList; import com.google.common.base.Preconditions; +import org.apache.cassandra.utils.Shared; + +import static org.apache.cassandra.utils.Shared.Scope.SIMULATION; + /** * A (possibly empty) collection of OrderOn */ +@Shared(scope = SIMULATION) public interface OrderOns { /** diff --git a/test/simulator/main/org/apache/cassandra/simulator/Ordered.java b/test/simulator/main/org/apache/cassandra/simulator/Ordered.java index 0281c04036..3e57c95335 100644 --- a/test/simulator/main/org/apache/cassandra/simulator/Ordered.java +++ b/test/simulator/main/org/apache/cassandra/simulator/Ordered.java @@ -64,10 +64,9 @@ class Ordered extends OrderedLink implements ActionListener { this.on = on; this.concurrency = on.concurrency(); - this.maybeRunning = !DEBUG ? new CountingCollection<>() - : concurrency == 1 - ? new ArrayList<>(1) - : new LinkedHashSet<>(); + this.maybeRunning = concurrency == 1 + ? new ArrayList<>(1) + : new LinkedHashSet<>(); } void add(O add, Function> memberOf) diff --git a/test/simulator/main/org/apache/cassandra/simulator/RunnableActionScheduler.java b/test/simulator/main/org/apache/cassandra/simulator/RunnableActionScheduler.java index 9609e66d34..95be9296fd 100644 --- a/test/simulator/main/org/apache/cassandra/simulator/RunnableActionScheduler.java +++ b/test/simulator/main/org/apache/cassandra/simulator/RunnableActionScheduler.java @@ -96,7 +96,7 @@ public abstract class RunnableActionScheduler implements Consumer public double priority() { double result = cur; - double step = (2*random.uniformDouble() - 1f) * maxStepSize; + double step = 2 * (random.uniformDouble() - 1f) * maxStepSize; this.cur = step > 0 ? Math.min(1d, cur + step) : Math.max(0d, cur + step); return result; diff --git a/test/simulator/main/org/apache/cassandra/simulator/SimulationRunner.java b/test/simulator/main/org/apache/cassandra/simulator/SimulationRunner.java index 93718448d7..3a651c9a08 100644 --- a/test/simulator/main/org/apache/cassandra/simulator/SimulationRunner.java +++ b/test/simulator/main/org/apache/cassandra/simulator/SimulationRunner.java @@ -42,9 +42,9 @@ import org.apache.cassandra.config.CassandraRelevantProperties; import org.apache.cassandra.simulator.Debug.Info; import org.apache.cassandra.simulator.Debug.Levels; import org.apache.cassandra.simulator.cluster.ClusterActions.TopologyChange; -import org.apache.cassandra.simulator.debug.Capture; import org.apache.cassandra.simulator.debug.SelfReconcile; import org.apache.cassandra.simulator.systems.InterceptedWait; +import org.apache.cassandra.simulator.systems.InterceptedWait.CaptureSites.Capture; import org.apache.cassandra.simulator.systems.InterceptibleThread; import org.apache.cassandra.simulator.systems.InterceptorOfGlobalMethods; import org.apache.cassandra.simulator.utils.ChanceRange; @@ -74,6 +74,7 @@ import static org.apache.cassandra.config.CassandraRelevantProperties.SHUTDOWN_A import static org.apache.cassandra.config.CassandraRelevantProperties.SYSTEM_AUTH_DEFAULT_RF; import static org.apache.cassandra.config.CassandraRelevantProperties.TEST_IGNORE_SIGAR; import static org.apache.cassandra.config.CassandraRelevantProperties.DISABLE_GOSSIP_ENDPOINT_REMOVAL; +import static org.apache.cassandra.config.CassandraRelevantProperties.TEST_JVM_DTEST_DISABLE_SSL; import static org.apache.cassandra.simulator.debug.Reconcile.reconcileWith; import static org.apache.cassandra.simulator.debug.Record.record; import static org.apache.cassandra.simulator.debug.SelfReconcile.reconcileWithSelf; @@ -85,6 +86,8 @@ public class SimulationRunner { private static final Logger logger = LoggerFactory.getLogger(SimulationRunner.class); + public enum RecordOption { NONE, VALUE, WITH_CALLSITES } + @BeforeClass public static void beforeAll() { @@ -92,7 +95,7 @@ public class SimulationRunner // Disallow time on the bootstrap classloader for (CassandraRelevantProperties property : Arrays.asList(CLOCK_GLOBAL, CLOCK_MONOTONIC_APPROX, CLOCK_MONOTONIC_PRECISE)) - property.setString("org.apache.cassandra.simulator.systems.SimulatedTime$Throwing"); + property.setString("org.apache.cassandra.simulator.systems.SimulatedTime$Delegating"); try { Clock.Global.nanoTime(); } catch (IllegalStateException e) {} // make sure static initializer gets called // TODO (cleanup): disable unnecessary things like compaction logger threads etc @@ -123,9 +126,10 @@ public class SimulationRunner MEMTABLE_OVERHEAD_SIZE.setInt(100); IGNORE_MISSING_NATIVE_FILE_HINTS.setBoolean(true); IS_DISABLED_MBEAN_REGISTRATION.setBoolean(true); + TEST_JVM_DTEST_DISABLE_SSL.setBoolean(true); // to support easily running without netty from dtest-jar if (Thread.currentThread() instanceof InterceptibleThread); // load InterceptibleThread class to avoid infinite loop in InterceptorOfGlobalMethods - new InterceptedWait.CaptureSites(Thread.currentThread(), false) + new InterceptedWait.CaptureSites(Thread.currentThread()) .toString(ste -> !ste.getClassName().equals(SelfReconcile.class.getName())); // ensure self reconcile verify can work without infinite looping InterceptorOfGlobalMethods.Global.unsafeReset(); ThreadLocalRandom.current(); @@ -138,7 +142,7 @@ public class SimulationRunner protected abstract static class BasicCommand> implements ICommand { - @Option(name = { "-s", "--seed"} , title = "0x|int", description = "Specify the first seed to test (each simulation will increment the seed by 1)") + @Option(name = { "--seed" } , title = "0x", description = "Specify the first seed to test (each simulation will increment the seed by 1)") protected String seed; @Option(name = { "--simulations"} , title = "int", description = "The number of simulations to run") @@ -167,9 +171,10 @@ public class SimulationRunner protected String topologyChanges = stream(TopologyChange.values()).map(Object::toString).collect(Collectors.joining(",")); @Option(name = { "--cluster-action-interval" }, title = "int...int(s|ms|us|ns)", description = "The period of time between two cluster actions (default 5..15s)") protected String topologyChangeInterval = "5..15s"; + @Option(name = { "--cluster-action-limit" }, title = "int", description = "The maximum number of topology change events to perform (default 0)") + protected String topologyChangeLimit = "0"; - - @Option(name = {"--run-time"}, title = "int", description = "Length of simulated time to run in seconds (default -1)") + @Option(name = { "-s", "--run-time" }, title = "int", description = "Length of simulated time to run in seconds (default -1)") protected int secondsToSimulate = -1; @Option(name = { "--reads" }, title = "[distribution:]float...float", description = "Proportion of actions that are reads (default: 0.05..0.95)") @@ -281,6 +286,7 @@ public class SimulationRunner .toArray(TopologyChange[]::new)); }); parseNanosRange(Optional.ofNullable(topologyChangeInterval)).ifPresent(builder::topologyChangeIntervalNanos); + builder.topologyChangeLimit(Integer.parseInt(topologyChangeLimit)); Optional.ofNullable(priority).ifPresent(kinds -> { builder.scheduler(stream(kinds.split(",")) .filter(v -> !v.isEmpty()) @@ -332,7 +338,7 @@ public class SimulationRunner propagate(builder); - long seed = parseLong(Optional.ofNullable(this.seed)).orElse(new Random(System.nanoTime()).nextLong()); + long seed = parseHex(Optional.ofNullable(this.seed)).orElse(new Random(System.nanoTime()).nextLong()); for (int i = 0 ; i < simulationCount ; ++i) { cleanup(); @@ -371,16 +377,16 @@ public class SimulationRunner @Option(name = {"--to"}, description = "Directory of recordings to reconcile with for the seed", required = true) private String dir; - @Option(name = {"--with-rng"}, description = "Record RNG values", arity = 0) - private boolean rng; + @Option(name = {"--with-rng"}, title = "0|1", description = "Record RNG values (with or without call sites)", allowedValues = {"0", "1"}) + private int rng = -1; - @Option(name = {"--with-rng-callsites"}, description = "Record RNG call sites", arity = 0) - private boolean rngCallSites; + @Option(name = {"--with-time"}, title = "0|1", description = "Record time values (with or without call sites)", allowedValues = {"0", "1"}) + private int time = -1; @Override protected void run(long seed, B builder) throws IOException { - record(dir, seed, rng, rngCallSites, builder); + record(dir, seed, RecordOption.values()[rng + 1], RecordOption.values()[time + 1], builder); } } @@ -390,11 +396,11 @@ public class SimulationRunner @Option(name = {"--with"}, description = "Directory of recordings to reconcile with for the seed") private String dir; - @Option(name = {"--with-rng"}, description = "Reconcile RNG values (if present in source)", arity = 0) - private boolean rng; + @Option(name = {"--with-rng"}, title = "0|1", description = "Reconcile RNG values (if present in source)", allowedValues = {"0", "1"}) + private int rng = -1; - @Option(name = {"--with-rng-callsites"}, description = "Reconcile RNG call sites (if present in source)", arity = 0) - private boolean rngCallSites; + @Option(name = {"--with-time"}, title = "0|1", description = "Reconcile time values (if present in source)", allowedValues = {"0", "1"}) + private int time = -1; @Option(name = {"--with-allocations"}, description = "Reconcile memtable allocations (only with --with-self)", arity = 0) private boolean allocations; @@ -405,9 +411,11 @@ public class SimulationRunner @Override protected void run(long seed, B builder) throws IOException { - if (withSelf) reconcileWithSelf(seed, rng, rngCallSites, allocations, builder); + RecordOption withRng = RecordOption.values()[rng + 1]; + RecordOption withTime = RecordOption.values()[time + 1]; + if (withSelf) reconcileWithSelf(seed, withRng, withTime, allocations, builder); else if (allocations) throw new IllegalArgumentException("--with-allocations is only compatible with --with-self"); - else reconcileWith(dir, seed, rng, rngCallSites, builder); + else reconcileWith(dir, seed, withRng, withTime, builder); } } @@ -421,11 +429,13 @@ public class SimulationRunner } - private static Optional parseLong(Optional value) + private static Optional parseHex(Optional value) { - return value.map(s -> s.startsWith("0x") - ? Hex.parseLong(s, 2, s.length()) - : Long.parseLong(s)); + return value.map(s -> { + if (s.startsWith("0x")) + return Hex.parseLong(s, 2, s.length()); + throw new IllegalArgumentException("Invalid hex string: " + s); + }); } private static final Pattern CHANCE_PATTERN = Pattern.compile("(uniform|(?qlog(\\((?[0-9]+)\\))?):)?(?0(\\.[0-9]+)?)(..(?0\\.[0-9]+))?", Pattern.CASE_INSENSITIVE); diff --git a/test/simulator/main/org/apache/cassandra/simulator/asm/InterceptAsClassTransformer.java b/test/simulator/main/org/apache/cassandra/simulator/asm/InterceptAsClassTransformer.java new file mode 100644 index 0000000000..200f984094 --- /dev/null +++ b/test/simulator/main/org/apache/cassandra/simulator/asm/InterceptAsClassTransformer.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.simulator.asm; + +import java.util.function.Predicate; + +import org.apache.cassandra.distributed.api.IClassTransformer; + +// an adapter to IClassTransformer that is loaded by the system classloader +public class InterceptAsClassTransformer extends InterceptClasses implements IClassTransformer +{ + public InterceptAsClassTransformer(ChanceSupplier monitorDelayChance, ChanceSupplier nemesisChance, NemesisFieldKind.Selector nemesisFieldSelector, ClassLoader prewarmClassLoader, Predicate prewarm) + { + super(monitorDelayChance, nemesisChance, nemesisFieldSelector, prewarmClassLoader, prewarm); + } + + public InterceptAsClassTransformer(int api, ChanceSupplier monitorDelayChance, ChanceSupplier nemesisChance, NemesisFieldKind.Selector nemesisFieldSelector, ClassLoader prewarmClassLoader, Predicate prewarm) + { + super(api, monitorDelayChance, nemesisChance, nemesisFieldSelector, prewarmClassLoader, prewarm); + } + + @Override + public byte[] transform(String name, byte[] bytecode) + { + return apply(name, bytecode); + } + + @Override + public IClassTransformer initialise() + { + return new SubTransformer()::apply; + } +} diff --git a/test/simulator/main/org/apache/cassandra/simulator/cluster/ClusterActions.java b/test/simulator/main/org/apache/cassandra/simulator/cluster/ClusterActions.java index 35fe882f3d..a1c4ccd8e5 100644 --- a/test/simulator/main/org/apache/cassandra/simulator/cluster/ClusterActions.java +++ b/test/simulator/main/org/apache/cassandra/simulator/cluster/ClusterActions.java @@ -44,7 +44,6 @@ import org.apache.cassandra.locator.ReplicaLayout; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.simulator.Action; import org.apache.cassandra.simulator.ActionList; -import org.apache.cassandra.simulator.Actions; import org.apache.cassandra.simulator.Actions.ReliableAction; import org.apache.cassandra.simulator.Actions.StrictAction; import org.apache.cassandra.simulator.Debug; @@ -57,9 +56,11 @@ import org.apache.cassandra.simulator.utils.KindOfSequence; import static org.apache.cassandra.distributed.impl.UnsafeGossipHelper.addToRingNormalRunner; import static org.apache.cassandra.simulator.Action.Modifiers.NO_TIMEOUTS; +import static org.apache.cassandra.simulator.Debug.EventType.CLUSTER; import static org.apache.cassandra.simulator.cluster.ClusterActions.TopologyChange.JOIN; import static org.apache.cassandra.simulator.cluster.ClusterActions.TopologyChange.LEAVE; import static org.apache.cassandra.simulator.cluster.ClusterActions.TopologyChange.REPLACE; +import static org.apache.cassandra.simulator.systems.NonInterceptible.Permit.REQUIRED; // TODO (feature): add Gossip failures (up to some acceptable number) @@ -79,6 +80,7 @@ public class ClusterActions extends SimulatedSystems public static class Options { + public final int topologyChangeLimit; public final KindOfSequence.Period topologyChangeInterval; public final Choices allChoices; public final Choices choicesNoLeave; @@ -94,6 +96,7 @@ public class ClusterActions extends SimulatedSystems public Options(Options copy, PaxosVariant changePaxosVariantTo) { + this.topologyChangeLimit = copy.topologyChangeLimit; this.topologyChangeInterval = copy.topologyChangeInterval; this.allChoices = copy.allChoices; this.choicesNoLeave = copy.choicesNoLeave; @@ -104,9 +107,13 @@ public class ClusterActions extends SimulatedSystems this.changePaxosVariantTo = changePaxosVariantTo; } - public Options(KindOfSequence.Period topologyChangeInterval, Choices choices, int[] minRf, int[] initialRf, int[] maxRf, PaxosVariant changePaxosVariantTo) + public Options(int topologyChangeLimit, KindOfSequence.Period topologyChangeInterval, Choices choices, int[] minRf, int[] initialRf, int[] maxRf, PaxosVariant changePaxosVariantTo) { + if (Arrays.equals(minRf, maxRf)) + choices = choices.without(TopologyChange.CHANGE_RF); + this.topologyChangeInterval = topologyChangeInterval; + this.topologyChangeLimit = topologyChangeLimit; this.minRf = minRf; this.initialRf = initialRf; this.maxRf = maxRf; @@ -184,6 +191,7 @@ public class ClusterActions extends SimulatedSystems } actions.add(ReliableAction.transitively("Sync Pending Ranges Executor", ClusterActions.this::syncPendingRanges)); + debug.debug(CLUSTER, time, cluster, null, null); return ActionList.of(actions); }); } @@ -197,7 +205,7 @@ public class ClusterActions extends SimulatedSystems void validateReplicasForKeys(IInvokableInstance on, String keyspace, String table, Topology topology) { int[] primaryKeys = topology.primaryKeys; - int[][] validate = NonInterceptible.apply(() -> { + int[][] validate = NonInterceptible.apply(REQUIRED, () -> { Map lookup = Cluster.getUniqueAddressLookup(cluster, i -> i.config().num()); int[][] result = new int[primaryKeys.length][]; for (int i = 0 ; i < primaryKeys.length ; ++i) diff --git a/test/simulator/main/org/apache/cassandra/simulator/cluster/KeyspaceActions.java b/test/simulator/main/org/apache/cassandra/simulator/cluster/KeyspaceActions.java index fa3e93936f..55ab20a219 100644 --- a/test/simulator/main/org/apache/cassandra/simulator/cluster/KeyspaceActions.java +++ b/test/simulator/main/org/apache/cassandra/simulator/cluster/KeyspaceActions.java @@ -79,6 +79,7 @@ public class KeyspaceActions extends ClusterActions final TokenMetadata tokenMetadata = new TokenMetadata(snitch.get()); Topology topology; boolean haveChangedVariant; + int topologyChangeCount = 0; public KeyspaceActions(SimulatedSystems simulated, String keyspace, String table, String createTableCql, @@ -180,7 +181,10 @@ public class KeyspaceActions extends ClusterActions private Action next() { - while (!ops.isEmpty() && !prejoin.isEmpty() || (ops.contains(LEAVE) && joined.size() > sum(minRf))) + if (options.topologyChangeLimit >= 0 && topologyChangeCount++ > options.topologyChangeLimit) + return null; + + while (!ops.isEmpty() && (!prejoin.isEmpty() || joined.size() > sum(minRf))) { if (options.changePaxosVariantTo != null && !haveChangedVariant && random.decide(1f / (1 + prejoin.size()))) { @@ -196,7 +200,7 @@ public class KeyspaceActions extends ClusterActions if (prejoin.size(dc) > 0 && joined.size(dc) > currentRf[dc]) next = options.allChoices.choose(random); else if (prejoin.size(dc) > 0 && ops.contains(JOIN)) next = options.choicesNoLeave.choose(random); else if (joined.size(dc) > currentRf[dc] && ops.contains(LEAVE)) next = options.choicesNoJoin.choose(random); - else if (joined.size(dc) > minRf[dc] && ops.contains(LEAVE)) next = CHANGE_RF; + else if (joined.size(dc) > minRf[dc]) next = CHANGE_RF; else continue; // TODO (feature): introduce some time period between cluster actions @@ -292,7 +296,7 @@ public class KeyspaceActions extends ClusterActions private Action schedule(Action action) { - action.setDeadline(time.nanoTime() + options.topologyChangeInterval.get(random)); + action.setDeadline(time, time.nanoTime() + options.topologyChangeInterval.get(random)); return action; } diff --git a/test/simulator/main/org/apache/cassandra/simulator/cluster/OnClusterChangeTopology.java b/test/simulator/main/org/apache/cassandra/simulator/cluster/OnClusterChangeTopology.java index 1372d877a0..5f0d46ff50 100644 --- a/test/simulator/main/org/apache/cassandra/simulator/cluster/OnClusterChangeTopology.java +++ b/test/simulator/main/org/apache/cassandra/simulator/cluster/OnClusterChangeTopology.java @@ -27,6 +27,7 @@ import org.apache.cassandra.simulator.systems.NonInterceptible; import static org.apache.cassandra.simulator.Action.Modifiers.RELIABLE; import static org.apache.cassandra.simulator.Action.Modifiers.STRICT; import static org.apache.cassandra.simulator.ActionListener.runAfterTransitiveClosure; +import static org.apache.cassandra.simulator.systems.NonInterceptible.Permit.REQUIRED; abstract class OnClusterChangeTopology extends Action implements Consumer { @@ -56,7 +57,7 @@ abstract class OnClusterChangeTopology extends Action implements Consumer { + NonInterceptible.execute(REQUIRED, () -> { actions.validateReplicasForKeys(instance, actions.keyspace, actions.table, before); validator.before(before, participatingKeys); }); diff --git a/test/simulator/main/org/apache/cassandra/simulator/cluster/OnClusterJoin.java b/test/simulator/main/org/apache/cassandra/simulator/cluster/OnClusterJoin.java index 51e8ec912e..cf393795da 100644 --- a/test/simulator/main/org/apache/cassandra/simulator/cluster/OnClusterJoin.java +++ b/test/simulator/main/org/apache/cassandra/simulator/cluster/OnClusterJoin.java @@ -41,6 +41,7 @@ class OnClusterJoin extends OnClusterChangeTopology // setup the node's own gossip state for pending ownership, and return gossip actions to disseminate new OnClusterUpdateGossip(actions, joining, new OnInstanceSetBootstrapping(actions, joining)), new OnInstanceSyncSchemaForBootstrap(actions, joining), + new OnInstanceTopologyChangePaxosRepair(actions, joining, "Join"), // stream/repair from a peer new OnInstanceBootstrap(actions, joinInstance), // setup the node's own gossip state for natural ownership, and return gossip actions to disseminate diff --git a/test/simulator/main/org/apache/cassandra/simulator/cluster/OnClusterLeave.java b/test/simulator/main/org/apache/cassandra/simulator/cluster/OnClusterLeave.java index a6063ba851..4250e243ee 100644 --- a/test/simulator/main/org/apache/cassandra/simulator/cluster/OnClusterLeave.java +++ b/test/simulator/main/org/apache/cassandra/simulator/cluster/OnClusterLeave.java @@ -25,7 +25,6 @@ import org.apache.cassandra.distributed.api.IInvokableInstance; import org.apache.cassandra.service.StorageService; import org.apache.cassandra.simulator.ActionList; import org.apache.cassandra.simulator.systems.SimulatedActionConsumer; -import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.concurrent.Future; import static org.apache.cassandra.simulator.Action.Modifiers.RELIABLE_NO_TIMEOUTS; @@ -51,8 +50,9 @@ class OnClusterLeave extends OnClusterChangeTopology new OnClusterUpdateGossip(actions, leaving, new OnInstanceSetLeaving(actions, leaving)), new SimulatedActionConsumer<>("Prepare unbootstrap on " + leaving, RELIABLE_NO_TIMEOUTS, RELIABLE_NO_TIMEOUTS, actions, leaveInstance, ref -> ref.set(StorageService.instance.prepareUnbootstrapStreaming()), preparedUnbootstrap), + new OnInstanceTopologyChangePaxosRepair(actions, leaving, "Leave"), new SimulatedActionConsumer<>("Execute unbootstrap on " + leaving, RELIABLE_NO_TIMEOUTS, RELIABLE_NO_TIMEOUTS, actions, leaveInstance, - ref -> FBUtilities.waitOnFuture(ref.get().get()), preparedUnbootstrap), + ref -> ref.get().get().syncThrowUncheckedOnInterrupt(), preparedUnbootstrap), // setup the node's own gossip state for natural ownership, and return gossip actions to disseminate new OnClusterUpdateGossip(actions, leaving, new OnInstanceSetLeft(actions, leaving)) ); diff --git a/test/simulator/main/org/apache/cassandra/simulator/cluster/OnClusterReplace.java b/test/simulator/main/org/apache/cassandra/simulator/cluster/OnClusterReplace.java index 15253f37c5..8e0af4061f 100644 --- a/test/simulator/main/org/apache/cassandra/simulator/cluster/OnClusterReplace.java +++ b/test/simulator/main/org/apache/cassandra/simulator/cluster/OnClusterReplace.java @@ -101,6 +101,7 @@ class OnClusterReplace extends OnClusterChangeTopology new OnClusterUpdateGossip(actions, joining, new OnInstanceSetBootstrapReplacing(actions, joining, leaving, hostId, movingToken)), new OnInstanceSyncSchemaForBootstrap(actions, joining), + new OnInstanceTopologyChangePaxosRepair(actions, joining, "Replace"), new OnInstanceBootstrap(actions, joinInstance, movingToken, true), // setup the node's own gossip state for natural ownership, and return gossip actions to disseminate diff --git a/test/simulator/main/org/apache/cassandra/simulator/cluster/OnInstanceRepair.java b/test/simulator/main/org/apache/cassandra/simulator/cluster/OnInstanceRepair.java index 1671e0b095..9da7317fdd 100644 --- a/test/simulator/main/org/apache/cassandra/simulator/cluster/OnInstanceRepair.java +++ b/test/simulator/main/org/apache/cassandra/simulator/cluster/OnInstanceRepair.java @@ -96,7 +96,7 @@ class OnInstanceRepair extends ClusterAction { Collection> ranges = rangesSupplier.call(); // no need to wait for completion, as we track all task submissions and message exchanges, and ensure they finish before continuing to next action - StorageService.instance.repair(keyspaceName, new RepairOption(RepairParallelism.SEQUENTIAL, isPrimaryRangeOnly, false, false, 1, ranges, false, false, force, PreviewKind.NONE, true, true), singletonList((tag, event) -> { + StorageService.instance.repair(keyspaceName, new RepairOption(RepairParallelism.SEQUENTIAL, isPrimaryRangeOnly, false, false, 1, ranges, false, false, force, PreviewKind.NONE, false, true, repairPaxos, repairOnlyPaxos), singletonList((tag, event) -> { if (event.getType() == ProgressEventType.COMPLETE) listener.run(); })); diff --git a/test/simulator/main/org/apache/cassandra/simulator/cluster/OnInstanceTopologyChangePaxosRepair.java b/test/simulator/main/org/apache/cassandra/simulator/cluster/OnInstanceTopologyChangePaxosRepair.java new file mode 100644 index 0000000000..daaa4744d2 --- /dev/null +++ b/test/simulator/main/org/apache/cassandra/simulator/cluster/OnInstanceTopologyChangePaxosRepair.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.simulator.cluster; + +import org.apache.cassandra.distributed.api.IIsolatedExecutor.SerializableRunnable; +import org.apache.cassandra.service.StorageService; +import org.apache.cassandra.utils.concurrent.Condition; +import org.apache.cassandra.utils.concurrent.Future; + +import static org.apache.cassandra.net.Verb.MUTATION_REQ; +import static org.apache.cassandra.net.Verb.PAXOS2_CLEANUP_COMPLETE_REQ; +import static org.apache.cassandra.net.Verb.PAXOS2_CLEANUP_FINISH_PREPARE_REQ; +import static org.apache.cassandra.net.Verb.PAXOS2_CLEANUP_REQ; +import static org.apache.cassandra.net.Verb.PAXOS2_CLEANUP_RSP; +import static org.apache.cassandra.net.Verb.PAXOS2_CLEANUP_RSP2; +import static org.apache.cassandra.net.Verb.PAXOS2_CLEANUP_START_PREPARE_REQ; +import static org.apache.cassandra.net.Verb.READ_REQ; +import static org.apache.cassandra.net.Verb.SCHEMA_PULL_REQ; +import static org.apache.cassandra.net.Verb.SCHEMA_PUSH_REQ; +import static org.apache.cassandra.simulator.Action.Modifiers.NO_TIMEOUTS; +import static org.apache.cassandra.simulator.Action.Modifiers.RELIABLE; +import static org.apache.cassandra.utils.concurrent.Condition.newOneTimeCondition; + +class OnInstanceTopologyChangePaxosRepair extends ClusterAction +{ + public OnInstanceTopologyChangePaxosRepair(ClusterActions actions, int on, String reason) + { + this("Paxos Topology Repair on " + on, RELIABLE, NO_TIMEOUTS, actions, on, invokableTopologyChangeRepair(reason)); + } + + public OnInstanceTopologyChangePaxosRepair(String id, Modifiers self, Modifiers transitive, ClusterActions actions, int on, SerializableRunnable runnable) + { + super(id, RELIABLE.with(self), NO_TIMEOUTS.with(transitive), actions, on, runnable); + setMessageModifiers(SCHEMA_PULL_REQ, RELIABLE, RELIABLE); + setMessageModifiers(SCHEMA_PUSH_REQ, RELIABLE, RELIABLE); + setMessageModifiers(PAXOS2_CLEANUP_START_PREPARE_REQ, RELIABLE, RELIABLE); + setMessageModifiers(PAXOS2_CLEANUP_REQ, RELIABLE, RELIABLE); + setMessageModifiers(PAXOS2_CLEANUP_FINISH_PREPARE_REQ, RELIABLE, RELIABLE); + setMessageModifiers(PAXOS2_CLEANUP_COMPLETE_REQ, RELIABLE, RELIABLE); + setMessageModifiers(PAXOS2_CLEANUP_RSP, RELIABLE, RELIABLE); + setMessageModifiers(PAXOS2_CLEANUP_RSP2, RELIABLE, RELIABLE); + setMessageModifiers(MUTATION_REQ, RELIABLE, RELIABLE); + setMessageModifiers(READ_REQ, RELIABLE, RELIABLE); + } + + protected static SerializableRunnable invokableTopologyChangeRepair(String reason) + { + return () -> { + Condition condition = newOneTimeCondition(); + Future future = StorageService.instance.startRepairPaxosForTopologyChange(reason); + future.addListener(condition::signal); // add listener so we don't use Futures.addAllAsList + condition.awaitThrowUncheckedOnInterrupt(); + }; + } +} diff --git a/test/simulator/main/org/apache/cassandra/simulator/debug/Reconcile.java b/test/simulator/main/org/apache/cassandra/simulator/debug/Reconcile.java index 64c61cb982..61700c5c73 100644 --- a/test/simulator/main/org/apache/cassandra/simulator/debug/Reconcile.java +++ b/test/simulator/main/org/apache/cassandra/simulator/debug/Reconcile.java @@ -39,10 +39,17 @@ import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.File; import org.apache.cassandra.simulator.ClusterSimulation; import org.apache.cassandra.simulator.RandomSource; +import org.apache.cassandra.simulator.SimulationRunner.RecordOption; +import org.apache.cassandra.simulator.systems.InterceptedWait; +import org.apache.cassandra.simulator.systems.InterceptedWait.CaptureSites.Capture; +import org.apache.cassandra.simulator.systems.SimulatedTime; import org.apache.cassandra.utils.Closeable; import org.apache.cassandra.utils.CloseableIterator; import org.apache.cassandra.utils.concurrent.Threads; +import static org.apache.cassandra.simulator.SimulationRunner.RecordOption.NONE; +import static org.apache.cassandra.simulator.SimulationRunner.RecordOption.VALUE; +import static org.apache.cassandra.simulator.SimulationRunner.RecordOption.WITH_CALLSITES; import static org.apache.cassandra.simulator.SimulatorUtils.failWithOOM; public class Reconcile @@ -63,12 +70,13 @@ public class Reconcile final List strings = new ArrayList<>(); final boolean inputHasCallSites; final boolean reconcileCallSites; + int line; - public AbstractReconciler(DataInputPlus in, boolean inputHasCallSites, boolean reconcileCallSites) + public AbstractReconciler(DataInputPlus in, boolean inputHasCallSites, RecordOption reconcile) { this.in = in; this.inputHasCallSites = inputHasCallSites; - this.reconcileCallSites = reconcileCallSites; + this.reconcileCallSites = reconcile == WITH_CALLSITES; } String readInterned() throws IOException @@ -85,6 +93,8 @@ public class Reconcile return ""; String trace = in.readUTF(); + for (int i = trace.indexOf('\n') ; i >= 0 ; i = trace.indexOf('\n', i + 1)) + ++line; return reconcileCallSites ? trace : ""; } @@ -96,7 +106,9 @@ public class Reconcile StackTraceElement[] ste = Thread.currentThread().getStackTrace(); return Arrays.stream(ste, 4, ste.length) .filter(st -> !st.getClassName().equals("org.apache.cassandra.simulator.debug.Reconcile") - && !st.getClassName().equals("org.apache.cassandra.simulator.SimulationRunner$Reconcile")) + && !st.getClassName().equals("org.apache.cassandra.simulator.SimulationRunner$Reconcile") + && !st.getClassName().equals("sun.reflect.NativeMethodAccessorImpl") // depends on async compile thread + && !st.getClassName().startsWith("sun.reflect.GeneratedMethodAccessor")) // depends on async compile thread .collect(new Threads.StackTraceCombiner(true, "", "\n", "")); } @@ -115,6 +127,45 @@ public class Reconcile } } + public static class TimeReconciler extends AbstractReconciler implements SimulatedTime.Listener, Closeable + { + boolean disabled; + + public TimeReconciler(DataInputPlus in, boolean inputHasCallSites, RecordOption reconcile) + { + super(in, inputHasCallSites, reconcile); + } + + @Override + public void close() + { + disabled = true; + } + + @Override + public synchronized void accept(String kind, long value) + { + if (disabled) + return; + + try + { + String testKind = readInterned(); + long testValue = in.readUnsignedVInt(); + checkThread(); + if (!kind.equals(testKind) || value != testValue) + { + logger.error("({},{}) != ({},{})", kind, value, testKind, testValue); + throw failWithOOM(); + } + } + catch (IOException e) + { + throw new RuntimeException(e); + } + } + } + public static class RandomSourceReconciler extends RandomSource.Abstract implements Supplier, Closeable { private static final Logger logger = LoggerFactory.getLogger(RandomSourceReconciler.class); @@ -126,11 +177,11 @@ public class Reconcile volatile Thread locked; volatile boolean disabled; - public RandomSourceReconciler(DataInputPlus in, RandomSource wrapped, boolean inputHasCallSites, boolean reconcileCallSites) + public RandomSourceReconciler(DataInputPlus in, RandomSource wrapped, boolean inputHasCallSites, RecordOption reconcile) { this.in = in; this.wrapped = wrapped; - this.threads = new AbstractReconciler(in, inputHasCallSites, reconcileCallSites); + this.threads = new AbstractReconciler(in, inputHasCallSites, reconcile); } private void enter() @@ -151,6 +202,35 @@ public class Reconcile locked = null; } + public void onDeterminismCheck(long value) + { + if (disabled) + return; + + enter(); + try + { + byte type = in.readByte(); + int c = (int) in.readVInt(); + long v = in.readLong(); + threads.checkThread(); + if (type != 7 || c != count || value != v) + { + logger.error(String.format("(%d,%d,%d) != (%d,%d,%d)", 7, count, value, type, c, v)); + throw failWithOOM(); + } + ++count; + } + catch (IOException e) + { + throw new RuntimeException(e); + } + finally + { + exit(); + } + } + public int uniform(int min, int max) { int v = wrapped.uniform(min, max); @@ -347,18 +427,17 @@ public class Reconcile } } - public static void reconcileWith(String loadFromDir, long seed, boolean withRng, boolean withRngCallSites, ClusterSimulation.Builder builder) + public static void reconcileWith(String loadFromDir, long seed, RecordOption withRng, RecordOption withTime, ClusterSimulation.Builder builder) { - if (withRngCallSites && !withRng) - throw new IllegalStateException(); - File eventFile = new File(new File(loadFromDir), Long.toHexString(seed) + ".gz"); File rngFile = new File(new File(loadFromDir), Long.toHexString(seed) + ".rng.gz"); + File timeFile = new File(new File(loadFromDir), Long.toHexString(seed) + ".time.gz"); try (BufferedReader eventIn = new BufferedReader(new InputStreamReader(new GZIPInputStream(eventFile.newInputStream()))); - DataInputPlus.DataInputStreamPlus rngIn = new DataInputPlus.DataInputStreamPlus(rngFile.exists() ? new GZIPInputStream(rngFile.newInputStream()) : new ByteArrayInputStream(new byte[0]))) + DataInputPlus.DataInputStreamPlus rngIn = new DataInputPlus.DataInputStreamPlus(rngFile.exists() && withRng != NONE ? new GZIPInputStream(rngFile.newInputStream()) : new ByteArrayInputStream(new byte[0])); + DataInputPlus.DataInputStreamPlus timeIn = new DataInputPlus.DataInputStreamPlus(timeFile.exists() && withTime != NONE ? new GZIPInputStream(timeFile.newInputStream()) : new ByteArrayInputStream(new byte[0]))) { - boolean inputHasWaitSites, inputHasWakeSites, inputHasRngCallSites; + boolean inputHasWaitSites, inputHasWakeSites, inputHasRngCallSites, inputHasTimeCallSites; { String modifiers = eventIn.readLine(); if (!modifiers.startsWith("modifiers:")) @@ -369,18 +448,29 @@ public class Reconcile builder.capture().wakeSites & (inputHasWakeSites = modifiers.contains("wakeSites")), builder.capture().nowSites) ); - withRng &= modifiers.contains("rng"); - withRngCallSites &= inputHasRngCallSites = modifiers.contains("rngCallSites"); + inputHasRngCallSites = modifiers.contains("rngCallSites"); + if (!modifiers.contains("rng")) withRng = NONE; + if (withRng == WITH_CALLSITES && !inputHasRngCallSites) withRng = VALUE; + + inputHasTimeCallSites = modifiers.contains("timeCallSites"); + if (!modifiers.contains("time")) withTime = NONE; + if (withTime == WITH_CALLSITES && !inputHasTimeCallSites) withTime = VALUE; } - if (withRng && !rngFile.exists()) + if (withRng != NONE && !rngFile.exists()) + throw new IllegalStateException(); + if (withTime != NONE && !timeFile.exists()) throw new IllegalStateException(); { Set modifiers = new LinkedHashSet<>(); - if (withRngCallSites) + if (withRng == WITH_CALLSITES) modifiers.add("rngCallSites"); - else if (withRng) + else if (withRng == VALUE) modifiers.add("rng"); + if (withTime == WITH_CALLSITES) + modifiers.add("timeCallSites"); + else if (withTime == VALUE) + modifiers.add("time"); if (builder.capture().waitSites) modifiers.add("WaitSites"); if (builder.capture().wakeSites) @@ -389,8 +479,14 @@ public class Reconcile } RandomSourceReconciler random = null; - if (withRng) - builder.random(random = new RandomSourceReconciler(rngIn, new RandomSource.Default(), inputHasRngCallSites, withRngCallSites)); + TimeReconciler time = null; + if (withRng != NONE) + { + builder.random(random = new RandomSourceReconciler(rngIn, new RandomSource.Default(), inputHasRngCallSites, withRng)); + builder.onThreadLocalRandomCheck(random::onDeterminismCheck); + } + if (withTime != NONE) + builder.timeListener(time = new TimeReconciler(timeIn, inputHasTimeCallSites, withTime)); class Line { int line = 1; } Line line = new Line(); // box for heap dump analysis try (ClusterSimulation cluster = builder.create(seed); @@ -413,6 +509,8 @@ public class Reconcile } if (random != null) random.close(); + if (time != null) + time.close(); } catch (Throwable t) { diff --git a/test/simulator/main/org/apache/cassandra/simulator/debug/Record.java b/test/simulator/main/org/apache/cassandra/simulator/debug/Record.java index cca2c5e85e..7ca7a20ac8 100644 --- a/test/simulator/main/org/apache/cassandra/simulator/debug/Record.java +++ b/test/simulator/main/org/apache/cassandra/simulator/debug/Record.java @@ -41,11 +41,16 @@ import org.apache.cassandra.io.util.DataOutputStreamPlus; import org.apache.cassandra.io.util.File; import org.apache.cassandra.simulator.ClusterSimulation; import org.apache.cassandra.simulator.RandomSource; +import org.apache.cassandra.simulator.SimulationRunner.RecordOption; +import org.apache.cassandra.simulator.systems.SimulatedTime; import org.apache.cassandra.utils.Closeable; import org.apache.cassandra.utils.CloseableIterator; import org.apache.cassandra.utils.concurrent.Threads; import static org.apache.cassandra.io.util.File.WriteMode.OVERWRITE; +import static org.apache.cassandra.simulator.SimulationRunner.RecordOption.NONE; +import static org.apache.cassandra.simulator.SimulationRunner.RecordOption.VALUE; +import static org.apache.cassandra.simulator.SimulationRunner.RecordOption.WITH_CALLSITES; import static org.apache.cassandra.simulator.SimulatorUtils.failWithOOM; public class Record @@ -54,17 +59,22 @@ public class Record private static final Pattern NORMALISE_THREAD_RECORDING_OUT = Pattern.compile("(Thread\\[[^]]+:[0-9]+),[0-9](,node[0-9]+)_[0-9]+]"); private static final Pattern NORMALISE_LAMBDA = Pattern.compile("((\\$\\$Lambda\\$[0-9]+/[0-9]+)?(@[0-9a-f]+)?)"); - public static void record(String saveToDir, long seed, boolean withRng, boolean withRngCallSites, ClusterSimulation.Builder builder) + public static void record(String saveToDir, long seed, RecordOption withRng, RecordOption withTime, ClusterSimulation.Builder builder) { File eventFile = new File(new File(saveToDir), Long.toHexString(seed) + ".gz"); File rngFile = new File(new File(saveToDir), Long.toHexString(seed) + ".rng.gz"); + File timeFile = new File(new File(saveToDir), Long.toHexString(seed) + ".time.gz"); { Set modifiers = new LinkedHashSet<>(); - if (withRngCallSites) + if (withRng == WITH_CALLSITES) modifiers.add("rngCallSites"); - else if (withRng) + else if (withRng == VALUE) modifiers.add("rng"); + if (withTime == WITH_CALLSITES) + modifiers.add("timeCallSites"); + else if (withTime == VALUE) + modifiers.add("time"); if (builder.capture().waitSites) modifiers.add("WaitSites"); if (builder.capture().wakeSites) @@ -73,22 +83,25 @@ public class Record } try (PrintWriter eventOut = new PrintWriter(new GZIPOutputStream(eventFile.newOutputStream(OVERWRITE), 1 << 16)); - DataOutputStreamPlus rngOut = new BufferedDataOutputStreamPlus(Channels.newChannel(withRng ? new GZIPOutputStream(rngFile.newOutputStream(OVERWRITE), 1 << 16) : new ByteArrayOutputStream(0)))) + DataOutputStreamPlus rngOut = new BufferedDataOutputStreamPlus(Channels.newChannel(withRng != NONE ? new GZIPOutputStream(rngFile.newOutputStream(OVERWRITE), 1 << 16) : new ByteArrayOutputStream(0))); + DataOutputStreamPlus timeOut = new BufferedDataOutputStreamPlus(Channels.newChannel(withTime != NONE ? new GZIPOutputStream(timeFile.newOutputStream(OVERWRITE), 1 << 16) : new ByteArrayOutputStream(0)))) { eventOut.println("modifiers:" - + (withRng ? "rng," : "") + (withRngCallSites ? "rngCallSites," : "") + + (withRng == VALUE ? "rng," : "") + (withRng == WITH_CALLSITES ? "rngCallSites," : "") + + (withTime == VALUE ? "time," : "") + (withTime == WITH_CALLSITES ? "timeCallSites," : "") + (builder.capture().waitSites ? "waitSites," : "") + (builder.capture().wakeSites ? "wakeSites," : "")); + TimeRecorder time; RandomSourceRecorder random; - if (withRng) + if (withRng != NONE) { - random = new RandomSourceRecorder(rngOut, new RandomSource.Default(), withRngCallSites); - builder.random(random); - } - else - { - random = null; + builder.random(random = new RandomSourceRecorder(rngOut, new RandomSource.Default(), withRng)); + builder.onThreadLocalRandomCheck(random::onDeterminismCheck); } + else random = null; + + if (withTime != NONE) builder.timeListener(time = new TimeRecorder(timeOut, withTime)); + else time = null; // periodic forced flush to ensure state is on disk after some kind of stall Thread flusher = new Thread(() -> { @@ -105,6 +118,13 @@ public class Record rngOut.flush(); } } + if (time != null) + { + synchronized (time) + { + timeOut.flush(); + } + } } } catch (IOException e) @@ -171,6 +191,42 @@ public class Record ).replaceAll("$1$2]"); } + public static class TimeRecorder extends AbstractRecorder implements SimulatedTime.Listener, java.io.Closeable + { + boolean disabled; + + public TimeRecorder(DataOutputStreamPlus out, RecordOption option) + { + super(out, option); + } + + @Override + public void close() throws IOException + { + disabled = true; + out.close(); + } + + @Override + public synchronized void accept(String kind, long value) + { + if (disabled) + return; + + try + { + writeInterned(kind); + out.writeUnsignedVInt(value); + writeThread(); + } + catch (IOException e) + { + throw new RuntimeException(e); + } + } + } + + // TODO: merge with TimeRecorder to produce one stream, use to support live reconciliation between two JVMs over socket public static class RandomSourceRecorder extends RandomSource.Abstract implements Supplier, Closeable { private static final AtomicReferenceFieldUpdater lockedUpdater = AtomicReferenceFieldUpdater.newUpdater(Record.RandomSourceRecorder.class, Thread.class, "locked"); @@ -182,11 +238,11 @@ public class Record volatile Thread locked; volatile boolean disabled; - public RandomSourceRecorder(DataOutputStreamPlus out, RandomSource wrapped, boolean withCallSites) + public RandomSourceRecorder(DataOutputStreamPlus out, RandomSource wrapped, RecordOption option) { this.out = out; this.wrapped = wrapped; - this.threads = new AbstractRecorder(out, withCallSites); + this.threads = new AbstractRecorder(out, option); } private void enter() @@ -214,6 +270,33 @@ public class Record locked = null; } + // determinism check is exclusively a ThreadLocalRandom issue at the moment + public void onDeterminismCheck(long value) + { + if (disabled) + return; + + enter(); + try + { + synchronized (this) + { + out.writeByte(7); + out.writeVInt(count++); + out.writeLong(value); + threads.writeThread(); + } + } + catch (IOException e) + { + throw new RuntimeException(e); + } + finally + { + exit(); + } + } + public int uniform(int min, int max) { int v = wrapped.uniform(min, max); @@ -403,10 +486,10 @@ public class Record final boolean withCallSites; final Map objects = new IdentityHashMap<>(); - public AbstractRecorder(DataOutputStreamPlus out, boolean withCallSites) + public AbstractRecorder(DataOutputStreamPlus out, RecordOption option) { this.out = out; - this.withCallSites = withCallSites; + this.withCallSites = option == WITH_CALLSITES; } public void writeThread() throws IOException @@ -417,8 +500,10 @@ public class Record { StackTraceElement[] ste = thread.getStackTrace(); String trace = Arrays.stream(ste, 3, ste.length) - .filter(st -> !st.getClassName().equals("org.apache.cassandra.simulator.debug.Record") - && !st.getClassName().equals("org.apache.cassandra.simulator.SimulationRunner$Record")) + .filter(st -> !st.getClassName().equals("org.apache.cassandra.simulator.debug.Record") + && !st.getClassName().equals("org.apache.cassandra.simulator.SimulationRunner$Record") + && !st.getClassName().equals("sun.reflect.NativeMethodAccessorImpl") // depends on async compile thread + && !st.getClassName().startsWith("sun.reflect.GeneratedMethodAccessor")) // depends on async compile thread .collect(new Threads.StackTraceCombiner(true, "", "\n", "")); out.writeUTF(trace); } diff --git a/test/simulator/main/org/apache/cassandra/simulator/debug/SelfReconcile.java b/test/simulator/main/org/apache/cassandra/simulator/debug/SelfReconcile.java index 122bea580a..be8c05a959 100644 --- a/test/simulator/main/org/apache/cassandra/simulator/debug/SelfReconcile.java +++ b/test/simulator/main/org/apache/cassandra/simulator/debug/SelfReconcile.java @@ -34,16 +34,20 @@ import org.apache.cassandra.distributed.api.IMessage; import org.apache.cassandra.simulator.ClusterSimulation; import org.apache.cassandra.simulator.OrderOn; import org.apache.cassandra.simulator.RandomSource; +import org.apache.cassandra.simulator.SimulationRunner.RecordOption; import org.apache.cassandra.simulator.systems.InterceptedExecution; import org.apache.cassandra.simulator.systems.InterceptedWait; import org.apache.cassandra.simulator.systems.InterceptedWait.CaptureSites; import org.apache.cassandra.simulator.systems.InterceptibleThread; import org.apache.cassandra.simulator.systems.InterceptorOfConsequences; +import org.apache.cassandra.simulator.systems.SimulatedTime; import org.apache.cassandra.utils.CloseableIterator; import org.apache.cassandra.utils.Pair; import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException; import org.apache.cassandra.utils.memory.HeapPool; +import static org.apache.cassandra.simulator.SimulationRunner.RecordOption.NONE; +import static org.apache.cassandra.simulator.SimulationRunner.RecordOption.WITH_CALLSITES; import static org.apache.cassandra.simulator.SimulatorUtils.failWithOOM; import static org.apache.cassandra.simulator.debug.Reconcile.NORMALISE_LAMBDA; import static org.apache.cassandra.simulator.debug.Reconcile.NORMALISE_THREAD; @@ -56,7 +60,7 @@ public class SelfReconcile private static final Logger logger = LoggerFactory.getLogger(SelfReconcile.class); static final Pattern NORMALISE_RECONCILE_THREAD = Pattern.compile("(Thread\\[Reconcile:)[0-9]+,[0-9],Reconcile(_[0-9]+)?]"); - static class InterceptReconciler implements InterceptorOfConsequences, Supplier + static class InterceptReconciler implements InterceptorOfConsequences, Supplier, SimulatedTime.Listener { final List events = new ArrayList<>(); final boolean withRngCallsites; @@ -82,7 +86,7 @@ public class SelfReconcile } @Override - public synchronized void interceptWakeup(InterceptedWait wakeup, InterceptorOfConsequences waitWasInterceptedBy) + public synchronized void interceptWakeup(InterceptedWait wakeup, InterceptedWait.Trigger trigger, InterceptorOfConsequences waitWasInterceptedBy) { verify(normalise("Wakeup " + wakeup.waiting() + wakeup)); } @@ -211,7 +215,7 @@ public class SelfReconcile return result; } InterceptReconciler.this.verify(withRngCallsites ? event + result + ' ' + Thread.currentThread() + ' ' - + new CaptureSites(Thread.currentThread(), false) + + new CaptureSites(Thread.currentThread()) .toString(ste -> !ste.getClassName().startsWith(SelfReconcile.class.getName())) : event + result); return result; @@ -223,14 +227,21 @@ public class SelfReconcile { closed = true; } + + @Override + public void accept(String kind, long value) + { + verify(Thread.currentThread() + ":" + kind + ':' + value); + } } - public static void reconcileWithSelf(long seed, boolean withRng, boolean withRngCallSites, boolean withAllocations, ClusterSimulation.Builder builder) + public static void reconcileWithSelf(long seed, RecordOption withRng, RecordOption withTime, boolean withAllocations, ClusterSimulation.Builder builder) { logger.error("Seed 0x{}", Long.toHexString(seed)); - InterceptReconciler reconciler = new InterceptReconciler(withRngCallSites); - if (withRng) builder.random(reconciler); + InterceptReconciler reconciler = new InterceptReconciler(withRng == WITH_CALLSITES); + if (withRng != NONE) builder.random(reconciler); + if (withTime != NONE) builder.timeListener(reconciler); HeapPool.Logged.Listener memoryListener = withAllocations ? reconciler::interceptAllocation : null; ExecutorService executor = ExecutorFactory.Global.executorFactory().pooled("Reconcile", 2); @@ -238,37 +249,44 @@ public class SelfReconcile try (ClusterSimulation cluster1 = builder.unique(0).memoryListener(memoryListener).create(seed); ClusterSimulation cluster2 = builder.unique(1).memoryListener(memoryListener).create(seed)) { - InterceptibleThread.setDebugInterceptor(reconciler); - reconciler.verifyUninterceptedRng = true; + try + { - Future f1 = executor.submit(() -> { - try (CloseableIterator iter = cluster1.simulation.iterator()) - { - while (iter.hasNext()) + InterceptibleThread.setDebugInterceptor(reconciler); + reconciler.verifyUninterceptedRng = true; + + Future f1 = executor.submit(() -> { + try (CloseableIterator iter = cluster1.simulation.iterator()) { - Object o = iter.next(); - reconciler.verify(Pair.create(normalise(o.toString()), o)); + while (iter.hasNext()) + { + Object o = iter.next(); + reconciler.verify(Pair.create(normalise(o.toString()), o)); + } } - } - reconciler.verify("done"); - }); - Future f2 = executor.submit(() -> { - try (CloseableIterator iter = cluster2.simulation.iterator()) - { - while (iter.hasNext()) + reconciler.verify("done"); + }); + Future f2 = executor.submit(() -> { + try (CloseableIterator iter = cluster2.simulation.iterator()) { - Object o = iter.next(); - reconciler.verify(Pair.create(normalise(o.toString()), o)); + while (iter.hasNext()) + { + Object o = iter.next(); + reconciler.verify(Pair.create(normalise(o.toString()), o)); + } } - } - reconciler.verify("done"); - }); - f1.get(); - f2.get(); + reconciler.verify("done"); + }); + f1.get(); + f2.get(); + } + finally + { + reconciler.close(); + } } catch (Throwable t) { - reconciler.close(); t.printStackTrace(); throw new RuntimeException("Failed on seed " + Long.toHexString(seed), t); } diff --git a/test/simulator/main/org/apache/cassandra/simulator/package-info.java b/test/simulator/main/org/apache/cassandra/simulator/package-info.java index ab5b54d98b..d6da8288d2 100644 --- a/test/simulator/main/org/apache/cassandra/simulator/package-info.java +++ b/test/simulator/main/org/apache/cassandra/simulator/package-info.java @@ -64,6 +64,7 @@ * JVM parameters: * -XX:ActiveProcessorCount=??? * -Xmx??? + * -XX:-BackgroundCompilation * * For performance reasons the following parameters are recommended: * -XX:Tier4CompileThreshold=1000 diff --git a/test/simulator/main/org/apache/cassandra/simulator/paxos/Ballots.java b/test/simulator/main/org/apache/cassandra/simulator/paxos/Ballots.java index d72e0829e8..7862f60d60 100644 --- a/test/simulator/main/org/apache/cassandra/simulator/paxos/Ballots.java +++ b/test/simulator/main/org/apache/cassandra/simulator/paxos/Ballots.java @@ -19,7 +19,6 @@ package org.apache.cassandra.simulator.paxos; import java.util.List; -import java.util.UUID; import com.google.common.collect.ImmutableList; @@ -45,17 +44,19 @@ import org.apache.cassandra.schema.ColumnMetadata; import org.apache.cassandra.schema.SchemaConstants; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.simulator.systems.NonInterceptible; +import org.apache.cassandra.simulator.systems.NonInterceptible.Permit; +import org.apache.cassandra.utils.TimeUUID; import org.apache.cassandra.service.paxos.Commit; import org.apache.cassandra.service.paxos.PaxosState; import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.Shared; -import org.apache.cassandra.utils.TimeUUID; -import org.apache.cassandra.utils.UUIDGen; import static java.lang.Long.max; import static java.util.Arrays.stream; import static org.apache.cassandra.db.SystemKeyspace.loadPaxosState; -import static org.apache.cassandra.service.paxos.Commit.isAfter; +import static org.apache.cassandra.service.paxos.Commit.latest; +import static org.apache.cassandra.service.paxos.PaxosState.unsafeGetIfPresent; +import static org.apache.cassandra.simulator.systems.NonInterceptible.Permit.OPTIONAL; import static org.apache.cassandra.utils.Shared.Scope.SIMULATION; public class Ballots @@ -96,37 +97,38 @@ public class Ballots } } - public static LatestBallots read(DecoratedKey key, TableMetadata metadata, int nowInSec, boolean includeEmptyProposals) + public static LatestBallots read(Permit permit, DecoratedKey key, TableMetadata metadata, int nowInSec, boolean includeEmptyProposals) { - return NonInterceptible.apply(() -> { - PaxosState state = loadPaxosState(key, metadata, nowInSec); - TimeUUID promised = state.promised.ballot; - Commit accepted = isAfter(state.accepted, state.mostRecentCommit) ? null : state.accepted; - Commit committed = state.mostRecentCommit; - long baseTable = latestBallotFromBaseTable(key, metadata); - return new LatestBallots( - promised.unixMicros(), - accepted == null || accepted.update.isEmpty() ? 0L : latestBallot(accepted.update), - latestBallot(committed.update), - baseTable - ); - }); + return NonInterceptible.apply(permit, () -> { + PaxosState.Snapshot state = unsafeGetIfPresent(key, metadata); + PaxosState.Snapshot persisted = loadPaxosState(key, metadata, nowInSec); + TimeUUID promised = latest(persisted.promised, state == null ? null : state.promised); + Commit.Accepted accepted = latest(persisted.accepted, state == null ? null : state.accepted); + Commit.Committed committed = latest(persisted.committed, state == null ? null : state.committed); + long baseTable = latestBallotFromBaseTable(key, metadata); + return new LatestBallots( + promised.unixMicros(), + accepted == null || accepted.update.isEmpty() ? 0L : latestBallot(accepted.update), + latestBallot(committed.update), + baseTable + ); + }); } - static LatestBallots[][] read(Cluster cluster, String keyspace, String table, int[] primaryKeys, int[][] replicasForKeys, boolean includeEmptyProposals) + static LatestBallots[][] read(Permit permit, Cluster cluster, String keyspace, String table, int[] primaryKeys, int[][] replicasForKeys, boolean includeEmptyProposals) { - return NonInterceptible.apply(() -> { + return NonInterceptible.apply(permit, () -> { LatestBallots[][] result = new LatestBallots[primaryKeys.length][]; for (int i = 0 ; i < primaryKeys.length ; ++i) { int primaryKey = primaryKeys[i]; result[i] = stream(replicasForKeys[i]) .mapToObj(cluster::get) - .map(node -> node.unsafeApplyOnThisThread((ks, tbl, pk, ie) -> { + .map(node -> node.unsafeApplyOnThisThread((p, ks, tbl, pk, ie) -> { TableMetadata metadata = Keyspace.open(ks).getColumnFamilyStore(tbl).metadata.get(); DecoratedKey key = metadata.partitioner.decorateKey(Int32Type.instance.decompose(pk)); - return read(key, metadata, FBUtilities.nowInSeconds(), ie); - }, keyspace, table, primaryKey, includeEmptyProposals)) + return read(p, key, metadata, FBUtilities.nowInSeconds(), ie); + }, permit, keyspace, table, primaryKey, includeEmptyProposals)) .toArray(LatestBallots[]::new); } return result; @@ -135,14 +137,16 @@ public class Ballots public static String paxosDebugInfo(DecoratedKey key, TableMetadata metadata, int nowInSec) { - return NonInterceptible.apply(() -> { - PaxosState paxosTable = loadPaxosState(key, metadata, nowInSec); - long[] paxosMemtable = latestBallotsFromPaxosMemtable(key, metadata); + return NonInterceptible.apply(OPTIONAL, () -> { + PaxosState.Snapshot state = unsafeGetIfPresent(key, metadata); + PaxosState.Snapshot persisted = loadPaxosState(key, metadata, nowInSec); + long[] memtable = latestBallotsFromPaxosMemtable(key, metadata); + PaxosState.Snapshot cache = state == null ? persisted : state; long baseTable = latestBallotFromBaseTable(key, metadata); long baseMemtable = latestBallotFromBaseMemtable(key, metadata); - return debugBallot(null, paxosMemtable[0], paxosTable.promised) + ", " - + debugBallot(null, paxosMemtable[1], paxosTable.accepted) + ", " - + debugBallot(null, paxosMemtable[2], paxosTable.mostRecentCommit) + ", " + return debugBallot(cache.promised, memtable[0], persisted.promised) + ", " + + debugBallot(cache.accepted, memtable[1], persisted.accepted) + ", " + + debugBallot(cache.committed, memtable[2], persisted.committed) + ", " + debugBallot(baseMemtable, 0L, baseTable); }); } diff --git a/test/simulator/main/org/apache/cassandra/simulator/paxos/PairOfSequencesPaxosSimulation.java b/test/simulator/main/org/apache/cassandra/simulator/paxos/PairOfSequencesPaxosSimulation.java index 10147e10c8..b94648d495 100644 --- a/test/simulator/main/org/apache/cassandra/simulator/paxos/PairOfSequencesPaxosSimulation.java +++ b/test/simulator/main/org/apache/cassandra/simulator/paxos/PairOfSequencesPaxosSimulation.java @@ -21,6 +21,7 @@ package org.apache.cassandra.simulator.paxos; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.concurrent.ExecutionException; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.LongSupplier; import java.util.function.Supplier; @@ -48,15 +49,22 @@ import org.apache.cassandra.simulator.Actions; import org.apache.cassandra.simulator.cluster.ClusterActions; import org.apache.cassandra.simulator.Debug; import org.apache.cassandra.simulator.cluster.KeyspaceActions; +import org.apache.cassandra.simulator.systems.SimulatedActionTask; import org.apache.cassandra.simulator.systems.SimulatedSystems; import org.apache.cassandra.simulator.utils.IntRange; import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException; import static java.lang.Boolean.TRUE; import static java.util.Collections.emptyList; import static java.util.Collections.singletonList; import static java.util.concurrent.TimeUnit.SECONDS; import static org.apache.cassandra.distributed.api.ConsistencyLevel.ANY; +import static org.apache.cassandra.simulator.Action.Modifiers.RELIABLE; +import static org.apache.cassandra.simulator.Action.Modifiers.RELIABLE_NO_TIMEOUTS; +import static org.apache.cassandra.simulator.ActionSchedule.Mode.STREAM_LIMITED; +import static org.apache.cassandra.simulator.ActionSchedule.Mode.TIME_AND_STREAM_LIMITED; +import static org.apache.cassandra.simulator.ActionSchedule.Mode.TIME_LIMITED; import static org.apache.cassandra.simulator.Debug.EventType.PARTITION; import static org.apache.cassandra.simulator.paxos.HistoryChecker.fail; @@ -71,8 +79,6 @@ public class PairOfSequencesPaxosSimulation extends PaxosSimulation private static final String INSERT = "INSERT INTO " + KEYSPACE + ".tbl (pk, count, seq1, seq2) VALUES (?, 0, '', []) IF NOT EXISTS"; private static final String INSERT1 = "INSERT INTO " + KEYSPACE + ".tbl (pk, count, seq1, seq2) VALUES (?, 0, '', []) USING TIMESTAMP 0"; private static final String UPDATE = "UPDATE " + KEYSPACE + ".tbl SET count = count + 1, seq1 = seq1 + ?, seq2 = seq2 + ? WHERE pk = ? IF EXISTS"; - private static final String DELETE = "DELETE FROM " + KEYSPACE + ".tbl WHERE pk = ? IF EXISTS"; - private static final String DELETE1 = "DELETE FROM " + KEYSPACE + ".tbl USING TIMESTAMP 4611686018427387904 WHERE pk = ?"; private static final String SELECT = "SELECT pk, count, seq1, seq2 FROM " + KEYSPACE + ".tbl WHERE pk = ?"; private static final ListType LIST_TYPE = ListType.getInstance(Int32Type.instance, true); @@ -127,7 +133,7 @@ public class PairOfSequencesPaxosSimulation extends PaxosSimulation } } - class ModifyingOperation extends Operation + public class ModifyingOperation extends Operation { final HistoryChecker historyChecker; public ModifyingOperation(int id, IInvokableInstance instance, ConsistencyLevel commitConsistency, ConsistencyLevel serialConsistency, int primaryKey, HistoryChecker historyChecker) @@ -174,7 +180,8 @@ public class PairOfSequencesPaxosSimulation extends PaxosSimulation long seed, int[] primaryKeys, long runForNanos, LongSupplier jitter) { - super(simulated, cluster, scheduler, runForNanos, jitter); + super(runForNanos < 0 ? STREAM_LIMITED : clusterOptions.topologyChangeLimit < 0 ? TIME_LIMITED : TIME_AND_STREAM_LIMITED, + simulated, cluster, scheduler, runForNanos, jitter); this.readRatio = readRatio; this.concurrency = concurrency; this.simulateKeyForSeconds = simulateKeyForSeconds; @@ -183,7 +190,8 @@ public class PairOfSequencesPaxosSimulation extends PaxosSimulation this.clusterOptions = clusterOptions; this.debug = debug; this.seed = seed; - this.primaryKeys = primaryKeys; + this.primaryKeys = primaryKeys.clone(); + Arrays.sort(this.primaryKeys); } public ActionPlan plan() @@ -196,7 +204,7 @@ public class PairOfSequencesPaxosSimulation extends PaxosSimulation cluster.stream().map(i -> simulated.run("Insert Partitions", i, executeForPrimaryKeys(INSERT1, primaryKeys))) ), ActionList.of( - cluster.stream().map(i -> simulated.run("Delete Partitions", i, executeForPrimaryKeys(DELETE1, primaryKeys))) + cluster.stream().map(i -> SimulatedActionTask.unsafeTask("Shutdown " + i.broadcastAddress(), RELIABLE, RELIABLE_NO_TIMEOUTS, simulated, i, i::shutdown)) ) )); diff --git a/test/simulator/main/org/apache/cassandra/simulator/paxos/PaxosClusterSimulation.java b/test/simulator/main/org/apache/cassandra/simulator/paxos/PaxosClusterSimulation.java index 17150f0c15..0fd9135f5b 100644 --- a/test/simulator/main/org/apache/cassandra/simulator/paxos/PaxosClusterSimulation.java +++ b/test/simulator/main/org/apache/cassandra/simulator/paxos/PaxosClusterSimulation.java @@ -34,7 +34,7 @@ class PaxosClusterSimulation extends ClusterSimulation implemen @SuppressWarnings("UnusedReturnValue") static class Builder extends ClusterSimulation.Builder { - PaxosVariant initialPaxosVariant = PaxosVariant.v1; + PaxosVariant initialPaxosVariant = PaxosVariant.v2; PaxosVariant finalPaxosVariant = null; Boolean stateCache; ConsistencyLevel serialConsistency = SERIAL; @@ -75,7 +75,10 @@ class PaxosClusterSimulation extends ClusterSimulation implemen { super(random, seed, uniqueNum, builder, config -> config.set("paxos_variant", builder.initialPaxosVariant.name()) - .set("paxos_cache_size_in_mb", (builder.stateCache != null ? builder.stateCache : random.uniformFloat() < 0.5) ? null : 0L), + .set("paxos_cache_size", (builder.stateCache != null ? builder.stateCache : random.uniformFloat() < 0.5) ? null : "0MiB") + .set("paxos_state_purging", "repaired") + .set("paxos_on_linearizability_violations", "log") + , (simulated, schedulers, cluster, options) -> { int[] primaryKeys = primaryKeys(seed, builder.primaryKeyCount()); KindOfSequence.Period jitter = RandomSource.Choices.uniform(KindOfSequence.values()).choose(random) diff --git a/test/simulator/main/org/apache/cassandra/simulator/paxos/PaxosRepairValidator.java b/test/simulator/main/org/apache/cassandra/simulator/paxos/PaxosRepairValidator.java index b209009927..3ce2740379 100644 --- a/test/simulator/main/org/apache/cassandra/simulator/paxos/PaxosRepairValidator.java +++ b/test/simulator/main/org/apache/cassandra/simulator/paxos/PaxosRepairValidator.java @@ -23,6 +23,7 @@ import org.apache.cassandra.simulator.cluster.ClusterActionListener.RepairValida import org.apache.cassandra.simulator.cluster.Topology; import static java.util.Arrays.stream; +import static org.apache.cassandra.simulator.systems.NonInterceptible.Permit.REQUIRED; public class PaxosRepairValidator implements RepairValidator { @@ -51,7 +52,7 @@ public class PaxosRepairValidator implements RepairValidator this.isPaxos = isPaxos; this.topology = topology; - this.ballotsBefore = Ballots.read(cluster, keyspace, table, topology.primaryKeys, topology.replicasForKeys, false); + this.ballotsBefore = Ballots.read(REQUIRED, cluster, keyspace, table, topology.primaryKeys, topology.replicasForKeys, false); } @Override @@ -64,7 +65,7 @@ public class PaxosRepairValidator implements RepairValidator int[][] replicasForKeys = topology.replicasForKeys; int quorumRf = topology.quorumRf; int quorum = quorumRf / 2 + 1; - Ballots.LatestBallots[][] ballotsAfter = Ballots.read(cluster, keyspace, table, primaryKeys, replicasForKeys, true); + Ballots.LatestBallots[][] ballotsAfter = Ballots.read(REQUIRED, cluster, keyspace, table, primaryKeys, replicasForKeys, true); for (int pki = 0; pki < primaryKeys.length ; ++pki) { Ballots.LatestBallots[] before = ballotsBefore[pki]; diff --git a/test/simulator/main/org/apache/cassandra/simulator/paxos/PaxosSimulation.java b/test/simulator/main/org/apache/cassandra/simulator/paxos/PaxosSimulation.java index 8b12860fbc..63e84dcf24 100644 --- a/test/simulator/main/org/apache/cassandra/simulator/paxos/PaxosSimulation.java +++ b/test/simulator/main/org/apache/cassandra/simulator/paxos/PaxosSimulation.java @@ -18,10 +18,12 @@ package org.apache.cassandra.simulator.paxos; +import java.util.Map; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; import java.util.function.BiConsumer; import java.util.function.LongSupplier; @@ -42,6 +44,7 @@ import org.apache.cassandra.exceptions.RequestExecutionException; import org.apache.cassandra.service.paxos.BallotGenerator; import org.apache.cassandra.simulator.ActionList; import org.apache.cassandra.simulator.ActionPlan; +import org.apache.cassandra.simulator.ActionSchedule; import org.apache.cassandra.simulator.RunnableActionScheduler; import org.apache.cassandra.simulator.Simulation; import org.apache.cassandra.simulator.cluster.ClusterActionListener; @@ -49,6 +52,7 @@ import org.apache.cassandra.simulator.systems.InterceptorOfGlobalMethods; import org.apache.cassandra.simulator.systems.SimulatedQuery; import org.apache.cassandra.simulator.systems.SimulatedSystems; import org.apache.cassandra.utils.CloseableIterator; +import org.apache.cassandra.utils.concurrent.Threads; import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException; import static org.apache.cassandra.simulator.Action.Modifiers.NONE; @@ -105,15 +109,17 @@ public abstract class PaxosSimulation implements Simulation, ClusterActionListen final SimulatedSystems simulated; final RunnableActionScheduler runnableScheduler; final AtomicInteger logicalClock = new AtomicInteger(1); + final ActionSchedule.Mode mode; final long runForNanos; final LongSupplier jitter; - public PaxosSimulation(SimulatedSystems simulated, Cluster cluster, RunnableActionScheduler runnableScheduler, long runForNanos, LongSupplier jitter) + public PaxosSimulation(ActionSchedule.Mode mode, SimulatedSystems simulated, Cluster cluster, RunnableActionScheduler runnableScheduler, long runForNanos, LongSupplier jitter) { this.cluster = cluster; this.simulated = simulated; this.runnableScheduler = runnableScheduler; this.runForNanos = runForNanos; + this.mode = mode; this.jitter = jitter; } @@ -121,6 +127,9 @@ public abstract class PaxosSimulation implements Simulation, ClusterActionListen public void run() { + AtomicReference> onFailedShutdown = new AtomicReference<>(); + AtomicInteger shutdown = new AtomicInteger(); + AtomicLong counter = new AtomicLong(); ScheduledExecutorPlus livenessChecker = null; ScheduledFuture liveness = null; @@ -133,37 +142,67 @@ public abstract class PaxosSimulation implements Simulation, ClusterActionListen @Override public void run() { - long cur = counter.get(); - if (cur == prev) + Thread.currentThread().setUncaughtExceptionHandler((th, ex) -> { + logger.error("Unexpected exception on {}", th, ex); + }); + if (shutdown.get() > 0) { - logger.error("Simulation appears to have stalled"); - throw failWithOOM(); + int attempts = shutdown.getAndIncrement(); + if (attempts > 2 || onFailedShutdown.get() == null) + { + logger.error("Failed to exit despite best efforts, dumping threads and forcing shutdown"); + for (Map.Entry stes : Thread.getAllStackTraces().entrySet()) + { + logger.error("{}", stes.getKey()); + logger.error("{}", Threads.prettyPrint(stes.getValue(), false, "\n")); + } + + System.exit(1); + } + else if (attempts > 1) + { + logger.error("Failed to exit cleanly, force closing simulation"); + onFailedShutdown.get().close(); + } + } + else + { + long cur = counter.get(); + if (cur == prev) + { + logger.error("Simulation appears to have stalled; terminating. To disable set -Dcassandra.test.simulator.livenesscheck=false"); + shutdown.set(1); + throw failWithOOM(); + } + prev = cur; } - prev = cur; } - }, 1L, 1L, TimeUnit.MINUTES); + }, 5L, 5L, TimeUnit.MINUTES); } try (CloseableIterator iter = iterator()) { + onFailedShutdown.set(iter); while (iter.hasNext()) { + if (shutdown.get() > 0) + throw failWithOOM(); + iter.next(); counter.incrementAndGet(); } } - finally - { - if (liveness != null) - liveness.cancel(true); - if (livenessChecker != null) - livenessChecker.shutdownNow(); - } + + // only cancel if successfully shutdown; otherwise we may have a shutdown liveness issue, and should kill process + if (liveness != null) + liveness.cancel(true); + if (livenessChecker != null) + livenessChecker.shutdownNow(); } public CloseableIterator iterator() { - CloseableIterator iterator = plan().iterator(runForNanos, jitter, simulated.time, new RunnableActionScheduler.Immediate(), runnableScheduler, simulated.futureScheduler); + CloseableIterator iterator = plan().iterator(mode, runForNanos, jitter, simulated.time, runnableScheduler, simulated.futureScheduler); return new CloseableIterator() { @Override @@ -231,7 +270,12 @@ public abstract class PaxosSimulation implements Simulation, ClusterActionListen { // stop intercepting message delivery cluster.setMessageSink(null); - cluster.forEach(i -> i.unsafeRunOnThisThread(() -> BallotGenerator.Global.unsafeSet(new BallotGenerator.Default()))); - cluster.forEach(i -> i.unsafeRunOnThisThread(InterceptorOfGlobalMethods.Global::unsafeReset)); + cluster.forEach(i -> { + if (!i.isShutdown()) + { + i.unsafeRunOnThisThread(() -> BallotGenerator.Global.unsafeSet(new BallotGenerator.Default())); + i.unsafeRunOnThisThread(InterceptorOfGlobalMethods.Global::unsafeReset); + } + }); } } diff --git a/test/simulator/main/org/apache/cassandra/simulator/paxos/PaxosSimulationRunner.java b/test/simulator/main/org/apache/cassandra/simulator/paxos/PaxosSimulationRunner.java index 0728855ae0..50a0ee5b51 100644 --- a/test/simulator/main/org/apache/cassandra/simulator/paxos/PaxosSimulationRunner.java +++ b/test/simulator/main/org/apache/cassandra/simulator/paxos/PaxosSimulationRunner.java @@ -20,6 +20,7 @@ package org.apache.cassandra.simulator.paxos; import java.io.IOException; import java.util.Optional; +import java.util.concurrent.atomic.AtomicInteger; import io.airlift.airline.Cli; import io.airlift.airline.Command; @@ -125,11 +126,17 @@ public class PaxosSimulationRunner extends SimulationRunner Optional.ofNullable(toVariant).map(Config.PaxosVariant::valueOf).ifPresent(builder::finalPaxosVariant); } + // for simple unit tests so we can simply invoke main() + private static final AtomicInteger uniqueNum = new AtomicInteger(); + /** * See {@link org.apache.cassandra.simulator} package info for execution tips */ public static void main(String[] args) throws IOException { + PaxosClusterSimulation.Builder builder = new PaxosClusterSimulation.Builder(); + builder.unique(uniqueNum.getAndIncrement()); + Cli.>builder("paxos") .withCommand(Run.class) .withCommand(Reconcile.class) @@ -138,6 +145,6 @@ public class PaxosSimulationRunner extends SimulationRunner .withDefaultCommand(Help.class) .build() .parse(args) - .run(new PaxosClusterSimulation.Builder()); + .run(builder); } } diff --git a/test/simulator/main/org/apache/cassandra/simulator/paxos/PaxosTopologyChangeVerifier.java b/test/simulator/main/org/apache/cassandra/simulator/paxos/PaxosTopologyChangeVerifier.java index 026b2c47fb..d9803bfa8d 100644 --- a/test/simulator/main/org/apache/cassandra/simulator/paxos/PaxosTopologyChangeVerifier.java +++ b/test/simulator/main/org/apache/cassandra/simulator/paxos/PaxosTopologyChangeVerifier.java @@ -24,6 +24,7 @@ import org.apache.cassandra.simulator.cluster.Topology; import org.apache.cassandra.simulator.systems.NonInterceptible; import static java.util.Arrays.stream; +import static org.apache.cassandra.simulator.systems.NonInterceptible.Permit.REQUIRED; public class PaxosTopologyChangeVerifier implements TopologyChangeValidator { @@ -47,9 +48,7 @@ public class PaxosTopologyChangeVerifier implements TopologyChangeValidator public void before(Topology before, int[] participatingKeys) { this.topologyBefore = before.select(participatingKeys); - this.ballotsBefore = NonInterceptible.apply(() -> - Ballots.read(cluster, keyspace, table, topologyBefore.primaryKeys, topologyBefore.replicasForKeys, true) - ); + this.ballotsBefore = Ballots.read(REQUIRED, cluster, keyspace, table, topologyBefore.primaryKeys, topologyBefore.replicasForKeys, true); for (int i = 0; i < topologyBefore.primaryKeys.length ; ++i) { if (ballotsBefore[i].length != topologyBefore.quorumRf) @@ -69,9 +68,7 @@ public class PaxosTopologyChangeVerifier implements TopologyChangeValidator int quorumBefore = topologyBefore.quorumRf / 2 + 1; int quorumAfter = topologyAfter.quorumRf / 2 + 1; Ballots.LatestBallots[][] allBefore = ballotsBefore; - Ballots.LatestBallots[][] allAfter = NonInterceptible.apply(() -> - Ballots.read(cluster, keyspace, table, primaryKeys, topologyAfter.replicasForKeys, true) - ); + Ballots.LatestBallots[][] allAfter = Ballots.read(REQUIRED, cluster, keyspace, table, primaryKeys, topologyAfter.replicasForKeys, true); for (int pki = 0; pki < primaryKeys.length; ++pki) { Ballots.LatestBallots[] before = allBefore[pki]; diff --git a/test/simulator/main/org/apache/cassandra/simulator/systems/InterceptedExecution.java b/test/simulator/main/org/apache/cassandra/simulator/systems/InterceptedExecution.java index a6065246ab..4002f4e9bf 100644 --- a/test/simulator/main/org/apache/cassandra/simulator/systems/InterceptedExecution.java +++ b/test/simulator/main/org/apache/cassandra/simulator/systems/InterceptedExecution.java @@ -50,7 +50,7 @@ public interface InterceptedExecution boolean submittedOrCancelled; Runnable onCancel; - protected InterceptedTaskExecution(InterceptingExecutor executor) + public InterceptedTaskExecution(InterceptingExecutor executor) { Preconditions.checkNotNull(executor); this.executor = executor; @@ -225,6 +225,7 @@ public interface InterceptedExecution } finally { + thread.onTermination(); thread.interceptTermination(true); } } diff --git a/test/simulator/main/org/apache/cassandra/simulator/systems/InterceptedWait.java b/test/simulator/main/org/apache/cassandra/simulator/systems/InterceptedWait.java index 64ee3d3169..999e9aa156 100644 --- a/test/simulator/main/org/apache/cassandra/simulator/systems/InterceptedWait.java +++ b/test/simulator/main/org/apache/cassandra/simulator/systems/InterceptedWait.java @@ -33,6 +33,11 @@ import org.apache.cassandra.utils.concurrent.Threads; import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException; import static org.apache.cassandra.simulator.SimulatorUtils.failWithOOM; +import static org.apache.cassandra.simulator.systems.InterceptedWait.CaptureSites.Capture.WAKE; +import static org.apache.cassandra.simulator.systems.InterceptedWait.CaptureSites.Capture.WAKE_AND_NOW; +import static org.apache.cassandra.simulator.systems.InterceptedWait.Trigger.SIGNAL; +import static org.apache.cassandra.simulator.systems.InterceptibleThread.interceptorOrDefault; +import static org.apache.cassandra.utils.Shared.Recursive.ALL; import static org.apache.cassandra.utils.Shared.Recursive.INTERFACES; import static org.apache.cassandra.utils.Shared.Scope.SIMULATION; @@ -44,6 +49,7 @@ import static org.apache.cassandra.utils.Shared.Scope.SIMULATION; public interface InterceptedWait extends NotifyThreadPaused { enum Kind { SLEEP_UNTIL, WAIT_UNTIL, UNBOUNDED_WAIT, NEMESIS } + enum Trigger { TIMEOUT, INTERRUPT, SIGNAL } interface TriggerListener { @@ -66,19 +72,29 @@ public interface InterceptedWait extends NotifyThreadPaused */ boolean isTriggered(); + /** + * true if this wait can be interrupted + */ + boolean isInterruptible(); + /** * If kind() == TIMED_WAIT or ABSOLUTE_TIMED_WAIT this returns the relative and absolute * period to wait in nanos */ long waitTime(); + /** + * Intercept a wakeup signal on this wait + */ + void interceptWakeup(Trigger trigger, Thread by); + /** * Signal the waiter immediately, and have the caller wait until its simulation has terminated. * * @param interceptor the interceptor to relay events to - * @param isTimeout propagate the signal to the wrapped condition we are waiting on + * @param trigger if SIGNAL, propagate the signal to the wrapped condition we are waiting on */ - void triggerAndAwaitDone(InterceptorOfConsequences interceptor, boolean isTimeout); + void triggerAndAwaitDone(InterceptorOfConsequences interceptor, Trigger trigger); /** * Signal all waiters immediately, bypassing the simulation @@ -108,9 +124,7 @@ public interface InterceptedWait extends NotifyThreadPaused final Condition propagateSignal; final List onTrigger = new ArrayList<>(3); final long waitTime; - boolean isTriggered; - boolean isDone; - boolean hasExited; + boolean isInterruptible, isSignalPending, isTriggered, isDone, hasExited; public InterceptedConditionWait(Kind kind, long waitTime, InterceptibleThread waiting, CaptureSites captureSites, Condition propagateSignal) { @@ -122,22 +136,25 @@ public interface InterceptedWait extends NotifyThreadPaused this.propagateSignal = propagateSignal; } - public synchronized void triggerAndAwaitDone(InterceptorOfConsequences interceptor, boolean isTimeout) + public synchronized void triggerAndAwaitDone(InterceptorOfConsequences interceptor, Trigger trigger) { if (isTriggered) return; if (hasExited) { - logger.error("{} exited without trigger {}", waiting, captureSites == null ? new CaptureSites(waiting, true) : captureSites); + logger.error("{} exited without trigger {}", waiting, captureSites == null ? new CaptureSites(waiting, WAKE_AND_NOW) : captureSites); throw failWithOOM(); } waiting.beforeInvocation(interceptor, this); isTriggered = true; onTrigger.forEach(listener -> listener.onTrigger(this)); - super.signal(); - if (!isTimeout && propagateSignal != null) + + if (!waiting.preWakeup(this) || !isInterruptible) + super.signal(); + + if (isSignalPending && propagateSignal != null) propagateSignal.signal(); try @@ -193,9 +210,25 @@ public interface InterceptedWait extends NotifyThreadPaused return waitTime; } + @Override + public void interceptWakeup(Trigger trigger, Thread by) + { + assert !isTriggered; + isSignalPending |= trigger == SIGNAL; + if (captureSites != null) + captureSites.registerWakeup(by); + interceptorOrDefault(by).interceptWakeup(this, trigger, interceptedBy); + } + public boolean isTriggered() { - return isSignalled(); + return isTriggered; + } + + @Override + public boolean isInterruptible() + { + return isInterruptible; } // ignore return value; always false as can only represent artificial (intercepted) signaled status @@ -203,6 +236,7 @@ public interface InterceptedWait extends NotifyThreadPaused { try { + isInterruptible = true; super.await(); } finally @@ -217,6 +251,7 @@ public interface InterceptedWait extends NotifyThreadPaused { try { + isInterruptible = true; super.await(); } finally @@ -231,6 +266,7 @@ public interface InterceptedWait extends NotifyThreadPaused { try { + isInterruptible = false; super.awaitUninterruptibly(); } finally @@ -245,6 +281,7 @@ public interface InterceptedWait extends NotifyThreadPaused { try { + isInterruptible = false; super.awaitUninterruptibly(time, units); } finally @@ -258,6 +295,22 @@ public interface InterceptedWait extends NotifyThreadPaused { try { + isInterruptible = true; + super.await(); + } + finally + { + hasExited = true; + } + return this; + } + + // declared as uninterruptible to the simulator to avoid unnecessary wakeups, but handles interrupts if they arise + public Condition awaitDeclaredUninterruptible() throws InterruptedException + { + try + { + isInterruptible = false; super.await(); } finally @@ -271,6 +324,7 @@ public interface InterceptedWait extends NotifyThreadPaused { try { + isInterruptible = false; super.awaitUninterruptibly(); } finally @@ -287,45 +341,76 @@ public interface InterceptedWait extends NotifyThreadPaused } // debug the place at which a thread waits or is signalled - @Shared(scope = SIMULATION) + @Shared(scope = SIMULATION, members = ALL) public static class CaptureSites { + public static class Capture + { + public static final Capture NONE = new Capture(false, false, false); + public static final Capture WAKE = new Capture(true, false, false); + public static final Capture WAKE_AND_NOW = new Capture(true, true, false); + + public final boolean waitSites; + public final boolean wakeSites; + public final boolean nowSites; + + public Capture(boolean waitSites, boolean wakeSites, boolean nowSites) + { + this.waitSites = waitSites; + this.wakeSites = wakeSites; + this.nowSites = nowSites; + } + + public boolean any() + { + return waitSites | wakeSites | nowSites; + } + } + final Thread waiting; final StackTraceElement[] waitSite; - final boolean printNowTrace; + final Capture capture; @SuppressWarnings("unused") Thread waker; StackTraceElement[] wakeupSite; - public CaptureSites(Thread waiting, StackTraceElement[] waitSite, boolean printNowTrace) + public CaptureSites(Thread waiting, StackTraceElement[] waitSite, Capture capture) { this.waiting = waiting; this.waitSite = waitSite; - this.printNowTrace = printNowTrace; + this.capture = capture; } - public CaptureSites(Thread waiting, boolean printNowTrace) + public CaptureSites(Thread waiting, Capture capture) { this.waiting = waiting; this.waitSite = waiting.getStackTrace(); - this.printNowTrace = printNowTrace; + this.capture = capture; } - void registerWakeup(Thread waking) + public CaptureSites(Thread waiting) + { + this.waiting = waiting; + this.waitSite = waiting.getStackTrace(); + this.capture = WAKE; + } + + public void registerWakeup(Thread waking) { this.waker = waking; - this.wakeupSite = waking.getStackTrace(); + if (capture.wakeSites) + this.wakeupSite = waking.getStackTrace(); } public String toString(Predicate include) { String tail; if (wakeupSite != null) - tail = Threads.prettyPrint(Stream.of(wakeupSite).filter(include), true, printNowTrace ? "]# by[" : waitSite != null ? " by[" : "by[", "; ", "]"); - else if (printNowTrace) + tail = Threads.prettyPrint(Stream.of(wakeupSite).filter(include), true, capture.nowSites ? "]# by[" : waitSite != null ? " by[" : "by[", "; ", "]"); + else if (capture.nowSites) tail = "]#"; else tail = ""; - if (printNowTrace) + if (capture.nowSites) tail = Threads.prettyPrint(Stream.of(waiting.getStackTrace()).filter(include), true, waitSite != null ? " #[" : "#[", "; ", tail); if (waitSite != null) tail =Threads.prettyPrint(Stream.of(waitSite).filter(include), true, "", "; ", tail); diff --git a/test/simulator/main/org/apache/cassandra/simulator/systems/InterceptibleThread.java b/test/simulator/main/org/apache/cassandra/simulator/systems/InterceptibleThread.java index 826d9c7536..0cb26bf0e5 100644 --- a/test/simulator/main/org/apache/cassandra/simulator/systems/InterceptibleThread.java +++ b/test/simulator/main/org/apache/cassandra/simulator/systems/InterceptibleThread.java @@ -26,11 +26,15 @@ import io.netty.util.concurrent.FastThreadLocalThread; import org.apache.cassandra.distributed.api.IInvokableInstance; import org.apache.cassandra.distributed.api.IMessage; import org.apache.cassandra.simulator.OrderOn; +import org.apache.cassandra.simulator.systems.InterceptedWait.Trigger; +import org.apache.cassandra.simulator.systems.SimulatedTime.LocalTime; import org.apache.cassandra.utils.Shared; import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException; import static org.apache.cassandra.simulator.systems.InterceptedWait.Kind.UNBOUNDED_WAIT; import static org.apache.cassandra.simulator.systems.InterceptedWait.Kind.WAIT_UNTIL; +import static org.apache.cassandra.simulator.systems.InterceptedWait.Trigger.INTERRUPT; +import static org.apache.cassandra.simulator.systems.InterceptedWait.Trigger.SIGNAL; import static org.apache.cassandra.simulator.systems.InterceptibleThread.WaitTimeKind.ABSOLUTE_MILLIS; import static org.apache.cassandra.simulator.systems.InterceptibleThread.WaitTimeKind.NONE; import static org.apache.cassandra.simulator.systems.InterceptibleThread.WaitTimeKind.RELATIVE_NANOS; @@ -40,6 +44,7 @@ import static org.apache.cassandra.utils.Shared.Scope.SIMULATION; @Shared(scope = SIMULATION, ancestors = ALL, members = ALL) public class InterceptibleThread extends FastThreadLocalThread implements InterceptorOfConsequences { + @Shared(scope = SIMULATION) enum WaitTimeKind { NONE, RELATIVE_NANOS, ABSOLUTE_MILLIS @@ -50,15 +55,17 @@ public class InterceptibleThread extends FastThreadLocalThread implements Interc private class Parked implements InterceptedWait { final Kind kind; + final CaptureSites captureSites; final InterceptorOfConsequences waitInterceptedBy; final List onTrigger = new ArrayList<>(3); final long waitTime; - boolean isWakeIntercepted; // we have intercepted a pending wakeup boolean isDone; // we have been signalled (by the simulation or otherwise) + Trigger trigger; - Parked(Kind kind, long waitTime, InterceptorOfConsequences waitInterceptedBy) + Parked(Kind kind, CaptureSites captureSites, long waitTime, InterceptorOfConsequences waitInterceptedBy) { this.kind = kind; + this.captureSites = captureSites; this.waitTime = waitTime; this.waitInterceptedBy = waitInterceptedBy; } @@ -82,7 +89,13 @@ public class InterceptibleThread extends FastThreadLocalThread implements Interc } @Override - public synchronized void triggerAndAwaitDone(InterceptorOfConsequences interceptor, boolean isTimeout) + public boolean isInterruptible() + { + return true; + } + + @Override + public synchronized void triggerAndAwaitDone(InterceptorOfConsequences interceptor, Trigger trigger) { if (parked == null) return; @@ -92,7 +105,8 @@ public class InterceptibleThread extends FastThreadLocalThread implements Interc parked = null; onTrigger.forEach(listener -> listener.onTrigger(this)); - notify(); + if (!preWakeup(this)) + notify(); try { @@ -139,9 +153,9 @@ public class InterceptibleThread extends FastThreadLocalThread implements Interc while (!isTriggered()) wait(); - if (interruptOnWakeup) + if (hasPendingInterrupt) doInterrupt(); - interruptOnWakeup = false; + hasPendingInterrupt = false; } catch (InterruptedException e) { @@ -150,16 +164,22 @@ public class InterceptibleThread extends FastThreadLocalThread implements Interc } } - void interceptWakeIfNotAlready(InterceptibleThread by) + @Override + public void interceptWakeup(Trigger trigger, Thread by) { - if (!isWakeIntercepted) by.interceptor.interceptWakeup(this, waitInterceptedBy); - isWakeIntercepted = true; + if (this.trigger != null && this.trigger.compareTo(trigger) >= 0) + return; + + this.trigger = trigger; + if (captureSites != null) + captureSites.registerWakeup(by); + interceptorOrDefault(by).interceptWakeup(this, trigger, waitInterceptedBy); } @Override public String toString() { - return ""; + return captureSites == null ? "" : captureSites.toString(); } } @@ -169,15 +189,16 @@ public class InterceptibleThread extends FastThreadLocalThread implements Interc final String toString; final Runnable onTermination; private final InterceptorOfGlobalMethods interceptorOfGlobalMethods; - private final SimulatedTime.LocalTime time; + private final LocalTime time; // this is set before the thread's execution begins/continues; events and cessation are reported back to this private InterceptorOfConsequences interceptor; private NotifyThreadPaused notifyOnPause; private boolean hasPendingUnpark; - private boolean interruptOnWakeup; + private boolean hasPendingInterrupt; private Parked parked; + private InterceptedWait waitingOn; volatile boolean trapInterrupts = true; // we need to avoid non-determinism when evaluating things in the debugger and toString() is the main culprit @@ -185,7 +206,7 @@ public class InterceptibleThread extends FastThreadLocalThread implements Interc // perform any non-deterministic actions private int determinismDepth; - public InterceptibleThread(ThreadGroup group, Runnable target, String name, Object extraToStringInfo, Runnable onTermination, InterceptorOfGlobalMethods interceptorOfGlobalMethods, SimulatedTime.LocalTime time) + public InterceptibleThread(ThreadGroup group, Runnable target, String name, Object extraToStringInfo, Runnable onTermination, InterceptorOfGlobalMethods interceptorOfGlobalMethods, LocalTime time) { super(group, target, name); this.onTermination = onTermination; @@ -212,13 +233,15 @@ public class InterceptibleThread extends FastThreadLocalThread implements Interc break; case RELATIVE_NANOS: kind = WAIT_UNTIL; - waitTime = time.localToGlobal(time.relativeNanosToAbsolute(waitTime)); + waitTime = time.localToGlobalNanos(time.relativeToLocalNanos(waitTime)); break; case ABSOLUTE_MILLIS: kind = WAIT_UNTIL; waitTime = time.translate().fromMillisSinceEpoch(waitTime); } - Parked parked = this.parked = new Parked(kind, waitTime, interceptor); + + Parked parked = new Parked(kind, interceptorOfGlobalMethods.captureWaitSite(this), waitTime, interceptor); + this.parked = parked; interceptWait(parked); parked.await(); } @@ -229,17 +252,34 @@ public class InterceptibleThread extends FastThreadLocalThread implements Interc { if (by.interceptor == null) return false; if (parked == null) hasPendingUnpark = true; - else parked.interceptWakeIfNotAlready(by); + else parked.interceptWakeup(SIGNAL, by); return true; } public void trapInterrupts(boolean trapInterrupts) { this.trapInterrupts = trapInterrupts; - if (interruptOnWakeup) + if (!trapInterrupts && hasPendingInterrupt) doInterrupt(); } + public boolean hasPendingInterrupt() + { + return hasPendingInterrupt; + } + + public boolean preWakeup(InterceptedWait wakingOn) + { + assert wakingOn == waitingOn; + waitingOn = null; + if (!hasPendingInterrupt) + return false; + + hasPendingInterrupt = false; + doInterrupt(); + return true; + } + public void doInterrupt() { super.interrupt(); @@ -252,9 +292,9 @@ public class InterceptibleThread extends FastThreadLocalThread implements Interc if (by == this || !(by instanceof InterceptibleThread) || !trapInterrupts) doInterrupt(); else { - interruptOnWakeup = true; - if (parked == null) hasPendingUnpark = true; - else parked.interceptWakeIfNotAlready((InterceptibleThread)by); + hasPendingInterrupt = true; + if (waitingOn != null && waitingOn.isInterruptible()) + waitingOn.interceptWakeup(INTERRUPT, by); } } @@ -281,13 +321,13 @@ public class InterceptibleThread extends FastThreadLocalThread implements Interc } @Override - public void interceptWakeup(InterceptedWait wakeup, InterceptorOfConsequences waitWasInterceptedBy) + public void interceptWakeup(InterceptedWait wakeup, Trigger trigger, InterceptorOfConsequences waitWasInterceptedBy) { ++determinismDepth; try { - interceptor.interceptWakeup(wakeup, waitWasInterceptedBy); - if (debug != null) debug.interceptWakeup(wakeup, waitWasInterceptedBy); + interceptor.interceptWakeup(wakeup, trigger, waitWasInterceptedBy); + if (debug != null) debug.interceptWakeup(wakeup, trigger, waitWasInterceptedBy); } finally { @@ -323,6 +363,7 @@ public class InterceptibleThread extends FastThreadLocalThread implements Interc NotifyThreadPaused notifyOnPause = this.notifyOnPause; this.interceptor = null; this.notifyOnPause = null; + this.waitingOn = wakeupWith; interceptor.interceptWait(wakeupWith); if (debug != null) debug.interceptWait(wakeupWith); @@ -342,9 +383,6 @@ public class InterceptibleThread extends FastThreadLocalThread implements Interc @Override public void interceptTermination(boolean isThreadTermination) { - if (isThreadTermination) - onTermination(); - ++determinismDepth; try { @@ -373,7 +411,7 @@ public class InterceptibleThread extends FastThreadLocalThread implements Interc this.interceptor = interceptor; this.notifyOnPause = notifyOnPause; - this.interceptor.beforeInvocation(this); + interceptor.beforeInvocation(this); } public boolean isEvaluationDeterministic() @@ -495,6 +533,24 @@ public class InterceptibleThread extends FastThreadLocalThread implements Interc LockSupport.unpark(thread); } + public static InterceptorOfConsequences interceptorOrDefault(Thread thread) + { + if (!(thread instanceof InterceptibleThread)) + return DEFAULT_INTERCEPTOR; + + return interceptorOrDefault((InterceptibleThread) thread); + } + + public static InterceptorOfConsequences interceptorOrDefault(InterceptibleThread thread) + { + return thread.isIntercepting() ? thread : DEFAULT_INTERCEPTOR; + } + + public LocalTime time() + { + return time; + } + public static void setDebugInterceptor(InterceptorOfConsequences interceptor) { debug = interceptor; diff --git a/test/simulator/main/org/apache/cassandra/simulator/systems/InterceptingAwaitable.java b/test/simulator/main/org/apache/cassandra/simulator/systems/InterceptingAwaitable.java index 18178a27ef..ef4e24da07 100644 --- a/test/simulator/main/org/apache/cassandra/simulator/systems/InterceptingAwaitable.java +++ b/test/simulator/main/org/apache/cassandra/simulator/systems/InterceptingAwaitable.java @@ -34,8 +34,11 @@ import org.apache.cassandra.utils.concurrent.WaitQueue; import static org.apache.cassandra.simulator.systems.InterceptedWait.Kind.WAIT_UNTIL; import static org.apache.cassandra.simulator.systems.InterceptedWait.Kind.UNBOUNDED_WAIT; +import static org.apache.cassandra.simulator.systems.InterceptedWait.Trigger.SIGNAL; +import static org.apache.cassandra.simulator.systems.InterceptorOfGlobalMethods.Global.captureWaitSite; +import static org.apache.cassandra.simulator.systems.InterceptorOfGlobalMethods.Global.ifIntercepted; import static org.apache.cassandra.simulator.systems.SimulatedTime.Global.localToGlobalNanos; -import static org.apache.cassandra.simulator.systems.SimulatedTime.Global.relativeToAbsoluteNanos; +import static org.apache.cassandra.simulator.systems.SimulatedTime.Global.relativeToLocalNanos; @PerClassLoader abstract class InterceptingAwaitable implements Awaitable @@ -97,21 +100,21 @@ abstract class InterceptingAwaitable implements Awaitable public boolean await(long time, TimeUnit units) throws InterruptedException { - long deadline = relativeToAbsoluteNanos(units.toNanos(time)); + long deadline = relativeToLocalNanos(units.toNanos(time)); maybeInterceptThrowChecked(WAIT_UNTIL, localToGlobalNanos(deadline)).awaitUntil(deadline); return isSignalled(); } public boolean awaitThrowUncheckedOnInterrupt(long time, TimeUnit units) { - long deadline = relativeToAbsoluteNanos(units.toNanos(time)); + long deadline = relativeToLocalNanos(units.toNanos(time)); maybeInterceptThrowUnchecked(WAIT_UNTIL, localToGlobalNanos(deadline)).awaitUntilThrowUncheckedOnInterrupt(deadline); return isSignalled(); } public boolean awaitUninterruptibly(long time, TimeUnit units) { - long deadline = relativeToAbsoluteNanos(units.toNanos(time)); + long deadline = relativeToLocalNanos(units.toNanos(time)); maybeIntercept(WAIT_UNTIL, localToGlobalNanos(deadline)).awaitUntilUninterruptibly(deadline); return isSignalled(); } @@ -119,13 +122,11 @@ abstract class InterceptingAwaitable implements Awaitable @PerClassLoader static class InterceptingCondition extends InterceptingAwaitable implements Condition, TriggerListener { - private final InterceptorOfWaits interceptorOfWaits; final Condition inner = new NotInterceptedSyncCondition(); private List intercepted; - public InterceptingCondition(InterceptorOfWaits interceptorOfWaits) + public InterceptingCondition() { - this.interceptorOfWaits = interceptorOfWaits; } Condition maybeIntercept(InterceptedWait.Kind kind, long waitNanos) @@ -133,11 +134,11 @@ abstract class InterceptingAwaitable implements Awaitable if (inner.isSignalled()) return inner; - InterceptibleThread thread = interceptorOfWaits.ifIntercepted(); + InterceptibleThread thread = ifIntercepted(); if (thread == null) return inner; - InterceptedConditionWait signal = new InterceptedConditionWait(kind, waitNanos, thread, interceptorOfWaits.captureWaitSite(thread), inner); + InterceptedConditionWait signal = new InterceptedConditionWait(kind, waitNanos, thread, captureWaitSite(thread), inner); synchronized (this) { if (intercepted == null) @@ -165,10 +166,7 @@ abstract class InterceptingAwaitable implements Awaitable if (intercepted != null) { Thread signalledBy = Thread.currentThread(); - intercepted.forEach(signal -> { - // TODO (cleanup): make captureSites and interceptedBy methods of InterceptedWait? - interceptorOfWaits.interceptSignal(signalledBy, signal, signal.captureSites, signal.interceptedBy); - }); + intercepted.forEach(signal -> signal.interceptWakeup(SIGNAL, signalledBy)); } } } @@ -185,9 +183,9 @@ abstract class InterceptingAwaitable implements Awaitable { private final AtomicInteger count; - public InterceptingCountDownLatch(InterceptorOfWaits interceptor, int count) + public InterceptingCountDownLatch(int count) { - super(interceptor); + super(); this.count = new AtomicInteger(count); } @@ -206,7 +204,6 @@ abstract class InterceptingAwaitable implements Awaitable @PerClassLoader static class InterceptingSignal extends InterceptingAwaitable implements WaitQueue.Signal { - final InterceptorOfWaits interceptorOfWaits; final Condition inner = new NotInterceptedSyncCondition(); final V supplyOnDone; final Consumer receiveOnDone; @@ -216,14 +213,13 @@ abstract class InterceptingAwaitable implements Awaitable boolean isSignalled; boolean isCancelled; - InterceptingSignal(InterceptorOfWaits interceptorOfWaits) + InterceptingSignal() { - this(interceptorOfWaits, null, ignore -> {}); + this(null, ignore -> {}); } - InterceptingSignal(InterceptorOfWaits interceptorOfWaits, V supplyOnDone, Consumer receiveOnDone) + InterceptingSignal(V supplyOnDone, Consumer receiveOnDone) { - this.interceptorOfWaits = interceptorOfWaits; this.supplyOnDone = supplyOnDone; this.receiveOnDone = receiveOnDone; } @@ -257,10 +253,7 @@ abstract class InterceptingAwaitable implements Awaitable receiveOnDone.accept(supplyOnDone); inner.signal(); if (intercepted != null && !intercepted.isTriggered()) - { - Thread thread = Thread.currentThread(); - interceptorOfWaits.interceptSignal(thread, intercepted, intercepted.captureSites, intercepted.interceptedBy); - } + intercepted.interceptWakeup(SIGNAL, Thread.currentThread()); return true; } @@ -284,11 +277,11 @@ abstract class InterceptingAwaitable implements Awaitable assert intercepted == null; assert !inner.isSignalled(); - InterceptibleThread thread = interceptorOfWaits.ifIntercepted(); + InterceptibleThread thread = ifIntercepted(); if (thread == null) return inner; - intercepted = new InterceptedConditionWait(kind, waitNanos, thread, interceptorOfWaits.captureWaitSite(thread), inner); + intercepted = new InterceptedConditionWait(kind, waitNanos, thread, captureWaitSite(thread), inner); thread.interceptWait(intercepted); return intercepted; } diff --git a/test/simulator/main/org/apache/cassandra/simulator/systems/InterceptingExecutor.java b/test/simulator/main/org/apache/cassandra/simulator/systems/InterceptingExecutor.java index fe3712ec84..26f096e84d 100644 --- a/test/simulator/main/org/apache/cassandra/simulator/systems/InterceptingExecutor.java +++ b/test/simulator/main/org/apache/cassandra/simulator/systems/InterceptingExecutor.java @@ -35,7 +35,8 @@ import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; -import io.netty.util.concurrent.FastThreadLocal; +import com.google.common.base.Preconditions; + import org.apache.cassandra.concurrent.ExecutorPlus; import org.apache.cassandra.concurrent.LocalAwareExecutorPlus; import org.apache.cassandra.concurrent.LocalAwareSequentialExecutorPlus; @@ -44,19 +45,20 @@ import org.apache.cassandra.concurrent.SequentialExecutorPlus; import org.apache.cassandra.concurrent.SingleThreadExecutorPlus; import org.apache.cassandra.concurrent.SyncFutureTask; import org.apache.cassandra.concurrent.TaskFactory; -import org.apache.cassandra.config.CassandraRelevantProperties; import org.apache.cassandra.simulator.OrderOn; -import org.apache.cassandra.simulator.OrderOn.OrderAppliesAfterScheduling; +import org.apache.cassandra.simulator.systems.InterceptingAwaitable.InterceptingCondition; import org.apache.cassandra.simulator.systems.NotifyThreadPaused.AwaitPaused; import org.apache.cassandra.utils.Shared; import org.apache.cassandra.utils.WithResources; import org.apache.cassandra.utils.concurrent.Condition; -import org.apache.cassandra.utils.concurrent.Condition.Sync; import org.apache.cassandra.utils.concurrent.Future; +import org.apache.cassandra.utils.concurrent.ImmediateFuture; +import org.apache.cassandra.utils.concurrent.NotScheduledFuture; import org.apache.cassandra.utils.concurrent.RunnableFuture; import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException; import static java.util.Collections.newSetFromMap; +import static java.util.Collections.synchronizedMap; import static java.util.Collections.synchronizedSet; import static java.util.concurrent.TimeUnit.NANOSECONDS; import static org.apache.cassandra.config.CassandraRelevantProperties.TEST_SIMULATOR_DEBUG; @@ -65,12 +67,11 @@ import static org.apache.cassandra.simulator.systems.SimulatedAction.Kind.SCHEDU import static org.apache.cassandra.simulator.systems.SimulatedAction.Kind.SCHEDULED_TASK; import static org.apache.cassandra.simulator.systems.SimulatedAction.Kind.SCHEDULED_TIMEOUT; import static org.apache.cassandra.simulator.systems.SimulatedExecution.callable; -import static org.apache.cassandra.simulator.systems.SimulatedTime.Global.absoluteToRelativeNanos; +import static org.apache.cassandra.simulator.systems.SimulatedTime.Global.localToRelativeNanos; import static org.apache.cassandra.simulator.systems.SimulatedTime.Global.localToGlobalNanos; -import static org.apache.cassandra.simulator.systems.SimulatedTime.Global.relativeToGlobalAbsoluteNanos; +import static org.apache.cassandra.simulator.systems.SimulatedTime.Global.relativeToGlobalNanos; import static org.apache.cassandra.utils.Shared.Recursive.INTERFACES; import static org.apache.cassandra.utils.Shared.Scope.SIMULATION; -import static org.apache.commons.collections.MapUtils.synchronizedMap; // An executor whose tasks we can intercept the execution of @Shared(scope = SIMULATION, inner = INTERFACES) @@ -141,13 +142,14 @@ public interface InterceptingExecutor extends OrderOn final InterceptingTaskFactory taskFactory; final Set debugPending = TEST_SIMULATOR_DEBUG.getBoolean() ? synchronizedSet(newSetFromMap(new IdentityHashMap<>())) : null; - final Condition isTerminated = new Sync(); + final Condition isTerminated; volatile boolean isShutdown; volatile int pending; protected AbstractInterceptingExecutor(InterceptorOfExecution interceptorOfExecution, InterceptingTaskFactory taskFactory) { this.interceptorOfExecution = interceptorOfExecution; + this.isTerminated = new InterceptingCondition(); this.taskFactory = taskFactory; } @@ -254,11 +256,6 @@ public interface InterceptingExecutor extends OrderOn abstract void terminate(); - protected void onThreadTermination() - { - FastThreadLocal.removeAll(); - } - public boolean isShutdown() { return isShutdown; @@ -271,6 +268,17 @@ public interface InterceptingExecutor extends OrderOn public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException { + Thread thread = Thread.currentThread(); + if (thread instanceof InterceptibleThread) + { + InterceptibleThread interceptibleThread = (InterceptibleThread) thread; + if (interceptibleThread.isIntercepting()) + { + // simpler to use no timeout than to ensure pending tasks all run first in simulation + isTerminated.await(); + return true; + } + } return isTerminated.await(timeout, unit); } @@ -321,6 +329,7 @@ public interface InterceptingExecutor extends OrderOn if (shutdown && remaining < threads.size()) { threads.remove(thread); + thread.onTermination(); if (threads.isEmpty()) isTerminated.signal(); // this has simulator side-effects, so try to perform before we interceptTermination thread.interceptTermination(true); @@ -453,6 +462,7 @@ public interface InterceptingExecutor extends OrderOn WaitingThread next; while (null != (next = waiting.poll())) next.terminate(); + if (pending == 0) terminate(); } @@ -547,13 +557,9 @@ public interface InterceptingExecutor extends OrderOn } } - enum ThreadState { WAITING, EXECUTING, TERMINATING, TERMINATED } - final InterceptibleThread thread; final ArrayDeque queue = new ArrayDeque<>(); - volatile boolean executing; - volatile boolean terminating; - volatile boolean terminated; + volatile boolean executing, terminating, terminated; AbstractSingleThreadedExecutorPlus(InterceptorOfExecution interceptorOfExecution, ThreadFactory threadFactory, InterceptingTaskFactory taskFactory) { @@ -568,12 +574,11 @@ public interface InterceptingExecutor extends OrderOn try { task = dequeue(); - if (task == null) - return; } catch (InterruptedException | UncheckedInterruptedException ignore) { - continue; + if (terminating) return; + else continue; } try @@ -589,47 +594,52 @@ public interface InterceptingExecutor extends OrderOn executing = false; boolean shutdown = isShutdown; if ((0 == completePending(task) && shutdown)) - { - isTerminated.signal(); // this has simulator side-effects, so try to perform before we interceptTermination - thread.interceptTermination(true); return; - } thread.interceptTermination(false); } } finally { - synchronized (this) + runDeterministic(thread::onTermination); + if (terminating) { - terminated = true; - notifyAll(); + synchronized (this) + { + terminated = true; + notifyAll(); + } } - if (!isTerminated()) + else { - runDeterministic(() -> { - isTerminated.signal(); - thread.interceptTermination(true); - }); + runDeterministic(this::terminate); } } }); thread.start(); } - synchronized void terminate() + void terminate() { - terminating = true; - notifyAll(); - try + synchronized (this) { - while (!terminated) - wait(); - } - catch (InterruptedException e) - { - throw new UncheckedInterruptedException(e); + assert pending == 0; + if (terminating) + return; + + terminating = true; + if (Thread.currentThread() != thread) + { + notifyAll(); + try { while (!terminated) wait(); } + catch (InterruptedException e) { throw new UncheckedInterruptedException(e); } + } + terminated = true; } + + isTerminated.signal(); // this has simulator side-effects, so try to perform before we interceptTermination + if (Thread.currentThread() == thread && thread.isIntercepting()) + thread.interceptTermination(true); } public synchronized void shutdown() @@ -651,8 +661,8 @@ public interface InterceptingExecutor extends OrderOn List cancelled = new ArrayList<>(queue); queue.clear(); cancelled.forEach(super::cancelPending); - if (pending > 0) thread.interrupt(); - else terminate(); + if (pending == 0) terminate(); + else thread.interrupt(); return cancelled; } @@ -664,9 +674,14 @@ public interface InterceptingExecutor extends OrderOn synchronized Runnable dequeue() throws InterruptedException { - while (queue.isEmpty() && !terminating) + Runnable next; + while (null == (next = queue.poll()) && !terminating) wait(); - return queue.poll(); + + if (next == null) + throw new InterruptedException(); + + return next; } public AtLeastOnce atLeastOnceTrigger(Runnable run) @@ -711,7 +726,7 @@ public interface InterceptingExecutor extends OrderOn synchronized (this) { // we don't check isShutdown as we could have a task queued by simulation from prior to shutdown - if (isTerminated()) throw new AssertionError(); + if (terminated) throw new AssertionError(); if (executing) throw new AssertionError(); if (debugPending != null && !debugPending.contains(task)) throw new AssertionError(); executing = true; @@ -751,7 +766,7 @@ public interface InterceptingExecutor extends OrderOn throw new RejectedExecutionException(); long delayNanos = unit.toNanos(delay); - return interceptorOfExecution.intercept().schedule(SCHEDULED_TASK, delayNanos, relativeToGlobalAbsoluteNanos(delayNanos), callable(run, null), this); + return interceptorOfExecution.intercept().schedule(SCHEDULED_TASK, delayNanos, relativeToGlobalNanos(delayNanos), callable(run, null), this); } public ScheduledFuture schedule(Callable callable, long delay, TimeUnit unit) @@ -760,12 +775,12 @@ public interface InterceptingExecutor extends OrderOn throw new RejectedExecutionException(); long delayNanos = unit.toNanos(delay); - return interceptorOfExecution.intercept().schedule(SCHEDULED_TASK, delayNanos, relativeToGlobalAbsoluteNanos(delayNanos), callable, this); + return interceptorOfExecution.intercept().schedule(SCHEDULED_TASK, delayNanos, relativeToGlobalNanos(delayNanos), callable, this); } public ScheduledFuture scheduleTimeoutWithDelay(Runnable run, long delay, TimeUnit unit) { - return scheduleTimeoutAt(run, relativeToGlobalAbsoluteNanos(unit.toNanos(delay))); + return scheduleTimeoutAt(run, relativeToGlobalNanos(unit.toNanos(delay))); } public ScheduledFuture scheduleAt(Runnable run, long deadlineNanos) @@ -773,7 +788,7 @@ public interface InterceptingExecutor extends OrderOn if (isShutdown) throw new RejectedExecutionException(); - return interceptorOfExecution.intercept().schedule(SCHEDULED_TASK, absoluteToRelativeNanos(deadlineNanos), localToGlobalNanos(deadlineNanos), callable(run, null), this); + return interceptorOfExecution.intercept().schedule(SCHEDULED_TASK, localToRelativeNanos(deadlineNanos), localToGlobalNanos(deadlineNanos), callable(run, null), this); } public ScheduledFuture scheduleTimeoutAt(Runnable run, long deadlineNanos) @@ -781,7 +796,7 @@ public interface InterceptingExecutor extends OrderOn if (isShutdown) throw new RejectedExecutionException(); - return interceptorOfExecution.intercept().schedule(SCHEDULED_TIMEOUT, absoluteToRelativeNanos(deadlineNanos), localToGlobalNanos(deadlineNanos), callable(run, null), this); + return interceptorOfExecution.intercept().schedule(SCHEDULED_TIMEOUT, localToRelativeNanos(deadlineNanos), localToGlobalNanos(deadlineNanos), callable(run, null), this); } public ScheduledFuture scheduleSelfRecurring(Runnable run, long delay, TimeUnit unit) @@ -790,7 +805,7 @@ public interface InterceptingExecutor extends OrderOn throw new RejectedExecutionException(); long delayNanos = unit.toNanos(delay); - return interceptorOfExecution.intercept().schedule(SCHEDULED_DAEMON, delayNanos, relativeToGlobalAbsoluteNanos(delayNanos), callable(run, null), this); + return interceptorOfExecution.intercept().schedule(SCHEDULED_DAEMON, delayNanos, relativeToGlobalNanos(delayNanos), callable(run, null), this); } public ScheduledFuture scheduleAtFixedRate(Runnable run, long initialDelay, long period, TimeUnit unit) @@ -799,13 +814,14 @@ public interface InterceptingExecutor extends OrderOn throw new RejectedExecutionException(); long delayNanos = unit.toNanos(initialDelay); - return interceptorOfExecution.intercept().schedule(SCHEDULED_DAEMON, delayNanos, relativeToGlobalAbsoluteNanos(delayNanos), new Callable() + return interceptorOfExecution.intercept().schedule(SCHEDULED_DAEMON, delayNanos, relativeToGlobalNanos(delayNanos), new Callable() { @Override public Object call() { run.run(); - scheduleAtFixedRate(run, period, period, unit); + if (!isShutdown) + scheduleAtFixedRate(run, period, period, unit); return null; } @@ -845,4 +861,202 @@ public interface InterceptingExecutor extends OrderOn super(interceptorOfExecution, threadFactory, taskFactory); } } + + @PerClassLoader + static class DiscardingSequentialExecutor implements LocalAwareSequentialExecutorPlus, ScheduledExecutorPlus + { + @Override + public void shutdown() + { + } + + @Override + public List shutdownNow() + { + return Collections.emptyList(); + } + + @Override + public boolean isShutdown() + { + return false; + } + + @Override + public boolean isTerminated() + { + return false; + } + + @Override + public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException + { + return false; + } + + @Override + public Future submit(Callable task) + { + return ImmediateFuture.cancelled(); + } + + @Override + public Future submit(Runnable task, T result) + { + return ImmediateFuture.cancelled(); + } + + @Override + public Future submit(Runnable task) + { + return ImmediateFuture.cancelled(); + } + + @Override + public void execute(WithResources withResources, Runnable task) + { + } + + @Override + public Future submit(WithResources withResources, Callable task) + { + return ImmediateFuture.cancelled(); + } + + @Override + public Future submit(WithResources withResources, Runnable task) + { + return ImmediateFuture.cancelled(); + } + + @Override + public Future submit(WithResources withResources, Runnable task, T result) + { + return ImmediateFuture.cancelled(); + } + + @Override + public boolean inExecutor() + { + return false; + } + + @Override + public int getCorePoolSize() + { + return 0; + } + + @Override + public void setCorePoolSize(int newCorePoolSize) + { + + } + + @Override + public int getMaximumPoolSize() + { + return 0; + } + + @Override + public void setMaximumPoolSize(int newMaximumPoolSize) + { + + } + + @Override + public int getActiveTaskCount() + { + return 0; + } + + @Override + public long getCompletedTaskCount() + { + return 0; + } + + @Override + public int getPendingTaskCount() + { + return 0; + } + + @Override + public AtLeastOnceTrigger atLeastOnceTrigger(Runnable runnable) + { + return new AtLeastOnceTrigger() + { + @Override + public boolean trigger() + { + return false; + } + + @Override + public void runAfter(Runnable run) + { + } + + @Override + public void sync() + { + } + }; + } + + @Override + public void execute(Runnable command) + { + } + + @Override + public ScheduledFuture scheduleSelfRecurring(Runnable run, long delay, TimeUnit units) + { + return new NotScheduledFuture<>(); + } + + @Override + public ScheduledFuture scheduleAt(Runnable run, long deadline) + { + return new NotScheduledFuture<>(); + } + + @Override + public ScheduledFuture scheduleTimeoutAt(Runnable run, long deadline) + { + return new NotScheduledFuture<>(); + } + + @Override + public ScheduledFuture scheduleTimeoutWithDelay(Runnable run, long delay, TimeUnit units) + { + return new NotScheduledFuture<>(); + } + + @Override + public ScheduledFuture schedule(Runnable command, long delay, TimeUnit unit) + { + return new NotScheduledFuture<>(); + } + + @Override + public ScheduledFuture schedule(Callable callable, long delay, TimeUnit unit) + { + return new NotScheduledFuture<>(); + } + + @Override + public ScheduledFuture scheduleAtFixedRate(Runnable command, long initialDelay, long period, TimeUnit unit) + { + return new NotScheduledFuture<>(); + } + + @Override + public ScheduledFuture scheduleWithFixedDelay(Runnable command, long initialDelay, long delay, TimeUnit unit) + { + return new NotScheduledFuture<>(); + } + } } diff --git a/test/simulator/main/org/apache/cassandra/simulator/systems/InterceptingExecutorFactory.java b/test/simulator/main/org/apache/cassandra/simulator/systems/InterceptingExecutorFactory.java index 6694c77047..a7915e1d6e 100644 --- a/test/simulator/main/org/apache/cassandra/simulator/systems/InterceptingExecutorFactory.java +++ b/test/simulator/main/org/apache/cassandra/simulator/systems/InterceptingExecutorFactory.java @@ -25,6 +25,7 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.RejectedExecutionHandler; import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; +import java.util.function.BiFunction; import java.util.function.Consumer; import java.util.function.Supplier; @@ -39,6 +40,7 @@ import org.apache.cassandra.concurrent.InfiniteLoopExecutor; import org.apache.cassandra.concurrent.InfiniteLoopExecutor.Daemon; import org.apache.cassandra.concurrent.InfiniteLoopExecutor.Interrupts; import org.apache.cassandra.concurrent.InfiniteLoopExecutor.SimulatorSafe; +import org.apache.cassandra.concurrent.Interruptible.Task; import org.apache.cassandra.concurrent.LocalAwareExecutorPlus; import org.apache.cassandra.concurrent.LocalAwareSequentialExecutorPlus; import org.apache.cassandra.concurrent.ScheduledExecutorPlus; @@ -47,12 +49,17 @@ import org.apache.cassandra.concurrent.Interruptible; import org.apache.cassandra.concurrent.SyncFutureTask; import org.apache.cassandra.concurrent.TaskFactory; import org.apache.cassandra.distributed.api.IIsolatedExecutor; +import org.apache.cassandra.distributed.api.IIsolatedExecutor.QuadFunction; import org.apache.cassandra.distributed.api.IIsolatedExecutor.SerializableBiFunction; +import org.apache.cassandra.distributed.api.IIsolatedExecutor.SerializableCallable; import org.apache.cassandra.distributed.api.IIsolatedExecutor.SerializableFunction; +import org.apache.cassandra.distributed.api.IIsolatedExecutor.SerializableQuadFunction; import org.apache.cassandra.distributed.api.IIsolatedExecutor.SerializableRunnable; +import org.apache.cassandra.distributed.api.IIsolatedExecutor.SerializableSupplier; import org.apache.cassandra.distributed.impl.IsolatedExecutor; import org.apache.cassandra.simulator.systems.InterceptibleThreadFactory.ConcreteInterceptibleThreadFactory; import org.apache.cassandra.simulator.systems.InterceptibleThreadFactory.PlainThreadFactory; +import org.apache.cassandra.simulator.systems.InterceptingExecutor.DiscardingSequentialExecutor; import org.apache.cassandra.simulator.systems.InterceptingExecutor.InterceptingTaskFactory; import org.apache.cassandra.simulator.systems.InterceptingExecutor.InterceptingLocalAwareSequentialExecutor; import org.apache.cassandra.simulator.systems.InterceptingExecutor.InterceptingPooledExecutor; @@ -60,6 +67,7 @@ import org.apache.cassandra.simulator.systems.InterceptingExecutor.InterceptingP import org.apache.cassandra.simulator.systems.InterceptingExecutor.InterceptingSequentialExecutor; import org.apache.cassandra.simulator.systems.InterceptorOfExecution.InterceptExecution; import org.apache.cassandra.simulator.systems.SimulatedTime.LocalTime; +import org.apache.cassandra.utils.Closeable; import org.apache.cassandra.utils.JVMStabilityInspector; import org.apache.cassandra.utils.WithResources; import org.apache.cassandra.utils.concurrent.RunnableFuture; @@ -67,7 +75,7 @@ import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException; import static org.apache.cassandra.simulator.systems.SimulatedAction.Kind.INFINITE_LOOP; -public class InterceptingExecutorFactory implements ExecutorFactory +public class InterceptingExecutorFactory implements ExecutorFactory, Closeable { static class StandardSyncTaskFactory extends TaskFactory.Standard implements InterceptingTaskFactory, Serializable { @@ -221,15 +229,12 @@ public class InterceptingExecutorFactory implements ExecutorFactory F factory(String name, Object extraInfo, ThreadGroup threadGroup, UncaughtExceptionHandler uncaughtExceptionHandler, InterceptibleThreadFactory.MetaFactory factory) { if (uncaughtExceptionHandler == null) - uncaughtExceptionHandler = transferToInstance.apply((SerializableFunction, UncaughtExceptionHandler>)(reportUnchecked) -> (thread, throwable) -> { - if (!(throwable instanceof UncheckedInterruptedException) || reportUnchecked.get()) - JVMStabilityInspector.uncaughtException(thread, throwable); - }).apply(() -> !isClosed); + uncaughtExceptionHandler = transferToInstance.apply((SerializableSupplier)() -> InterceptorOfGlobalMethods.Global::uncaughtException).get(); if (threadGroup == null) threadGroup = this.threadGroup; else if (!this.threadGroup.parentOf(threadGroup)) throw new IllegalArgumentException(); Runnable onTermination = transferToInstance.apply((SerializableRunnable)FastThreadLocal::removeAll); - LocalTime time = transferToInstance.apply((IIsolatedExecutor.SerializableCallable) SimulatedTime.Global::current).call(); + LocalTime time = transferToInstance.apply((SerializableCallable) SimulatedTime.Global::current).call(); return factory.create(name, Thread.NORM_PRIORITY, classLoader, uncaughtExceptionHandler, threadGroup, onTermination, time, this, extraInfo); } @@ -307,9 +312,16 @@ public class InterceptingExecutorFactory implements ExecutorFactory } @Override - public ScheduledExecutorPlus scheduled(boolean executeOnShutdown, String name, int priority) + public ScheduledExecutorPlus scheduled(boolean executeOnShutdown, String name, int priority, SimulatorSemantics simulatorSemantics) { - return transferToInstance.apply((SerializableBiFunction) (interceptSupplier, threadFactory) -> new InterceptingSequentialExecutor(interceptSupplier, threadFactory, new StandardSyncTaskFactory())).apply(simulatedExecution, factory(name)); + switch (simulatorSemantics) + { + default: throw new AssertionError(); + case NORMAL: + return transferToInstance.apply((SerializableBiFunction) (interceptSupplier, threadFactory) -> new InterceptingSequentialExecutor(interceptSupplier, threadFactory, new StandardSyncTaskFactory())).apply(simulatedExecution, factory(name)); + case DISCARD: + return transferToInstance.apply((SerializableSupplier) DiscardingSequentialExecutor::new).get(); + } } @Override @@ -334,10 +346,11 @@ public class InterceptingExecutorFactory implements ExecutorFactory } @Override - public Interruptible infiniteLoop(String name, Interruptible.Task task, SimulatorSafe simulatorSafe, Daemon daemon, Interrupts interrupts) + public Interruptible infiniteLoop(String name, Task task, SimulatorSafe simulatorSafe, Daemon daemon, Interrupts interrupts) { if (simulatorSafe != SimulatorSafe.SAFE) { + // avoid use rewritten classes here (so use system class loader's ILE), as we cannot fully control the thread's execution return new InfiniteLoopExecutor((n, t) -> { Thread thread = plainFactory(n, t, threadGroup, null).newThread(t); thread.start(); @@ -346,8 +359,8 @@ public class InterceptingExecutorFactory implements ExecutorFactory } InterceptExecution interceptor = simulatedExecution.intercept(); - return new InfiniteLoopExecutor((n, r) -> interceptor.start(INFINITE_LOOP, factory(n, task)::newThread, r), - name, task, interrupts); + return transferToInstance.apply((SerializableQuadFunction, String, Task, Interrupts, Interruptible>)InfiniteLoopExecutor::new) + .apply((n, r) -> interceptor.start(INFINITE_LOOP, factory(n, task)::newThread, r), name, task, interrupts); } @Override diff --git a/test/simulator/main/org/apache/cassandra/simulator/systems/InterceptingGlobalMethods.java b/test/simulator/main/org/apache/cassandra/simulator/systems/InterceptingGlobalMethods.java index 74909ee09e..45a64f4241 100644 --- a/test/simulator/main/org/apache/cassandra/simulator/systems/InterceptingGlobalMethods.java +++ b/test/simulator/main/org/apache/cassandra/simulator/systems/InterceptingGlobalMethods.java @@ -19,61 +19,140 @@ package org.apache.cassandra.simulator.systems; import java.util.UUID; +import java.util.function.Consumer; +import java.util.function.LongConsumer; + +import javax.annotation.Nullable; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.apache.cassandra.simulator.RandomSource; +import org.apache.cassandra.simulator.systems.InterceptedWait.CaptureSites.Capture; +import org.apache.cassandra.simulator.systems.InterceptedWait.InterceptedConditionWait; +import org.apache.cassandra.utils.Clock; import org.apache.cassandra.utils.concurrent.Condition; import org.apache.cassandra.utils.concurrent.CountDownLatch; import org.apache.cassandra.utils.concurrent.WaitQueue; +import static org.apache.cassandra.config.CassandraRelevantProperties.TEST_SIMULATOR_DETERMINISM_CHECK; +import static org.apache.cassandra.simulator.SimulatorUtils.failWithOOM; import static org.apache.cassandra.simulator.systems.InterceptedWait.Kind.NEMESIS; +import static org.apache.cassandra.simulator.systems.NonInterceptible.Permit.OPTIONAL; +import static org.apache.cassandra.simulator.systems.NonInterceptible.Permit.REQUIRED; @PerClassLoader public class InterceptingGlobalMethods extends InterceptingMonitors implements InterceptorOfGlobalMethods { + private static final Logger logger = LoggerFactory.getLogger(InterceptingGlobalMethods.class); + private static final boolean isDeterminismCheckStrict = TEST_SIMULATOR_DETERMINISM_CHECK.convert(name -> name.equals("strict")); + + private final @Nullable LongConsumer onThreadLocalRandomCheck; + private final Capture capture; private int uniqueUuidCounter = 0; - public InterceptingGlobalMethods(InterceptorOfWaits interceptorOfWaits, RandomSource random) + private final Consumer onUncaughtException; + + public InterceptingGlobalMethods(Capture capture, LongConsumer onThreadLocalRandomCheck, Consumer onUncaughtException, RandomSource random) { - super(interceptorOfWaits, random); + super(random); + this.capture = capture.any() ? capture : null; + this.onThreadLocalRandomCheck = onThreadLocalRandomCheck; + this.onUncaughtException = onUncaughtException; } @Override public WaitQueue newWaitQueue() { - return new InterceptingWaitQueue(interceptorOfWaits); + return new InterceptingWaitQueue(); } @Override public CountDownLatch newCountDownLatch(int count) { - return new InterceptingAwaitable.InterceptingCountDownLatch(interceptorOfWaits, count); + return new InterceptingAwaitable.InterceptingCountDownLatch(count); } @Override public Condition newOneTimeCondition() { - return new InterceptingAwaitable.InterceptingCondition(interceptorOfWaits); + return new InterceptingAwaitable.InterceptingCondition(); + } + + @Override + public InterceptedWait.CaptureSites captureWaitSite(Thread thread) + { + if (capture == null) + return null; + + return new InterceptedWait.CaptureSites(thread, capture); + } + + @Override + public InterceptibleThread ifIntercepted() + { + Thread thread = Thread.currentThread(); + if (thread instanceof InterceptibleThread) + { + InterceptibleThread interceptibleThread = (InterceptibleThread) thread; + if (interceptibleThread.isIntercepting()) + return interceptibleThread; + } + + if (NonInterceptible.isPermitted()) + return null; + + if (!disabled) + throw failWithOOM(); + + return null; + } + + @Override + public void uncaughtException(Thread thread, Throwable throwable) + { + onUncaughtException.accept(throwable); } @Override public void nemesis(float chance) { - InterceptibleThread thread = interceptorOfWaits.ifIntercepted(); + InterceptibleThread thread = ifIntercepted(); if (thread == null || thread.isEvaluationDeterministic() || !random.decide(chance)) return; - InterceptedWait.InterceptedConditionWait signal = new InterceptedWait.InterceptedConditionWait(NEMESIS, 0L, thread, interceptorOfWaits.captureWaitSite(thread), null); + InterceptedConditionWait signal = new InterceptedConditionWait(NEMESIS, 0L, thread, captureWaitSite(thread), null); thread.interceptWait(signal); // save interrupt state to restore afterwards - new ones only arrive if terminating simulation - boolean wasInterrupted = Thread.interrupted(); - signal.awaitThrowUncheckedOnInterrupt(); - if (wasInterrupted) thread.interrupt(); + boolean restoreInterrupt = Thread.interrupted(); + try + { + while (true) + { + try + { + signal.awaitDeclaredUninterruptible(); + return; + } + catch (InterruptedException e) + { + restoreInterrupt = true; + if (disabled) + return; + } + } + } + finally + { + if (restoreInterrupt) + thread.interrupt(); + } } @Override public long randomSeed() { - InterceptibleThread thread = interceptorOfWaits.ifIntercepted(); + InterceptibleThread thread = ifIntercepted(); if (thread == null || thread.isEvaluationDeterministic()) return Thread.currentThread().getName().hashCode(); @@ -87,4 +166,60 @@ public class InterceptingGlobalMethods extends InterceptingMonitors implements I msb = ((msb << 4) & 0xffffffffffff0000L) | 0x4000 | (msb & 0xfff); return new UUID(msb, (1L << 63) | uniqueUuidCounter++); } + + @Override + public void threadLocalRandomCheck(long seed) + { + if (onThreadLocalRandomCheck != null) + onThreadLocalRandomCheck.accept(seed); + } + + public static class ThreadLocalRandomCheck implements LongConsumer + { + final LongConsumer wrapped; + private boolean disabled; + + public ThreadLocalRandomCheck(LongConsumer wrapped) + { + this.wrapped = wrapped; + } + + @Override + public void accept(long value) + { + if (wrapped != null) + wrapped.accept(value); + + Thread thread = Thread.currentThread(); + if (thread instanceof InterceptibleThread) + { + InterceptibleThread interceptibleThread = (InterceptibleThread) thread; + if (interceptibleThread.isIntercepting()) + return; + } + + if (NonInterceptible.isPermitted(isDeterminismCheckStrict ? OPTIONAL : REQUIRED)) + return; + + if (!disabled) + throw failWithOOM(); + } + + public void stop() + { + disabled = true; + } + } + + @Override + public long nanoTime() + { + return Clock.Global.nanoTime(); + } + + @Override + public long currentTimeMillis() + { + return Clock.Global.currentTimeMillis(); + } } diff --git a/test/simulator/main/org/apache/cassandra/simulator/systems/InterceptingMonitors.java b/test/simulator/main/org/apache/cassandra/simulator/systems/InterceptingMonitors.java index 54cfe197d3..eab35de07a 100644 --- a/test/simulator/main/org/apache/cassandra/simulator/systems/InterceptingMonitors.java +++ b/test/simulator/main/org/apache/cassandra/simulator/systems/InterceptingMonitors.java @@ -18,7 +18,9 @@ package org.apache.cassandra.simulator.systems; +import java.util.ArrayDeque; import java.util.ArrayList; +import java.util.Deque; import java.util.IdentityHashMap; import java.util.List; import java.util.Map; @@ -37,21 +39,24 @@ import org.apache.cassandra.utils.concurrent.Threads; import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException; import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static org.apache.cassandra.config.CassandraRelevantProperties.TEST_SIMULATOR_DEBUG; import static org.apache.cassandra.simulator.SimulatorUtils.failWithOOM; import static org.apache.cassandra.simulator.systems.InterceptedWait.Kind.NEMESIS; import static org.apache.cassandra.simulator.systems.InterceptedWait.Kind.SLEEP_UNTIL; import static org.apache.cassandra.simulator.systems.InterceptedWait.Kind.UNBOUNDED_WAIT; import static org.apache.cassandra.simulator.systems.InterceptedWait.Kind.WAIT_UNTIL; +import static org.apache.cassandra.simulator.systems.InterceptedWait.Trigger.SIGNAL; +import static org.apache.cassandra.simulator.systems.InterceptibleThread.interceptorOrDefault; import static org.apache.cassandra.simulator.systems.InterceptingMonitors.WaitListAccessor.LOCK; import static org.apache.cassandra.simulator.systems.InterceptingMonitors.WaitListAccessor.NOTIFY; -import static org.apache.cassandra.simulator.systems.SimulatedTime.Global.relativeToGlobalAbsoluteNanos; -import static org.apache.cassandra.utils.Clock.Global.nanoTime; +import static org.apache.cassandra.simulator.systems.SimulatedTime.Global.relativeToGlobalNanos; @PerClassLoader @SuppressWarnings("SynchronizationOnLocalVariableOrMethodParameter") public abstract class InterceptingMonitors implements InterceptorOfGlobalMethods, Closeable { private static final Logger logger = LoggerFactory.getLogger(InterceptingMonitors.class); + private static final boolean DEBUG_MONITOR_STATE = TEST_SIMULATOR_DEBUG.getBoolean(); static class MonitorState { @@ -66,6 +71,7 @@ public abstract class InterceptingMonitors implements InterceptorOfGlobalMethods InterceptibleThread heldBy; int depth; int suspended; + Deque recentActions = DEBUG_MONITOR_STATE ? new ArrayDeque<>() : null; boolean isEmpty() { @@ -94,6 +100,7 @@ public abstract class InterceptingMonitors implements InterceptorOfGlobalMethods { InterceptedMonitorWait head = remove.waitingOn.head(this); remove.waitingOn.setHead(this, head.remove(remove)); + assert remove.next == null; } } @@ -149,6 +156,7 @@ public abstract class InterceptingMonitors implements InterceptorOfGlobalMethods else { list.setHead(this, wait); + wait.nextLength = 0; } } @@ -165,10 +173,41 @@ public abstract class InterceptingMonitors implements InterceptorOfGlobalMethods { assert heldBy == null || heldBy == wait.waiting; assert depth == 0; + assert suspended > 0; heldBy = wait.waiting; depth = wait.unsuspendMonitor(); --suspended; } + + void claim(InterceptedMonitorWait wait) + { + assert heldBy == null || heldBy == wait.waiting; + assert depth == 0; + heldBy = wait.waiting; + depth = wait.unsuspendMonitor(); + } + + void log(Object event, Thread toThread, Thread byThread) + { + if (recentActions != null) + log(event + " " + toThread + " by " + byThread); + } + + void log(Object event, Thread toThread) + { + if (recentActions != null) + log(event + " " + toThread); + } + + void log(Object event) + { + if (recentActions == null) + return; + + if (recentActions.size() > 20) + recentActions.poll(); + recentActions.add(event + " " + depth); + } } interface WaitListAccessor @@ -191,7 +230,7 @@ public abstract class InterceptingMonitors implements InterceptorOfGlobalMethods static class InterceptedMonitorWait implements InterceptedWait { - final Kind kind; + Kind kind; final long waitTime; final InterceptibleThread waiting; final CaptureSites captureSites; @@ -200,7 +239,7 @@ public abstract class InterceptingMonitors implements InterceptorOfGlobalMethods final Object monitor; int suspendedMonitorDepth; - boolean isTriggeredByTimeout; + Trigger trigger; boolean isTriggered; final List onTrigger = new ArrayList<>(3); @@ -259,13 +298,30 @@ public abstract class InterceptingMonitors implements InterceptorOfGlobalMethods return isTriggered; } + public boolean isInterruptible() + { + return true; + } + @Override public long waitTime() { return waitTime; } - public void triggerAndAwaitDone(InterceptorOfConsequences interceptor, boolean isTimeout) + @Override + public void interceptWakeup(Trigger trigger, Thread by) + { + if (this.trigger != null && this.trigger.compareTo(trigger) >= 0) + return; + + this.trigger = trigger; + if (captureSites != null) + captureSites.registerWakeup(by); + interceptorOrDefault(by).interceptWakeup(this, trigger, interceptedBy); + } + + public void triggerAndAwaitDone(InterceptorOfConsequences interceptor, Trigger trigger) { if (isTriggered) return; @@ -277,10 +333,12 @@ public abstract class InterceptingMonitors implements InterceptorOfGlobalMethods // we may have been assigned ownership of the lock if we attempted to trigger but found the lock held if (state.heldBy != null && state.heldBy != waiting) - { + { // reset this condition to wait on lock release state.waitOn(LOCK, this); + this.kind = UNBOUNDED_WAIT; + this.trigger = null; interceptor.beforeInvocation(waiting); - interceptor.interceptWait(null); + interceptor.interceptWait(this); return; } @@ -290,11 +348,11 @@ public abstract class InterceptingMonitors implements InterceptorOfGlobalMethods { waiting.beforeInvocation(interceptor, this); - this.isTriggeredByTimeout = isTimeout; isTriggered = true; onTrigger.forEach(listener -> listener.onTrigger(this)); - monitor.notifyAll(); + if (!waiting.preWakeup(this)) + monitor.notifyAll(); // TODO: could use interrupts to target waiting anyway, avoiding notifyAll() while (!notifiedOfPause) monitor.wait(); @@ -302,7 +360,7 @@ public abstract class InterceptingMonitors implements InterceptorOfGlobalMethods if (waitingOnRelinquish) { waitingOnRelinquish = false; - monitor.notifyAll(); + monitor.notifyAll(); // TODO: could use interrupts to target waiting anyway, avoiding notifyAll() } } } @@ -394,6 +452,7 @@ public abstract class InterceptingMonitors implements InterceptorOfGlobalMethods if (cur != null) { cur.next = remove.next; + remove.next = null; --nextLength; } return this; @@ -405,15 +464,13 @@ public abstract class InterceptingMonitors implements InterceptorOfGlobalMethods } } - final InterceptorOfWaits interceptorOfWaits; final RandomSource random; private final Map monitors = new IdentityHashMap<>(); - private boolean disabled; private final Map waitingOn = new IdentityHashMap<>(); + protected boolean disabled; - public InterceptingMonitors(InterceptorOfWaits interceptorOfWaits, RandomSource random) + public InterceptingMonitors(RandomSource random) { - this.interceptorOfWaits = interceptorOfWaits; this.random = random; } @@ -436,7 +493,7 @@ public abstract class InterceptingMonitors implements InterceptorOfGlobalMethods @Override public void waitUntil(long deadline) throws InterruptedException { - InterceptibleThread thread = interceptorOfWaits.ifIntercepted(); + InterceptibleThread thread = ifIntercepted(); if (thread == null) { Clock.waitUntil(deadline); @@ -446,7 +503,7 @@ public abstract class InterceptingMonitors implements InterceptorOfGlobalMethods if (Thread.interrupted()) throw new InterruptedException(); - InterceptedMonitorWait trigger = new InterceptedMonitorWait(SLEEP_UNTIL, deadline, new MonitorState(), thread, interceptorOfWaits.captureWaitSite(thread)); + InterceptedMonitorWait trigger = new InterceptedMonitorWait(SLEEP_UNTIL, deadline, new MonitorState(), thread, captureWaitSite(thread)); thread.interceptWait(trigger); synchronized (trigger) { @@ -484,7 +541,7 @@ public abstract class InterceptingMonitors implements InterceptorOfGlobalMethods public boolean waitUntil(Object monitor, long deadline) throws InterruptedException { - InterceptibleThread thread = interceptorOfWaits.ifIntercepted(); + InterceptibleThread thread = ifIntercepted(); if (thread == null) return SyncAwaitable.waitUntil(monitor, deadline); else return wait(monitor, thread, WAIT_UNTIL, deadline); } @@ -492,7 +549,7 @@ public abstract class InterceptingMonitors implements InterceptorOfGlobalMethods @Override public void wait(Object monitor) throws InterruptedException { - InterceptibleThread thread = interceptorOfWaits.ifIntercepted(); + InterceptibleThread thread = ifIntercepted(); if (thread == null) monitor.wait(); else wait(monitor, thread, UNBOUNDED_WAIT, -1L); } @@ -500,17 +557,17 @@ public abstract class InterceptingMonitors implements InterceptorOfGlobalMethods @Override public void wait(Object monitor, long millis) throws InterruptedException { - InterceptibleThread thread = interceptorOfWaits.ifIntercepted(); + InterceptibleThread thread = ifIntercepted(); if (thread == null) monitor.wait(millis); - else wait(monitor, thread, WAIT_UNTIL, relativeToGlobalAbsoluteNanos(MILLISECONDS.toNanos(millis))); + else wait(monitor, thread, WAIT_UNTIL, relativeToGlobalNanos(MILLISECONDS.toNanos(millis))); } @Override public void wait(Object monitor, long millis, int nanos) throws InterruptedException { - InterceptibleThread thread = interceptorOfWaits.ifIntercepted(); + InterceptibleThread thread = ifIntercepted(); if (thread == null) monitor.wait(millis, nanos); - else wait(monitor, thread, WAIT_UNTIL, relativeToGlobalAbsoluteNanos(MILLISECONDS.toNanos(millis) + nanos)); + else wait(monitor, thread, WAIT_UNTIL, relativeToGlobalNanos(MILLISECONDS.toNanos(millis) + nanos)); } private boolean wait(Object monitor, InterceptibleThread thread, InterceptedWait.Kind kind, long waitNanos) throws InterruptedException @@ -519,14 +576,22 @@ public abstract class InterceptingMonitors implements InterceptorOfGlobalMethods throw new InterruptedException(); MonitorState state = state(monitor); - InterceptedMonitorWait trigger = new InterceptedMonitorWait(kind, waitNanos, state, thread, interceptorOfWaits.captureWaitSite(thread), monitor); + InterceptedMonitorWait trigger = new InterceptedMonitorWait(kind, waitNanos, state, thread, captureWaitSite(thread), monitor); + state.log("enterwait", thread); state.suspend(trigger); state.waitOn(NOTIFY, trigger); wakeOneWaitingOnLock(thread, state); thread.interceptWait(trigger); - trigger.await(); - state.restore(trigger); - return !trigger.isTriggeredByTimeout; + try + { + trigger.await(); + } + finally + { + state.restore(trigger); + state.log("exitwait", thread); + } + return trigger.trigger == SIGNAL; } public void notify(Object monitor) @@ -534,12 +599,14 @@ public abstract class InterceptingMonitors implements InterceptorOfGlobalMethods MonitorState state = state(monitor); if (state != null) { - InterceptedMonitorWait wait = state.removeOneWaitingOn(NOTIFY, random); - if (wait != null) + InterceptedMonitorWait wake = state.removeOneWaitingOn(NOTIFY, random); + if (wake != null) { - assert wait.waitingOn == null; + // TODO: assign ownership on monitorExit + assert wake.waitingOn == null; Thread waker = Thread.currentThread(); - interceptorOfWaits.interceptSignal(waker, wait, wait.captureSites, wait.interceptedBy); + wake.interceptWakeup(SIGNAL, waker); + state.log("notify", wake.waiting, waker); return; } } @@ -552,18 +619,20 @@ public abstract class InterceptingMonitors implements InterceptorOfGlobalMethods MonitorState state = state(monitor); if (state != null) { - InterceptedMonitorWait wait = state.removeAllWaitingOn(NOTIFY); - if (wait != null) + InterceptedMonitorWait wake = state.removeAllWaitingOn(NOTIFY); + if (wake != null) { Thread waker = Thread.currentThread(); - interceptorOfWaits.interceptSignal(waker, wait, wait.captureSites, wait.interceptedBy); + wake.interceptWakeup(SIGNAL, waker); + state.log("notify", wake.waiting, waker); - wait = wait.next; - while (wait != null) + wake = wake.next; + while (wake != null) { - InterceptedMonitorWait next = wait.next; - state.waitOn(LOCK, wait); - wait = next; + InterceptedMonitorWait next = wake.next; + state.waitOn(LOCK, wake); + state.log("movetowaitonlock ", wake.waiting, waker); + wake = next; } return; } @@ -581,62 +650,107 @@ public abstract class InterceptingMonitors implements InterceptorOfGlobalMethods if (!(anyThread instanceof InterceptibleThread)) return; + boolean restoreInterrupt = false; InterceptibleThread thread = (InterceptibleThread) anyThread; - if ( !thread.isEvaluationDeterministic() - && random.decide(preMonitorDelayChance)) + try { - // TODO (feature): hold a stack of threads already paused by the nemesis, and, if one of the threads - // is entering the monitor, put the contents of this stack into `waitingOn` for this monitor. - InterceptedConditionWait signal = new InterceptedConditionWait(NEMESIS, 0L, thread, interceptorOfWaits.captureWaitSite(thread), null); - thread.interceptWait(signal); - - // save interrupt state to restore afterwards - new ones only arrive if terminating simulation - boolean wasInterrupted = Thread.interrupted(); - signal.awaitThrowUncheckedOnInterrupt(); - if (wasInterrupted) thread.interrupt(); - } - - MonitorState state = state(monitor); - if (state.heldBy != thread) - { - if (state.heldBy != null) + if ( !thread.isEvaluationDeterministic() + && random.decide(preMonitorDelayChance)) { - if (!thread.isIntercepting() && disabled) return; - else if (!thread.isIntercepting()) - throw new AssertionError(); + // TODO (feature): hold a stack of threads already paused by the nemesis, and, if one of the threads + // is entering the monitor, put the contents of this stack into `waitingOn` for this monitor. + InterceptedConditionWait signal = new InterceptedConditionWait(NEMESIS, 0L, thread, captureWaitSite(thread), null); + thread.interceptWait(signal); - checkForDeadlock(thread, state.heldBy); - InterceptedMonitorWait wait = new InterceptedMonitorWait(UNBOUNDED_WAIT, 0L, state, thread, interceptorOfWaits.captureWaitSite(thread)); - wait.suspendedMonitorDepth = 1; - state.waitOn(LOCK, wait); - thread.interceptWait(wait); - synchronized (wait) + // save interrupt state to restore afterwards - new ones only arrive if terminating simulation + restoreInterrupt = Thread.interrupted(); + while (true) { - waitingOn.put(thread, monitor); try { - wait.await(); + signal.awaitDeclaredUninterruptible(); + break; } catch (InterruptedException e) { - throw new UncheckedInterruptedException(e); - } - finally - { - waitingOn.remove(thread); + if (disabled) + throw new UncheckedInterruptedException(e); + restoreInterrupt = true; } } - state.restore(wait); + } + + MonitorState state = state(monitor); + if (state.heldBy != thread) + { + if (state.heldBy != null) + { + if (!thread.isIntercepting() && disabled) return; + else if (!thread.isIntercepting()) + throw new AssertionError(); + + checkForDeadlock(thread, state.heldBy); + InterceptedMonitorWait wait = new InterceptedMonitorWait(UNBOUNDED_WAIT, 0L, state, thread, captureWaitSite(thread)); + wait.suspendedMonitorDepth = 1; + state.log("monitorenter_wait", thread); + state.waitOn(LOCK, wait); + thread.interceptWait(wait); + synchronized (wait) + { + waitingOn.put(thread, monitor); + restoreInterrupt |= Thread.interrupted(); + try + { + while (true) + { + try + { + wait.await(); + break; + } + catch (InterruptedException e) + { + if (disabled) + { + if (state.heldBy == thread) + { + state.heldBy = null; + state.depth = 0; + } + throw new UncheckedInterruptedException(e); + } + + restoreInterrupt = true; + if (wait.isTriggered) + break; + } + } + } + finally + { + waitingOn.remove(thread); + } + } + state.claim(wait); + state.log("monitorenter_claim", thread); + } + else + { + state.log("monitorenter_free", thread); + state.heldBy = thread; + state.depth = 1; + } } else { - state.heldBy = thread; - state.depth = 1; + state.log("monitorreenter", thread); + state.depth++; } } - else + finally { - state.depth++; + if (restoreInterrupt) + thread.interrupt(); } } @@ -658,8 +772,12 @@ public abstract class InterceptingMonitors implements InterceptorOfGlobalMethods throw new AssertionError(); if (--state.depth > 0) + { + state.log("monitorreexit", thread); return; + } + state.log("monitorexit", thread); state.heldBy = null; if (!wakeOneWaitingOnLock(thread, state)) @@ -668,7 +786,7 @@ public abstract class InterceptingMonitors implements InterceptorOfGlobalMethods } } - private boolean wakeOneWaitingOnLock(Thread thread, MonitorState state) + private boolean wakeOneWaitingOnLock(Thread waker, MonitorState state) { InterceptedMonitorWait wake = state.removeOneWaitingOn(LOCK, random); if (wake != null) @@ -676,11 +794,12 @@ public abstract class InterceptingMonitors implements InterceptorOfGlobalMethods assert wake.waitingOn == null; assert !wake.isTriggered(); - interceptorOfWaits.interceptSignal(thread, wake, wake.captureSites, wake.interceptedBy); + wake.interceptWakeup(SIGNAL, waker); // assign them the lock, so they'll definitely get it when they wake assert state.heldBy == null; state.heldBy = wake.waiting; + state.log("wake", wake.waiting); return true; } return false; diff --git a/test/simulator/main/org/apache/cassandra/simulator/systems/InterceptingWaitQueue.java b/test/simulator/main/org/apache/cassandra/simulator/systems/InterceptingWaitQueue.java index f1d9b4d661..d14bcfb863 100644 --- a/test/simulator/main/org/apache/cassandra/simulator/systems/InterceptingWaitQueue.java +++ b/test/simulator/main/org/apache/cassandra/simulator/systems/InterceptingWaitQueue.java @@ -26,32 +26,33 @@ import java.util.function.Predicate; import org.apache.cassandra.simulator.systems.InterceptingAwaitable.InterceptingSignal; import org.apache.cassandra.utils.concurrent.WaitQueue; +import static org.apache.cassandra.simulator.systems.InterceptorOfGlobalMethods.Global.ifIntercepted; + +@PerClassLoader class InterceptingWaitQueue extends WaitQueue.Standard implements WaitQueue { - private final InterceptorOfWaits interceptorOfWaits; final Queue> interceptible = new ConcurrentLinkedQueue<>(); - public InterceptingWaitQueue(InterceptorOfWaits interceptorOfWaits) + public InterceptingWaitQueue() { - this.interceptorOfWaits = interceptorOfWaits; } public Signal register() { - if (interceptorOfWaits.ifIntercepted() == null) + if (ifIntercepted() == null) return super.register(); - InterceptingSignal signal = new InterceptingSignal<>(interceptorOfWaits); + InterceptingSignal signal = new InterceptingSignal<>(); interceptible.add(signal); return signal; } public Signal register(V value, Consumer consumer) { - if (interceptorOfWaits.ifIntercepted() == null) + if (ifIntercepted() == null) return super.register(value, consumer); - InterceptingSignal signal = new InterceptingSignal<>(interceptorOfWaits, value, consumer); + InterceptingSignal signal = new InterceptingSignal<>(value, consumer); interceptible.add(signal); return signal; } diff --git a/test/simulator/main/org/apache/cassandra/simulator/systems/InterceptorOfConsequences.java b/test/simulator/main/org/apache/cassandra/simulator/systems/InterceptorOfConsequences.java index 5907dd8a09..801b35367d 100644 --- a/test/simulator/main/org/apache/cassandra/simulator/systems/InterceptorOfConsequences.java +++ b/test/simulator/main/org/apache/cassandra/simulator/systems/InterceptorOfConsequences.java @@ -21,6 +21,7 @@ package org.apache.cassandra.simulator.systems; import org.apache.cassandra.distributed.api.IInvokableInstance; import org.apache.cassandra.distributed.api.IMessage; import org.apache.cassandra.simulator.OrderOn; +import org.apache.cassandra.simulator.systems.InterceptedWait.Trigger; import org.apache.cassandra.utils.Shared; import static org.apache.cassandra.utils.Shared.Scope.SIMULATION; @@ -28,9 +29,49 @@ import static org.apache.cassandra.utils.Shared.Scope.SIMULATION; @Shared(scope = SIMULATION) public interface InterceptorOfConsequences { + public static final InterceptorOfConsequences DEFAULT_INTERCEPTOR = new InterceptorOfConsequences() + { + @Override + public void beforeInvocation(InterceptibleThread realThread) + { + } + + @Override + public void interceptMessage(IInvokableInstance from, IInvokableInstance to, IMessage message) + { + throw new AssertionError(); + } + + @Override + public void interceptWait(InterceptedWait wakeupWith) + { + throw new AssertionError(); + } + + @Override + public void interceptWakeup(InterceptedWait wakeup, Trigger trigger, InterceptorOfConsequences waitWasInterceptedBy) + { + // TODO (now): should we be asserting here? + wakeup.triggerBypass(); + } + + @Override + public void interceptExecution(InterceptedExecution invoke, OrderOn orderOn) + { + throw new AssertionError(); + } + + @Override + public void interceptTermination(boolean isThreadTermination) + { + throw new AssertionError(); + } + }; + + void beforeInvocation(InterceptibleThread realThread); void interceptMessage(IInvokableInstance from, IInvokableInstance to, IMessage message); - void interceptWakeup(InterceptedWait wakeup, InterceptorOfConsequences waitWasInterceptedBy); + void interceptWakeup(InterceptedWait wakeup, Trigger trigger, InterceptorOfConsequences waitWasInterceptedBy); void interceptExecution(InterceptedExecution invoke, OrderOn orderOn); void interceptWait(InterceptedWait wakeupWith); void interceptTermination(boolean isThreadTermination); diff --git a/test/simulator/main/org/apache/cassandra/simulator/systems/InterceptorOfExecution.java b/test/simulator/main/org/apache/cassandra/simulator/systems/InterceptorOfExecution.java index 872525d663..ac8255d506 100644 --- a/test/simulator/main/org/apache/cassandra/simulator/systems/InterceptorOfExecution.java +++ b/test/simulator/main/org/apache/cassandra/simulator/systems/InterceptorOfExecution.java @@ -22,6 +22,7 @@ import java.util.concurrent.Callable; import java.util.concurrent.ScheduledFuture; import java.util.function.Function; +import org.apache.cassandra.simulator.systems.SimulatedAction.Kind; import org.apache.cassandra.utils.Shared; import org.apache.cassandra.utils.concurrent.RunnableFuture; @@ -37,7 +38,7 @@ public interface InterceptorOfExecution interface InterceptExecution { > T addTask(T task, InterceptingExecutor executor); - ScheduledFuture schedule(SimulatedAction.Kind kind, long delayNanos, long deadlineNanos, Callable runnable, InterceptingExecutor executor); - Thread start(SimulatedAction.Kind kind, Function factory, Runnable run); + ScheduledFuture schedule(Kind kind, long delayNanos, long deadlineNanos, Callable runnable, InterceptingExecutor executor); + Thread start(Kind kind, Function factory, Runnable run); } } diff --git a/test/simulator/main/org/apache/cassandra/simulator/systems/InterceptorOfGlobalMethods.java b/test/simulator/main/org/apache/cassandra/simulator/systems/InterceptorOfGlobalMethods.java index 2f57279806..dde5870ffd 100644 --- a/test/simulator/main/org/apache/cassandra/simulator/systems/InterceptorOfGlobalMethods.java +++ b/test/simulator/main/org/apache/cassandra/simulator/systems/InterceptorOfGlobalMethods.java @@ -22,9 +22,15 @@ import java.util.ArrayDeque; import java.util.UUID; import java.util.concurrent.BlockingQueue; import java.util.function.IntSupplier; +import java.util.function.LongConsumer; import java.util.function.ToIntFunction; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import net.openhft.chronicle.core.util.WeakIdentityHashMap; +import org.apache.cassandra.simulator.systems.InterceptedWait.CaptureSites; +import org.apache.cassandra.utils.Clock; import org.apache.cassandra.utils.Closeable; import org.apache.cassandra.utils.Shared; import org.apache.cassandra.utils.concurrent.BlockingQueues; @@ -45,8 +51,27 @@ public interface InterceptorOfGlobalMethods extends InterceptorOfSystemMethods, CountDownLatch newCountDownLatch(int count); Condition newOneTimeCondition(); + /** + * If this interceptor is debugging wait/wake/now sites, return one initialised with the current trace of the + * provided thread; otherwise return null. + */ + CaptureSites captureWaitSite(Thread thread); + + /** + * Returns the current thread as an InterceptibleThread IF it has its InterceptConsequences interceptor set. + * Otherwise, one of the following will happen: + * * if the InterceptorOfWaits permits it, null will be returned; + * * if it does not, the process will be failed. + */ + InterceptibleThread ifIntercepted(); + + void uncaughtException(Thread thread, Throwable throwable); + + @PerClassLoader public static class IfInterceptibleThread extends None implements InterceptorOfGlobalMethods { + static LongConsumer threadLocalRandomCheck; + @Override public WaitQueue newWaitQueue() { @@ -77,6 +102,29 @@ public interface InterceptorOfGlobalMethods extends InterceptorOfSystemMethods, return Condition.newOneTimeCondition(); } + @Override + public CaptureSites captureWaitSite(Thread thread) + { + if (thread instanceof InterceptibleThread) + return ((InterceptibleThread) thread).interceptorOfGlobalMethods().captureWaitSite(thread); + + Thread currentThread = Thread.currentThread(); + if (currentThread instanceof InterceptibleThread) + return ((InterceptibleThread) currentThread).interceptorOfGlobalMethods().captureWaitSite(thread); + + return null; + } + + @Override + public InterceptibleThread ifIntercepted() + { + Thread thread = Thread.currentThread(); + if (thread instanceof InterceptibleThread) + return ((InterceptibleThread) thread).interceptorOfGlobalMethods().ifIntercepted(); + + return null; + } + @Override public void waitUntil(long deadlineNanos) throws InterruptedException { @@ -271,6 +319,37 @@ public interface InterceptorOfGlobalMethods extends InterceptorOfSystemMethods, } } + @Override + public void threadLocalRandomCheck(long seed) + { + if (threadLocalRandomCheck != null) + threadLocalRandomCheck.accept(seed); + } + + @Override + public void uncaughtException(Thread thread, Throwable throwable) + { + if (thread instanceof InterceptibleThread) + ((InterceptibleThread) thread).interceptorOfGlobalMethods().uncaughtException(thread, throwable); + } + + @Override + public long nanoTime() + { + return Clock.Global.nanoTime(); + } + + @Override + public long currentTimeMillis() + { + return Clock.Global.currentTimeMillis(); + } + + public static void setThreadLocalRandomCheck(LongConsumer runnable) + { + threadLocalRandomCheck = runnable; + } + @Override public void close() { @@ -317,6 +396,23 @@ public interface InterceptorOfGlobalMethods extends InterceptorOfSystemMethods, return new BlockingQueues.Sync<>(capacity, new ArrayDeque<>()); } + public static CaptureSites captureWaitSite(Thread thread) + { + return methods.captureWaitSite(thread); + } + + public static InterceptibleThread ifIntercepted() + { + return methods.ifIntercepted(); + } + + public static void uncaughtException(Thread thread, Throwable throwable) + { + System.err.println(thread); + throwable.printStackTrace(System.err); + methods.uncaughtException(thread, throwable); + } + public static void unsafeReset() { Global.methods = new IfInterceptibleThread(); diff --git a/test/simulator/main/org/apache/cassandra/simulator/systems/InterceptorOfWaits.java b/test/simulator/main/org/apache/cassandra/simulator/systems/InterceptorOfWaits.java deleted file mode 100644 index 179dda4820..0000000000 --- a/test/simulator/main/org/apache/cassandra/simulator/systems/InterceptorOfWaits.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.cassandra.simulator.systems; - -import javax.annotation.Nullable; - -import org.apache.cassandra.simulator.systems.InterceptedWait.CaptureSites; -import org.apache.cassandra.utils.Shared; - -import static org.apache.cassandra.utils.Shared.Scope.SIMULATION; - -@Shared(scope = SIMULATION) -public interface InterceptorOfWaits -{ - /** - * If this interceptor is debugging wait/wake/now sites, return one initialised with the current trace of the - * provided thread; otherwise return null. - */ - @Nullable CaptureSites captureWaitSite(Thread thread); - - /** - * Returns the current thread as an InterceptibleThread IF it has its InterceptConsequences interceptor set. - * Otherwise, one of the following will happen: - * * if the InterceptorOfWaits permits it, null will be returned; - * * if it does not, the process will be failed. - */ - @Nullable InterceptibleThread ifIntercepted(); - - void interceptSignal(Thread signalledBy, InterceptedWait signalled, CaptureSites waitSites, InterceptorOfConsequences interceptedBy); -} diff --git a/test/simulator/main/org/apache/cassandra/simulator/systems/NonInterceptible.java b/test/simulator/main/org/apache/cassandra/simulator/systems/NonInterceptible.java index 2080ecb763..61f7f7878a 100644 --- a/test/simulator/main/org/apache/cassandra/simulator/systems/NonInterceptible.java +++ b/test/simulator/main/org/apache/cassandra/simulator/systems/NonInterceptible.java @@ -18,6 +18,7 @@ package org.apache.cassandra.simulator.systems; +import java.util.concurrent.Callable; import java.util.function.Supplier; import org.apache.cassandra.utils.Shared; @@ -29,14 +30,23 @@ import static org.apache.cassandra.utils.Shared.Scope.SIMULATION; @Shared(scope = SIMULATION) public class NonInterceptible { - private static final ThreadLocal PERMIT = new ThreadLocal<>(); + @Shared(scope = SIMULATION) + public enum Permit { REQUIRED, OPTIONAL } + + private static final ThreadLocal PERMIT = new ThreadLocal<>(); + + public static boolean isPermitted(Permit permit) + { + Permit current = PERMIT.get(); + return current != null && current.compareTo(permit) >= 0; + } public static boolean isPermitted() { - return TRUE.equals(PERMIT.get()); + return PERMIT.get() != null; } - public static void execute(Runnable runnable) + public static void execute(Permit permit, Runnable runnable) { if (isPermitted()) { @@ -44,19 +54,19 @@ public class NonInterceptible } else { - PERMIT.set(TRUE); + PERMIT.set(permit); try { runnable.run(); } finally { - PERMIT.set(FALSE); + PERMIT.set(null); } } } - public static V apply(Supplier supplier) + public static V apply(Permit permit, Supplier supplier) { if (isPermitted()) { @@ -64,14 +74,34 @@ public class NonInterceptible } else { - PERMIT.set(TRUE); + PERMIT.set(permit); try { return supplier.get(); } finally { - PERMIT.set(FALSE); + PERMIT.set(null); + } + } + } + + public static V call(Permit permit, Callable call) throws Exception + { + if (isPermitted()) + { + return call.call(); + } + else + { + PERMIT.set(permit); + try + { + return call.call(); + } + finally + { + PERMIT.set(null); } } } diff --git a/test/simulator/main/org/apache/cassandra/simulator/systems/SimulatedAction.java b/test/simulator/main/org/apache/cassandra/simulator/systems/SimulatedAction.java index 9e86000602..551616ff60 100644 --- a/test/simulator/main/org/apache/cassandra/simulator/systems/SimulatedAction.java +++ b/test/simulator/main/org/apache/cassandra/simulator/systems/SimulatedAction.java @@ -27,9 +27,9 @@ import java.util.Map; import java.util.concurrent.Executor; import javax.annotation.Nullable; - - import org.apache.cassandra.concurrent.ImmediateExecutor; +import com.google.common.base.Preconditions; + import org.apache.cassandra.distributed.api.IInvokableInstance; import org.apache.cassandra.distributed.api.IMessage; import org.apache.cassandra.exceptions.RequestFailureReason; @@ -43,6 +43,7 @@ import org.apache.cassandra.simulator.Actions; import org.apache.cassandra.simulator.FutureActionScheduler.Deliver; import org.apache.cassandra.simulator.OrderOn; import org.apache.cassandra.simulator.systems.InterceptedExecution.InterceptedRunnableExecution; +import org.apache.cassandra.simulator.systems.InterceptedWait.Trigger; import org.apache.cassandra.simulator.systems.InterceptedWait.TriggerListener; import org.apache.cassandra.utils.LazyToString; import org.apache.cassandra.utils.Shared; @@ -50,6 +51,7 @@ import org.apache.cassandra.utils.Shared; import static org.apache.cassandra.net.MessagingService.instance; import static org.apache.cassandra.simulator.Action.Modifiers.DROP; import static org.apache.cassandra.simulator.Action.Modifiers.NONE; +import static org.apache.cassandra.simulator.Action.Modifiers.PSEUDO_ORPHAN; import static org.apache.cassandra.simulator.Action.Modifiers.START_DAEMON_TASK; import static org.apache.cassandra.simulator.Action.Modifiers.START_SCHEDULED_TASK; import static org.apache.cassandra.simulator.Action.Modifiers.START_INFINITE_LOOP; @@ -58,8 +60,12 @@ import static org.apache.cassandra.simulator.Action.Modifiers.START_THREAD; import static org.apache.cassandra.simulator.Action.Modifiers.START_TIMEOUT_TASK; import static org.apache.cassandra.simulator.Action.Modifiers.WAKE_UP_THREAD; import static org.apache.cassandra.simulator.FutureActionScheduler.Deliver.DELIVER; -import static org.apache.cassandra.simulator.FutureActionScheduler.Deliver.TIMEOUT; +import static org.apache.cassandra.simulator.FutureActionScheduler.Deliver.DELIVER_AND_TIMEOUT; +import static org.apache.cassandra.simulator.FutureActionScheduler.Deliver.FAILURE; +import static org.apache.cassandra.simulator.systems.InterceptedWait.Trigger.SIGNAL; +import static org.apache.cassandra.simulator.systems.InterceptedWait.Trigger.TIMEOUT; import static org.apache.cassandra.simulator.systems.SimulatedAction.Kind.MESSAGE; +import static org.apache.cassandra.simulator.systems.SimulatedAction.Kind.REDUNDANT_MESSAGE_TIMEOUT; import static org.apache.cassandra.simulator.systems.SimulatedAction.Kind.SCHEDULED_TIMEOUT; import static org.apache.cassandra.simulator.systems.SimulatedAction.Kind.TASK; import static org.apache.cassandra.simulator.Debug.Info.LOG; @@ -77,6 +83,7 @@ public abstract class SimulatedAction extends Action implements InterceptorOfCon public enum Kind { MESSAGE(NONE, NONE, WAKE_UP_THREAD), + REDUNDANT_MESSAGE_TIMEOUT(PSEUDO_ORPHAN, NONE, WAKE_UP_THREAD), TASK(START_TASK, NONE, WAKE_UP_THREAD), SCHEDULED_TASK(START_SCHEDULED_TASK, NONE, WAKE_UP_THREAD), SCHEDULED_TIMEOUT(START_TIMEOUT_TASK, NONE, WAKE_UP_THREAD), @@ -100,17 +107,19 @@ public abstract class SimulatedAction extends Action implements InterceptorOfCon class Signal extends Action implements TriggerListener { - final boolean isTimeout; final InterceptedWait wakeup; + final Trigger trigger; boolean signalling; // note that we do not inherit from the parent thread's self, as anything relevantly heritable by continuations is likely already transitive - protected Signal(Object description, Modifiers self, long deadlineNanos, InterceptedWait wakeup) + protected Signal(Object description, Modifiers self, InterceptedWait wakeup, Trigger trigger, long deadlineNanos) { super(description, self.inheritIfContinuation(SimulatedAction.this.self()), NONE); - if (isTimeout = deadlineNanos >= 0) - setDeadline(deadlineNanos); this.wakeup = wakeup; + this.trigger = trigger; + assert deadlineNanos < 0 || trigger == TIMEOUT; + if (deadlineNanos >= 0) + setDeadline(simulated.time, deadlineNanos); assert !wakeup.isTriggered(); wakeup.addListener(this); } @@ -131,6 +140,9 @@ public abstract class SimulatedAction extends Action implements InterceptorOfCon { assert !wakeup.isTriggered(); assert !isFinished(); + + if (SimulatedAction.this.isFinished()) + return super.performed(ActionList.empty(), true, true); assert !realThreadHasTerminated; signalling = true; @@ -140,7 +152,7 @@ public abstract class SimulatedAction extends Action implements InterceptorOfCon @Override protected ActionList performSimple() { - return simulate(() -> wakeup.triggerAndAwaitDone(SimulatedAction.this, isTimeout)); + return simulate(() -> wakeup.triggerAndAwaitDone(SimulatedAction.this, trigger)); } @Override @@ -175,6 +187,9 @@ public abstract class SimulatedAction extends Action implements InterceptorOfCon public SimulatedAction(Object description, Kind kind, OrderOn orderOn, Modifiers self, Modifiers transitive, Map verbModifiers, Verb forVerb, SimulatedSystems simulated) { super(description, orderOn, self, transitive); + Preconditions.checkNotNull(kind); + Preconditions.checkNotNull(verbModifiers); + Preconditions.checkNotNull(simulated); this.kind = kind; this.simulated = simulated; this.verbModifiers = verbModifiers; @@ -185,20 +200,20 @@ public abstract class SimulatedAction extends Action implements InterceptorOfCon public void interceptMessage(IInvokableInstance from, IInvokableInstance to, IMessage message) { if (!to.isShutdown()) - consequences.add(applyToMessage(from, to, message)); + consequences.addAll(applyToMessage(from, to, message)); } @Override - public void interceptWakeup(InterceptedWait wakeup, InterceptorOfConsequences waitWasInterceptedBy) + public void interceptWakeup(InterceptedWait wakeup, Trigger trigger, InterceptorOfConsequences waitWasInterceptedBy) { SimulatedAction action = (SimulatedAction) waitWasInterceptedBy; - action.applyToWakeup(consequences, wakeup); + action.applyToWakeup(consequences, wakeup, trigger); } @Override public void interceptExecution(InterceptedExecution invoke, OrderOn orderOn) { - if (invoke.kind() == SCHEDULED_TIMEOUT && transitive().is(Modifier.RELIABLE) && transitive().is(Modifier.NO_TIMEOUTS)) + if (invoke.kind() == SCHEDULED_TIMEOUT && transitive().is(Modifier.RELIABLE) && transitive().is(Modifier.NO_THREAD_TIMEOUTS)) invoke.cancel(); else consequences.add(applyToExecution(invoke, orderOn)); @@ -241,6 +256,12 @@ public abstract class SimulatedAction extends Action implements InterceptorOfCon consequences.add(Actions.empty(Modifiers.INFO, lazy(() -> "Waiting[" + wakeUpWith + "] " + realThread))); } + for (int i = consequences.size() - 1; i >= 0 ; --i) + { + // a scheduled future might be cancelled by the same action that creates it + if (consequences.get(i).isCancelled()) + consequences.remove(i); + } return ActionList.of(consequences); } finally @@ -275,73 +296,78 @@ public abstract class SimulatedAction extends Action implements InterceptorOfCon switch (wakeupWith.kind()) { case WAIT_UNTIL: - applyToSignal(out, START_TIMEOUT_TASK, "Timeout", wakeupWith, wakeupWith.waitTime()); + applyToSignal(out, START_TIMEOUT_TASK, "Timeout", wakeupWith, TIMEOUT, wakeupWith.waitTime()); break; case NEMESIS: - applyToSignal(out, WAKE_UP_THREAD, "Nemesis", wakeupWith, -1L); + applyToSignal(out, WAKE_UP_THREAD, "Nemesis", wakeupWith, SIGNAL, -1L); break; default : - applyToSignal(out, WAKE_UP_THREAD, "Continue", wakeupWith, -1L); + applyToSignal(out, WAKE_UP_THREAD, "Continue", wakeupWith, SIGNAL, -1L); break; } } - void applyToWakeup(List out, InterceptedWait wakeup) + void applyToWakeup(List out, InterceptedWait wakeup, Trigger trigger) { - applyToSignal(out, kind.signal, "Wakeup", wakeup, -1); + applyToSignal(out, kind.signal, "Wakeup", wakeup, trigger, -1); } - void applyToSignal(List out, Modifiers self, String kind, InterceptedWait wakeup, long deadlineNanos) + void applyToSignal(List out, Modifiers self, String kind, InterceptedWait wakeup, Trigger trigger, long deadlineNanos) { - applyToSignal(out, lazy(() -> kind + wakeup + ' ' + realThread), self, wakeup, deadlineNanos); + applyToSignal(out, lazy(() -> kind + wakeup + ' ' + realThread), self, wakeup, trigger, deadlineNanos); } - void applyToSignal(List out, LazyToString id, Modifiers self, InterceptedWait wakeup, long deadlineNanos) + void applyToSignal(List out, LazyToString id, Modifiers self, InterceptedWait wakeup, Trigger trigger, long deadlineNanos) { - if (deadlineNanos >= 0 && !self.is(Modifier.TIMEOUT)) + if (deadlineNanos >= 0 && !self.is(Modifier.THREAD_TIMEOUT)) throw new IllegalStateException(); - out.add(new Signal(id, self, deadlineNanos, wakeup)); + out.add(new Signal(id, self, wakeup, trigger, deadlineNanos)); } - Action applyToMessage(IInvokableInstance from, IInvokableInstance to, IMessage message) + List applyToMessage(IInvokableInstance from, IInvokableInstance to, IMessage message) { Executor executor = to.executorFor(message.verb()); if (executor instanceof ImmediateExecutor) executor = to.executor(); - InterceptedExecution.InterceptedTaskExecution task = new InterceptedRunnableExecution( - (InterceptingExecutor) executor, () -> to.receiveMessageWithInvokingThread(message) - ); - Verb verb = Verb.fromId(message.verb()); Modifiers self = verbModifiers.getOrDefault(verb, NONE); int fromNum = from.config().num(); int toNum = to.config().num(); - Deliver deliver; - if (is(Modifier.RELIABLE) || self.is(Modifier.RELIABLE)) deliver = DELIVER; - else deliver = simulated.futureScheduler.shouldDeliver(fromNum, toNum); + long expiresAtNanos = simulated.time.get(fromNum).localToGlobalNanos(message.expiresAtNanos()); + boolean isReliable = is(Modifier.RELIABLE) || self.is(Modifier.RELIABLE); + Deliver deliver = isReliable ? DELIVER : simulated.futureScheduler.shouldDeliver(fromNum, toNum); - Action action; + List actions = new ArrayList<>(deliver == DELIVER_AND_TIMEOUT ? 2 : 1); switch (deliver) { default: throw new AssertionError(); case DELIVER: + case DELIVER_AND_TIMEOUT: { - Object description = lazy(() -> String.format("%s(%d) from %s to %s", Verb.fromId(message.verb()), message.id(), message.from(), to.broadcastAddress())); + InterceptedExecution.InterceptedTaskExecution task = new InterceptedRunnableExecution( + (InterceptingExecutor) executor, () -> to.receiveMessageWithInvokingThread(message) + ); + Object description = lazy(() -> String.format("%s(%d) from %s to %s", verb, message.id(), message.from(), to.broadcastAddress())); OrderOn orderOn = task.executor.orderAppliesAfterScheduling(); - action = applyTo(description, MESSAGE, orderOn, self, verb, task); - action.setDeadline(simulated.futureScheduler.messageDeadlineNanos(fromNum, toNum)); - break; + Action action = applyTo(description, MESSAGE, orderOn, self, verb, task); + long deadlineNanos = simulated.futureScheduler.messageDeadlineNanos(fromNum, toNum); + if (deliver == DELIVER && deadlineNanos >= expiresAtNanos) + { + if (isReliable) deadlineNanos = verb.isResponse() ? expiresAtNanos : expiresAtNanos / 2; + else deliver = DELIVER_AND_TIMEOUT; + } + action.setDeadline(simulated.time, deadlineNanos); + actions.add(action); + if (deliver == DELIVER) + break; } case FAILURE: case TIMEOUT: { - task.cancel(); - self = DROP.with(self); - InetSocketAddress failedOn; IInvokableInstance notify; if (verb.isResponse()) @@ -354,39 +380,47 @@ public abstract class SimulatedAction extends Action implements InterceptorOfCon failedOn = to.broadcastAddress(); notify = from; } + boolean isTimeout = deliver != FAILURE; InterceptedExecution.InterceptedTaskExecution failTask = new InterceptedRunnableExecution( (InterceptingExecutor) notify.executorFor(verb.id), - () -> notify.unsafeApplyOnThisThread((socketAddress, id, isTimeout) -> { + () -> notify.unsafeApplyOnThisThread((socketAddress, id, innerIsTimeout) -> { InetAddressAndPort address = InetAddressAndPort.getByAddress(socketAddress); RequestCallbacks.CallbackInfo callback = instance().callbacks.remove(id, address); if (callback != null) { RequestCallback invokeOn = (RequestCallback) callback.callback; - RequestFailureReason reason = isTimeout ? RequestFailureReason.TIMEOUT : RequestFailureReason.UNKNOWN; + RequestFailureReason reason = innerIsTimeout ? RequestFailureReason.TIMEOUT : RequestFailureReason.UNKNOWN; invokeOn.onFailure(address, reason); } return null; - }, failedOn, message.id(), deliver == TIMEOUT) + }, failedOn, message.id(), isTimeout) ); Object description = (lazy(() -> String.format("Report Timeout of %s(%d) from %s to %s", Verb.fromId(message.verb()), message.id(), failedOn, notify.broadcastAddress()))); OrderOn orderOn = failTask.executor.orderAppliesAfterScheduling(); - action = applyTo(description, MESSAGE, orderOn, self, failTask); + self = DROP.with(self); + Kind kind = deliver == DELIVER_AND_TIMEOUT ? REDUNDANT_MESSAGE_TIMEOUT : MESSAGE; + Action action = applyTo(description, kind, orderOn, self, failTask); switch (deliver) { default: throw new AssertionError(); - case TIMEOUT: - long expiresAfterNanos = from.unsafeApplyOnThisThread(id -> Verb.fromId(id).expiresAfterNanos(), (verb.isResponse() ? forVerb : verb).id); - action.setDeadline(simulated.futureScheduler.messageTimeoutNanos(expiresAfterNanos)); - break; case FAILURE: - action.setDeadline(simulated.futureScheduler.messageFailureNanos(toNum, fromNum)); + long deadlineNanos = simulated.futureScheduler.messageFailureNanos(toNum, fromNum); + if (deadlineNanos < expiresAtNanos) + { + action.setDeadline(simulated.time, deadlineNanos); + break; + } + case DELIVER_AND_TIMEOUT: + case TIMEOUT: + long expirationIntervalNanos = from.unsafeCallOnThisThread(RequestCallbacks::defaultExpirationInterval); + action.setDeadline(simulated.time, simulated.futureScheduler.messageTimeoutNanos(expiresAtNanos, expirationIntervalNanos)); break; } - break; + actions.add(action); } } - return action; + return actions; } Action applyToExecution(InterceptedExecution invoke, OrderOn orderOn) @@ -398,7 +432,7 @@ public abstract class SimulatedAction extends Action implements InterceptorOfCon case SCHEDULED_DAEMON: case SCHEDULED_TASK: case SCHEDULED_TIMEOUT: - result.setDeadline(invoke.deadlineNanos()); + result.setDeadline(simulated.time, invoke.deadlineNanos()); } return result; diff --git a/test/simulator/main/org/apache/cassandra/simulator/systems/SimulatedActionCallable.java b/test/simulator/main/org/apache/cassandra/simulator/systems/SimulatedActionCallable.java index 2a2074adbb..2dc3d6a9ab 100644 --- a/test/simulator/main/org/apache/cassandra/simulator/systems/SimulatedActionCallable.java +++ b/test/simulator/main/org/apache/cassandra/simulator/systems/SimulatedActionCallable.java @@ -18,7 +18,6 @@ package org.apache.cassandra.simulator.systems; -import java.util.Collections; import java.util.function.BiConsumer; import org.apache.cassandra.distributed.api.IInvokableInstance; diff --git a/test/simulator/main/org/apache/cassandra/simulator/systems/SimulatedActionTask.java b/test/simulator/main/org/apache/cassandra/simulator/systems/SimulatedActionTask.java index e5adb07398..6607e19bab 100644 --- a/test/simulator/main/org/apache/cassandra/simulator/systems/SimulatedActionTask.java +++ b/test/simulator/main/org/apache/cassandra/simulator/systems/SimulatedActionTask.java @@ -55,6 +55,21 @@ public class SimulatedActionTask extends SimulatedAction implements Runnable task.onCancel(this); } + private SimulatedActionTask(Object description, Modifiers self, Modifiers children, SimulatedSystems simulated, IInvokableInstance on, InterceptedExecution task) + { + super(description, self, children, null, simulated); + this.task = task; + task.onCancel(this); + } + + /** + * To be used to create actions on runnable that are not serializable but are anyway safe to invoke + */ + public static SimulatedActionTask unsafeTask(Object description, Modifiers self, Modifiers transitive, SimulatedSystems simulated, IInvokableInstance on, Runnable run) + { + return new SimulatedActionTask(description, self, transitive, simulated, on, unsafeAsTask(on, run, simulated.failures)); + } + protected static Runnable asSafeRunnable(IInvokableInstance on, SerializableRunnable run) { return () -> on.unsafeRunOnThisThread(run); @@ -97,9 +112,13 @@ public class SimulatedActionTask extends SimulatedAction implements Runnable { try { - task.onCancel(null); - task.cancel(); - task = null; + if (task != null) + { + task.onCancel(null); + task.cancel(); + task = null; + } + return super.safeInvalidate(isCancellation); } catch (Throwable t) @@ -111,7 +130,7 @@ public class SimulatedActionTask extends SimulatedAction implements Runnable @Override public void run() { - // cancellation invoked by the task + // cancellation invoked on the task by the application task = null; super.cancel(); } diff --git a/test/simulator/main/org/apache/cassandra/simulator/systems/SimulatedBallots.java b/test/simulator/main/org/apache/cassandra/simulator/systems/SimulatedBallots.java index 1a188cb15a..906dc5a2c4 100644 --- a/test/simulator/main/org/apache/cassandra/simulator/systems/SimulatedBallots.java +++ b/test/simulator/main/org/apache/cassandra/simulator/systems/SimulatedBallots.java @@ -18,16 +18,16 @@ package org.apache.cassandra.simulator.systems; -import java.util.UUID; import java.util.concurrent.atomic.AtomicLong; import java.util.function.LongSupplier; import java.util.function.Supplier; +import org.apache.cassandra.service.paxos.Ballot; import org.apache.cassandra.service.paxos.BallotGenerator; import org.apache.cassandra.simulator.RandomSource; import org.apache.cassandra.simulator.RandomSource.Choices; -import org.apache.cassandra.utils.TimeUUID; -import org.apache.cassandra.utils.UUIDGen; + +import static org.apache.cassandra.service.paxos.Ballot.atUnixMicrosWithLsb; // TODO (feature): link with SimulateTime, and otherwise improve public class SimulatedBallots @@ -55,38 +55,43 @@ public class SimulatedBallots super(1L); } - public TimeUUID randomBallot(long timestamp, boolean isSerial) + public Ballot atUnixMicros(long unixMicros, Ballot.Flag flag) { - return TimeUUID.atUnixMicrosWithLsb(timestamp, uniqueSupplier.getAsLong(), isSerial); + return atUnixMicrosWithLsb(unixMicros, uniqueSupplier.getAsLong(), flag); } - public TimeUUID randomBallot(long from, long to, boolean isSerial) + public Ballot next(long minUnixMicros, Ballot.Flag flag) { - return TimeUUID.atUnixMicrosWithLsb(random.uniform(from, to), uniqueSupplier.getAsLong(), isSerial); + return Ballot.atUnixMicrosWithLsb(nextBallotTimestampMicros(minUnixMicros), uniqueSupplier.getAsLong(), flag); } - public long nextBallotTimestampMicros(long minTimestamp) + public Ballot stale(long from, long to, Ballot.Flag flag) + { + return Ballot.atUnixMicrosWithLsb(random.uniform(from, to), uniqueSupplier.getAsLong(), flag); + } + + private long nextBallotTimestampMicros(long minUnixMicros) { long next; switch (nextChoice.choose(random)) { default: throw new IllegalStateException(); case TO_LATEST: - minTimestamp = Math.max(latest.get(), minTimestamp); + minUnixMicros = Math.max(latest.get(), minUnixMicros); case ONE: - next = accumulateAndGet(minTimestamp, (a, b) -> Math.max(a, b) + 1); + next = accumulateAndGet(minUnixMicros, (a, b) -> Math.max(a, b) + 1); break; case JUMP: long jump = Math.max(1, nextJump.getAsLong()); next = addAndGet(jump); - if (next < minTimestamp) - next = accumulateAndGet(minTimestamp, (a, b) -> Math.max(a, b) + 1); + if (next < minUnixMicros) + next = accumulateAndGet(minUnixMicros, (a, b) -> Math.max(a, b) + 1); } latest.accumulateAndGet(next, Math::max); return next; } - public long prevBallotTimestampMicros() + public long prevUnixMicros() { return get(); } diff --git a/test/simulator/main/org/apache/cassandra/simulator/systems/SimulatedExecution.java b/test/simulator/main/org/apache/cassandra/simulator/systems/SimulatedExecution.java index d3fb015862..bb6387908e 100644 --- a/test/simulator/main/org/apache/cassandra/simulator/systems/SimulatedExecution.java +++ b/test/simulator/main/org/apache/cassandra/simulator/systems/SimulatedExecution.java @@ -50,6 +50,7 @@ public class SimulatedExecution implements InterceptorOfExecution @Override public void cancelPending(Object task) { throw new UnsupportedOperationException(); } @Override public void submitUnmanaged(Runnable task) { throw new UnsupportedOperationException(); } @Override public void submitAndAwaitPause(Runnable task, InterceptorOfConsequences interceptor) { throw new UnsupportedOperationException(); } + @Override public OrderOn orderAppliesAfterScheduling() { throw new UnsupportedOperationException(); } @Override public int concurrency() { return Integer.MAX_VALUE; } } diff --git a/test/simulator/main/org/apache/cassandra/simulator/systems/SimulatedFutureActionScheduler.java b/test/simulator/main/org/apache/cassandra/simulator/systems/SimulatedFutureActionScheduler.java index 1f1a31c561..f66999bb5a 100644 --- a/test/simulator/main/org/apache/cassandra/simulator/systems/SimulatedFutureActionScheduler.java +++ b/test/simulator/main/org/apache/cassandra/simulator/systems/SimulatedFutureActionScheduler.java @@ -32,6 +32,7 @@ import org.apache.cassandra.simulator.utils.KindOfSequence.NetworkDecision; import org.apache.cassandra.simulator.utils.KindOfSequence.Period; import static org.apache.cassandra.simulator.FutureActionScheduler.Deliver.DELIVER; +import static org.apache.cassandra.simulator.FutureActionScheduler.Deliver.DELIVER_AND_TIMEOUT; import static org.apache.cassandra.simulator.FutureActionScheduler.Deliver.FAILURE; import static org.apache.cassandra.simulator.FutureActionScheduler.Deliver.TIMEOUT; @@ -148,10 +149,13 @@ public class SimulatedFutureActionScheduler implements FutureActionScheduler, To if (isInDropPartition.get(from) != isInDropPartition.get(to)) return TIMEOUT; - if(!config.dropMessage.get(random, from, to)) + if (!config.dropMessage.get(random, from, to)) return DELIVER; - if(random.decide(0.5f)) + if (random.decide(0.5f)) + return DELIVER_AND_TIMEOUT; + + if (random.decide(0.5f)) return TIMEOUT; return FAILURE; @@ -167,9 +171,9 @@ public class SimulatedFutureActionScheduler implements FutureActionScheduler, To } @Override - public long messageTimeoutNanos(long expiresAfterNanos) + public long messageTimeoutNanos(long expiresAtNanos, long expirationIntervalNanos) { - return time.nanoTime() + expiresAfterNanos + random.uniform(0, expiresAfterNanos / 2); + return expiresAtNanos + random.uniform(0, expirationIntervalNanos / 2); } @Override diff --git a/test/simulator/main/org/apache/cassandra/simulator/systems/SimulatedSystems.java b/test/simulator/main/org/apache/cassandra/simulator/systems/SimulatedSystems.java index 6859fc709e..c3cf46a5d8 100644 --- a/test/simulator/main/org/apache/cassandra/simulator/systems/SimulatedSystems.java +++ b/test/simulator/main/org/apache/cassandra/simulator/systems/SimulatedSystems.java @@ -38,7 +38,6 @@ public class SimulatedSystems { public final RandomSource random; public final SimulatedTime time; - public final SimulatedWaits waits; public final SimulatedMessageDelivery delivery; public final SimulatedExecution execution; public final SimulatedBallots ballots; @@ -51,19 +50,18 @@ public class SimulatedSystems public SimulatedSystems(SimulatedSystems copy) { - this(copy.random, copy.time, copy.waits, copy.delivery, copy.execution, copy.ballots, copy.failureDetector, copy.snitch, copy.futureScheduler, copy.debug, copy.failures, copy.topologyListeners); + this(copy.random, copy.time, copy.delivery, copy.execution, copy.ballots, copy.failureDetector, copy.snitch, copy.futureScheduler, copy.debug, copy.failures, copy.topologyListeners); } - public SimulatedSystems(RandomSource random, SimulatedTime time, SimulatedWaits waits, SimulatedMessageDelivery delivery, SimulatedExecution execution, SimulatedBallots ballots, SimulatedFailureDetector failureDetector, SimulatedSnitch snitch, FutureActionScheduler futureScheduler, Debug debug, Failures failures) + public SimulatedSystems(RandomSource random, SimulatedTime time, SimulatedMessageDelivery delivery, SimulatedExecution execution, SimulatedBallots ballots, SimulatedFailureDetector failureDetector, SimulatedSnitch snitch, FutureActionScheduler futureScheduler, Debug debug, Failures failures) { - this(random, time, waits, delivery, execution, ballots, failureDetector, snitch, futureScheduler, debug, failures, new ArrayList<>()); + this(random, time, delivery, execution, ballots, failureDetector, snitch, futureScheduler, debug, failures, new ArrayList<>()); } - private SimulatedSystems(RandomSource random, SimulatedTime time, SimulatedWaits waits, SimulatedMessageDelivery delivery, SimulatedExecution execution, SimulatedBallots ballots, SimulatedFailureDetector failureDetector, SimulatedSnitch snitch, FutureActionScheduler futureScheduler, Debug debug, Failures failures, List topologyListeners) + private SimulatedSystems(RandomSource random, SimulatedTime time, SimulatedMessageDelivery delivery, SimulatedExecution execution, SimulatedBallots ballots, SimulatedFailureDetector failureDetector, SimulatedSnitch snitch, FutureActionScheduler futureScheduler, Debug debug, Failures failures, List topologyListeners) { this.random = random; this.time = time; - this.waits = waits; this.delivery = delivery; this.execution = execution; this.ballots = ballots; diff --git a/test/simulator/main/org/apache/cassandra/simulator/systems/SimulatedTime.java b/test/simulator/main/org/apache/cassandra/simulator/systems/SimulatedTime.java index aac9674dda..ae20d57590 100644 --- a/test/simulator/main/org/apache/cassandra/simulator/systems/SimulatedTime.java +++ b/test/simulator/main/org/apache/cassandra/simulator/systems/SimulatedTime.java @@ -18,6 +18,13 @@ package org.apache.cassandra.simulator.systems; +import java.util.ArrayList; +import java.util.List; +import java.util.function.LongConsumer; +import java.util.regex.Pattern; + +import com.google.common.base.Preconditions; + import org.apache.cassandra.distributed.api.IIsolatedExecutor; import org.apache.cassandra.distributed.impl.IsolatedExecutor; import org.apache.cassandra.simulator.RandomSource; @@ -25,7 +32,10 @@ import org.apache.cassandra.simulator.utils.KindOfSequence; import org.apache.cassandra.simulator.utils.KindOfSequence.Period; import org.apache.cassandra.simulator.utils.LongRange; import org.apache.cassandra.utils.Clock; +import org.apache.cassandra.utils.Closeable; +import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.MonotonicClock; +import org.apache.cassandra.utils.MonotonicClock.AbstractEpochSamplingClock.AlmostSameTime; import org.apache.cassandra.utils.MonotonicClockTranslation; import org.apache.cassandra.utils.Shared; @@ -39,26 +49,119 @@ import static org.apache.cassandra.simulator.RandomSource.Choices.uniform; // TODO (cleanup): when we encounter an exception and unwind the simulation, we should restore normal time to go with normal waits etc. public class SimulatedTime { - public static class Throwing implements Clock, MonotonicClock + private static final Pattern PERMITTED_TIME_THREADS = Pattern.compile("(logback|SimulationLiveness|Reconcile)[-:][0-9]+"); + + @Shared(scope = Shared.Scope.SIMULATION) + public interface Listener { - public long nanoTime() { throw new IllegalStateException("Using time is not allowed during simulation"); } - public long currentTimeMillis() { throw new IllegalStateException("Using time is not allowed during simulation"); } - public long now() { throw new IllegalStateException("Using time is not allowed during simulation"); } - public long error() { throw new IllegalStateException("Using time is not allowed during simulation"); } - public MonotonicClockTranslation translate() { throw new IllegalStateException("Using time is not allowed during simulation"); } - public boolean isAfter(long instant) { throw new IllegalStateException("Using time is not allowed during simulation"); } - public boolean isAfter(long now, long instant) { throw new IllegalStateException("Using time is not allowed during simulation"); } + void accept(String kind, long value); } @Shared(scope = Shared.Scope.SIMULATION) - public interface LocalTime extends Clock, MonotonicClock + public interface ClockAndMonotonicClock extends Clock, MonotonicClock { - long relativeNanosToAbsolute(long relativeNanos); - long absoluteToRelativeNanos(long absoluteNanos); - long localToGlobal(long absoluteNanos); + } + + @Shared(scope = Shared.Scope.SIMULATION) + public interface LocalTime extends ClockAndMonotonicClock + { + long relativeToLocalNanos(long relativeNanos); + long relativeToGlobalNanos(long relativeNanos); + long localToRelativeNanos(long absoluteLocalNanos); + long localToGlobalNanos(long absoluteLocalNanos); long nextGlobalMonotonicMicros(); } + @PerClassLoader + private static class Disabled extends Clock.Default implements LocalTime + { + @Override + public long now() + { + return nanoTime(); + } + + @Override + public long error() + { + return 0; + } + + @Override + public MonotonicClockTranslation translate() + { + return new AlmostSameTime(System.currentTimeMillis(), System.nanoTime(), 0L); + } + + @Override + public boolean isAfter(long instant) + { + return isAfter(System.nanoTime(), instant); + } + + @Override + public boolean isAfter(long now, long instant) + { + return now > instant; + } + + @Override + public long relativeToLocalNanos(long relativeNanos) + { + return System.nanoTime() + relativeNanos; + } + + @Override + public long relativeToGlobalNanos(long relativeNanos) + { + return System.nanoTime() + relativeNanos; + } + + @Override + public long localToRelativeNanos(long absoluteLocalNanos) + { + return absoluteLocalNanos - System.nanoTime(); + } + + @Override + public long localToGlobalNanos(long absoluteLocalNanos) + { + return absoluteLocalNanos; + } + + @Override + public long nextGlobalMonotonicMicros() + { + return FBUtilities.timestampMicros(); + } + } + + public static class Delegating implements ClockAndMonotonicClock + { + final Disabled disabled = new Disabled(); + private ClockAndMonotonicClock check() + { + Thread thread = Thread.currentThread(); + if (thread instanceof InterceptibleThread) + { + InterceptibleThread interceptibleThread = ((InterceptibleThread) thread); + if (interceptibleThread.isIntercepting()) + return interceptibleThread.time(); + } + if (PERMITTED_TIME_THREADS.matcher(Thread.currentThread().getName()).matches()) + return disabled; + throw new IllegalStateException("Using time is not allowed during simulation"); + } + + public long nanoTime() { return check().nanoTime(); } + public long currentTimeMillis() { return check().currentTimeMillis(); } + public long now() { return check().now(); } + public long error() { return check().error(); } + public MonotonicClockTranslation translate() { return check().translate(); } + public boolean isAfter(long instant) { return check().isAfter(instant); } + public boolean isAfter(long now, long instant) { return check().isAfter(now, instant); } + } + @PerClassLoader public static class Global implements Clock, MonotonicClock { @@ -104,24 +207,24 @@ public class SimulatedTime return current.isAfter(now, instant); } - public static long relativeToGlobalAbsoluteNanos(long relativeNanos) + public static long relativeToGlobalNanos(long relativeNanos) { - return current.localToGlobal(current.relativeNanosToAbsolute(relativeNanos)); + return current.relativeToGlobalNanos(relativeNanos); } - public static long relativeToAbsoluteNanos(long relativeNanos) + public static long relativeToLocalNanos(long relativeNanos) { - return current.relativeNanosToAbsolute(relativeNanos); + return current.relativeToLocalNanos(relativeNanos); } - public static long absoluteToRelativeNanos(long absoluteNanos) + public static long localToRelativeNanos(long absoluteNanos) { - return current.absoluteToRelativeNanos(absoluteNanos); + return current.localToRelativeNanos(absoluteNanos); } public static long localToGlobalNanos(long absoluteNanos) { - return current.localToGlobal(absoluteNanos); + return current.localToGlobalNanos(absoluteNanos); } public static LocalTime current() @@ -139,13 +242,19 @@ public class SimulatedTime { current = newLocalTime; } + + public static void disable() + { + current = new Disabled(); + } } - private class InstanceTime implements LocalTime + public class InstanceTime implements LocalTime { final Period nanosDriftSupplier; - long localNanoTime; - long nanosDrift; + long from, to; + long baseDrift, nextDrift, lastLocalNanoTime, lastDrift, lastGlobal; + double diffPerGlobal; private InstanceTime(Period nanosDriftSupplier) { @@ -155,12 +264,27 @@ public class SimulatedTime @Override public long nanoTime() { - if (globalNanoTime + nanosDrift > localNanoTime) + long global = globalNanoTime; + if (lastGlobal == global) + return lastLocalNanoTime; + + if (global >= to) { - localNanoTime = globalNanoTime + nanosDrift; - nanosDrift = nanosDriftSupplier.get(random); + baseDrift = nextDrift; + nextDrift = nanosDriftSupplier.get(random); + from = global; + to = global + Math.max(baseDrift, nextDrift); + diffPerGlobal = (nextDrift - baseDrift) / (double)(to - from); + listener.accept("SetNextDrift", nextDrift); } - return localNanoTime; + + long drift = baseDrift + (long)(diffPerGlobal * (global - from)); + long local = global + drift; + lastGlobal = global; + lastDrift = drift; + lastLocalNanoTime = local; + listener.accept("ReadLocal", local); + return local; } @Override @@ -225,21 +349,27 @@ public class SimulatedTime } @Override - public long relativeNanosToAbsolute(long relativeNanos) + public long relativeToLocalNanos(long relativeNanos) { - return relativeNanos + localNanoTime; + return relativeNanos + lastLocalNanoTime; } @Override - public long absoluteToRelativeNanos(long absoluteNanos) + public long relativeToGlobalNanos(long relativeNanos) { - return absoluteNanos - localNanoTime; + return relativeNanos + globalNanoTime; } @Override - public long localToGlobal(long absoluteNanos) + public long localToRelativeNanos(long absoluteLocalNanos) { - return absoluteNanos + (globalNanoTime - localNanoTime); + return absoluteLocalNanos - lastLocalNanoTime; + } + + @Override + public long localToGlobalNanos(long absoluteLocalNanos) + { + return absoluteLocalNanos - lastDrift; } } @@ -248,12 +378,15 @@ public class SimulatedTime private final RandomSource random; private final Period discontinuityTimeSupplier; private final long millisEpoch; - private long globalNanoTime; + private volatile long globalNanoTime; private long futureTimestamp; private long discontinuityTime; private boolean permitDiscontinuities; + private final List onDiscontinuity = new ArrayList<>(); + private final Listener listener; + private InstanceTime[] instanceTimes; - public SimulatedTime(RandomSource random, long millisEpoch, LongRange nanoDriftRange, KindOfSequence kindOfDrift, Period discontinuityTimeSupplier) + public SimulatedTime(int nodeCount, RandomSource random, long millisEpoch, LongRange nanoDriftRange, KindOfSequence kindOfDrift, Period discontinuityTimeSupplier, Listener listener) { this.random = random; this.millisEpoch = millisEpoch; @@ -262,51 +395,81 @@ public class SimulatedTime this.kindOfDrift = kindOfDrift; this.discontinuityTime = MILLISECONDS.toNanos(random.uniform(500L, 30000L)); this.discontinuityTimeSupplier = discontinuityTimeSupplier; + this.listener = listener; + this.instanceTimes = new InstanceTime[nodeCount]; } - public void setup(ClassLoader classLoader) + public Closeable setup(int nodeNum, ClassLoader classLoader) { + Preconditions.checkState(instanceTimes[nodeNum - 1] == null); InstanceTime instanceTime = new InstanceTime(kindOfDrift.period(nanosDriftRange, random)); IsolatedExecutor.transferAdhoc((IIsolatedExecutor.SerializableConsumer) Global::setup, classLoader) .accept(instanceTime); + instanceTimes[nodeNum - 1] = instanceTime; + return IsolatedExecutor.transferAdhoc((IIsolatedExecutor.SerializableRunnable) Global::disable, classLoader)::run; + } + + public InstanceTime get(int nodeNum) + { + return instanceTimes[nodeNum - 1]; } public void permitDiscontinuities() { + listener.accept("PermitDiscontinuity", 1); permitDiscontinuities = true; - maybeApplyDiscontinuity(); - } - - private void maybeApplyDiscontinuity() - { - if (permitDiscontinuities && globalNanoTime >= discontinuityTime) - { - globalNanoTime += uniform(DAYS, HOURS, MINUTES).choose(random).toNanos(1L); - discontinuityTime = globalNanoTime + discontinuityTimeSupplier.get(random); - } + updateAndMaybeApplyDiscontinuity(globalNanoTime); } public void forbidDiscontinuities() { + listener.accept("PermitDiscontinuity", 0); permitDiscontinuities = false; } + private void updateAndMaybeApplyDiscontinuity(long newGlobal) + { + if (permitDiscontinuities && newGlobal >= discontinuityTime) + { + updateAndApplyDiscontinuity(newGlobal); + } + else + { + globalNanoTime = newGlobal; + listener.accept("SetGlobal", newGlobal); + } + } + + private void updateAndApplyDiscontinuity(long newGlobal) + { + long discontinuity = uniform(DAYS, HOURS, MINUTES).choose(random).toNanos(1L); + listener.accept("ApplyDiscontinuity", discontinuity); + discontinuityTime = newGlobal + discontinuity + discontinuityTimeSupplier.get(random); + globalNanoTime = newGlobal + discontinuity; + listener.accept("SetGlobal", newGlobal + discontinuity); + onDiscontinuity.forEach(l -> l.accept(discontinuity)); + } + public void tick(long nanos) { - if (nanos > globalNanoTime) + listener.accept("Tick", nanos); + long global = globalNanoTime; + if (nanos > global) { - globalNanoTime = nanos; - maybeApplyDiscontinuity(); + updateAndMaybeApplyDiscontinuity(nanos); } else { - ++globalNanoTime; + globalNanoTime = global + 1; + listener.accept("IncrGlobal", global + 1); } } public long nanoTime() { - return globalNanoTime; + long global = globalNanoTime; + listener.accept("ReadGlobal", global); + return global; } // make sure schema changes persist @@ -314,4 +477,14 @@ public class SimulatedTime { return ++futureTimestamp; } + + public void onDiscontinuity(LongConsumer onDiscontinuity) + { + this.onDiscontinuity.add(onDiscontinuity); + } + + public void onTimeEvent(String kind, long value) + { + listener.accept(kind, value); + } } diff --git a/test/simulator/main/org/apache/cassandra/simulator/systems/SimulatedWaits.java b/test/simulator/main/org/apache/cassandra/simulator/systems/SimulatedWaits.java deleted file mode 100644 index cd284900e8..0000000000 --- a/test/simulator/main/org/apache/cassandra/simulator/systems/SimulatedWaits.java +++ /dev/null @@ -1,156 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.cassandra.simulator.systems; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - -import org.apache.cassandra.distributed.api.IInvokableInstance; -import org.apache.cassandra.distributed.api.IIsolatedExecutor.SerializableBiFunction; -import org.apache.cassandra.distributed.api.IMessage; -import org.apache.cassandra.distributed.impl.IsolatedExecutor; -import org.apache.cassandra.simulator.OrderOn; -import org.apache.cassandra.simulator.RandomSource; -import org.apache.cassandra.simulator.systems.InterceptedWait.CaptureSites; -import org.apache.cassandra.utils.Closeable; - -import static org.apache.cassandra.simulator.SimulatorUtils.failWithOOM; - -public class SimulatedWaits implements InterceptorOfWaits -{ - private static final InterceptorOfConsequences DEFAULT_INTERCEPTOR = new InterceptorOfConsequences() - { - @Override - public void beforeInvocation(InterceptibleThread realThread) - { - } - - @Override - public void interceptMessage(IInvokableInstance from, IInvokableInstance to, IMessage message) - { - throw new AssertionError(); - } - - @Override - public void interceptWait(InterceptedWait wakeupWith) - { - throw new AssertionError(); - } - - @Override - public void interceptWakeup(InterceptedWait wakeup, InterceptorOfConsequences waitWasInterceptedBy) - { - wakeup.triggerBypass(); - } - - @Override - public void interceptExecution(InterceptedExecution invoke, OrderOn orderOn) - { - throw new AssertionError(); - } - - @Override - public void interceptTermination(boolean isThreadTermination) - { - throw new AssertionError(); - } - }; - - private final RandomSource random; - private boolean disabled = false; - private boolean captureWaitSites, captureWakeSites, captureNowSites; - private final List onStop = Collections.synchronizedList(new ArrayList<>()); - - public SimulatedWaits(RandomSource random) - { - this.random = random; - } - - public InterceptorOfGlobalMethods interceptGlobalMethods(ClassLoader classLoader) - { - InterceptorOfGlobalMethods interceptor = IsolatedExecutor.transferAdhoc((SerializableBiFunction) InterceptingGlobalMethods::new, classLoader).apply(this, random); - onStop.add(interceptor); - return interceptor; - } - - public CaptureSites captureWaitSite(Thread thread) - { - return captureWaitSites ? new CaptureSites(thread, captureNowSites) : captureWakeSites | captureNowSites ? new CaptureSites(thread, null, captureNowSites) : null; - } - - private void captureWakeSite(Thread thread, CaptureSites captureSites) - { - if (captureSites != null && captureWakeSites) - captureSites.registerWakeup(thread); - } - - @Override - public void interceptSignal(Thread signalledBy, InterceptedWait signalled, CaptureSites waitSites, InterceptorOfConsequences interceptedBy) - { - captureWakeSite(signalledBy, waitSites); - interceptorOrDefault(signalledBy).interceptWakeup(signalled, interceptedBy); - } - - private InterceptorOfConsequences interceptorOrDefault(Thread thread) - { - if (!(thread instanceof InterceptibleThread)) - return DEFAULT_INTERCEPTOR; - - return interceptorOrDefault((InterceptibleThread) thread); - } - - private InterceptorOfConsequences interceptorOrDefault(InterceptibleThread thread) - { - return thread.isIntercepting() ? thread : DEFAULT_INTERCEPTOR; - } - - public InterceptibleThread ifIntercepted() - { - Thread thread = Thread.currentThread(); - if (thread instanceof InterceptibleThread) - { - InterceptibleThread interceptibleThread = (InterceptibleThread) thread; - if (interceptibleThread.isIntercepting()) - return interceptibleThread; - } - - if (NonInterceptible.isPermitted()) - return null; - - if (!disabled) - throw failWithOOM(); - - return null; - } - - public void stop() - { - this.disabled = true; - onStop.forEach(Closeable::close); - // TODO (cleanup): signal remaining waiters for cleaner shutdown? - } - - public void captureStackTraces(boolean waitSites, boolean wakeSites, boolean nowSites) - { - captureWaitSites = waitSites; - captureWakeSites = wakeSites; - captureNowSites = nowSites; - } -} diff --git a/test/simulator/test/org/apache/cassandra/simulator/test/ShortPaxosSimulationTest.java b/test/simulator/test/org/apache/cassandra/simulator/test/ShortPaxosSimulationTest.java new file mode 100644 index 0000000000..119095f2c7 --- /dev/null +++ b/test/simulator/test/org/apache/cassandra/simulator/test/ShortPaxosSimulationTest.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.simulator.test; + +import java.io.IOException; +import java.util.concurrent.ThreadLocalRandom; + +import org.junit.Ignore; +import org.junit.Test; + +import org.apache.cassandra.simulator.paxos.PaxosSimulationRunner; + +public class ShortPaxosSimulationTest +{ + @Test + public void simulationTest() throws IOException + { + PaxosSimulationRunner.main(new String[] { "run", "-n", "3..6", "-t", "1000", "-c", "2", "--cluster-action-limit", "2", "-s", "30" }); + } + + @Test + public void selfReconcileTest() throws IOException + { + PaxosSimulationRunner.main(new String[] { "reconcile", "-n", "3..6", "-t", "1000", "-c", "2", "--cluster-action-limit", "2", "-s", "30", "--with-self" }); + } +} diff --git a/test/simulator/test/org/apache/cassandra/simulator/test/SimulationTestBase.java b/test/simulator/test/org/apache/cassandra/simulator/test/SimulationTestBase.java index 7e997fe7a2..87eb9b995c 100644 --- a/test/simulator/test/org/apache/cassandra/simulator/test/SimulationTestBase.java +++ b/test/simulator/test/org/apache/cassandra/simulator/test/SimulationTestBase.java @@ -24,6 +24,7 @@ import java.util.Collections; import java.util.function.Consumer; import java.util.function.Function; import java.util.function.IntSupplier; +import java.util.function.Predicate; import com.google.common.collect.Iterators; @@ -51,23 +52,24 @@ import org.apache.cassandra.simulator.asm.NemesisFieldSelectors; import org.apache.cassandra.simulator.systems.Failures; import org.apache.cassandra.simulator.systems.InterceptibleThread; import org.apache.cassandra.simulator.systems.InterceptingExecutorFactory; +import org.apache.cassandra.simulator.systems.InterceptingGlobalMethods; import org.apache.cassandra.simulator.systems.InterceptorOfGlobalMethods; import org.apache.cassandra.simulator.systems.SimulatedExecution; import org.apache.cassandra.simulator.systems.SimulatedQuery; import org.apache.cassandra.simulator.systems.SimulatedSystems; import org.apache.cassandra.simulator.systems.SimulatedTime; -import org.apache.cassandra.simulator.systems.SimulatedWaits; -import org.apache.cassandra.simulator.utils.KindOfSequence; import org.apache.cassandra.simulator.utils.LongRange; import org.apache.cassandra.utils.CloseableIterator; import static java.util.concurrent.TimeUnit.MILLISECONDS; import static java.util.concurrent.TimeUnit.NANOSECONDS; import static java.util.concurrent.TimeUnit.SECONDS; +import static org.apache.cassandra.simulator.ActionSchedule.Mode.STREAM_LIMITED; import static org.apache.cassandra.simulator.ActionSchedule.Mode.UNLIMITED; import static org.apache.cassandra.simulator.ClusterSimulation.ISOLATE; import static org.apache.cassandra.simulator.ClusterSimulation.SHARE; import static org.apache.cassandra.simulator.SimulatorUtils.failWithOOM; +import static org.apache.cassandra.simulator.systems.InterceptedWait.CaptureSites.Capture.NONE; import static org.apache.cassandra.simulator.utils.KindOfSequence.UNIFORM; import static org.apache.cassandra.utils.Shared.Scope.ANY; import static org.apache.cassandra.utils.Shared.Scope.SIMULATION; @@ -117,7 +119,7 @@ public class SimulationTestBase return new ActionPlan(ActionList.of(initialize()), Collections.singletonList(execute()), ActionList.empty()) - .iterator(-1, () -> 0L, simulated.time, new RunnableActionScheduler.Immediate(), scheduler, simulated.futureScheduler); + .iterator(STREAM_LIMITED, -1, () -> 0L, simulated.time, scheduler, simulated.futureScheduler); } public void run() @@ -190,21 +192,23 @@ public class SimulationTestBase public static void simulate(IIsolatedExecutor.SerializableRunnable[] runnables, IIsolatedExecutor.SerializableRunnable check) { + Failures failures = new Failures(); RandomSource random = new RandomSource.Default(); - SimulatedWaits waits = new SimulatedWaits(random); - SimulatedTime time = new SimulatedTime(random, 1577836800000L /*Jan 1st UTC*/, new LongRange(1, 100, MILLISECONDS, NANOSECONDS), - UNIFORM, UNIFORM.period(new LongRange(10L, 60L, SECONDS, NANOSECONDS), random)); + SimulatedTime time = new SimulatedTime(1, random, 1577836800000L /*Jan 1st UTC*/, new LongRange(1, 100, MILLISECONDS, NANOSECONDS), + UNIFORM, UNIFORM.period(new LongRange(10L, 60L, SECONDS, NANOSECONDS), random), (i1, i2) -> {}); SimulatedExecution execution = new SimulatedExecution(); + Predicate sharedClassPredicate = AbstractCluster.getSharedClassPredicate(ISOLATE, SHARE, ANY, SIMULATION); InstanceClassLoader classLoader = new InstanceClassLoader(1, 1, AbstractCluster.CURRENT_VERSION.classpath, Thread.currentThread().getContextClassLoader(), - AbstractCluster.getSharedClassPredicate(ISOLATE, SHARE, ANY, SIMULATION), - new InterceptClasses(() -> 1.0f, () -> 1.0f, NemesisFieldSelectors.get())::apply); + sharedClassPredicate, + new InterceptClasses(() -> 1.0f, () -> 1.0f, NemesisFieldSelectors.get(), ClassLoader.getSystemClassLoader(), sharedClassPredicate.negate())::apply); ThreadGroup tg = new ThreadGroup("test"); - InterceptorOfGlobalMethods interceptorOfGlobalMethods = waits.interceptGlobalMethods(classLoader); + InterceptorOfGlobalMethods interceptorOfGlobalMethods = new InterceptingGlobalMethods(NONE, null, failures, random); InterceptingExecutorFactory factory = execution.factory(interceptorOfGlobalMethods, classLoader, tg); + time.setup(1, classLoader); IsolatedExecutor.transferAdhoc((IIsolatedExecutor.SerializableConsumer) ExecutorFactory.Global::unsafeSet, classLoader) .accept(factory); @@ -215,7 +219,7 @@ public class SimulationTestBase return random.uniform(Integer.MIN_VALUE, Integer.MAX_VALUE); }); - SimulatedSystems simulated = new SimulatedSystems(random, time, waits, null, execution, null, null, null, new FutureActionScheduler() + SimulatedSystems simulated = new SimulatedSystems(random, time, null, execution, null, null, null, new FutureActionScheduler() { @Override public Deliver shouldDeliver(int from, int to) @@ -230,7 +234,7 @@ public class SimulationTestBase } @Override - public long messageTimeoutNanos(long expiresAfterNanos) + public long messageTimeoutNanos(long expiresAfterNanos, long expirationIntervalNanos) { return 0; } @@ -246,7 +250,7 @@ public class SimulationTestBase { return 0; } - }, new Debug(), new Failures()); + }, new Debug(), failures); RunnableActionScheduler runnableScheduler = new RunnableActionScheduler.RandomUniform(random); @@ -263,9 +267,9 @@ public class SimulationTestBase }; - ActionSchedule testSchedule = new ActionSchedule(simulated.time, simulated.futureScheduler, () -> 0, new Work(UNLIMITED, runnableScheduler, Collections.singletonList(ActionList.of(entrypoint)))); + ActionSchedule testSchedule = new ActionSchedule(simulated.time, simulated.futureScheduler, () -> 0, runnableScheduler, new Work(UNLIMITED, Collections.singletonList(ActionList.of(entrypoint)))); Iterators.advance(testSchedule, Integer.MAX_VALUE); - ActionSchedule checkSchedule = new ActionSchedule(simulated.time, simulated.futureScheduler, () -> 0, new Work(UNLIMITED, runnableScheduler, Collections.singletonList(ActionList.of(toAction(check, classLoader, factory, simulated))))); + ActionSchedule checkSchedule = new ActionSchedule(simulated.time, simulated.futureScheduler, () -> 0, runnableScheduler, new Work(UNLIMITED, Collections.singletonList(ActionList.of(toAction(check, classLoader, factory, simulated))))); Iterators.advance(checkSchedule, Integer.MAX_VALUE); } diff --git a/test/simulator/test/org/apache/cassandra/simulator/test/SimulationTest.java b/test/simulator/test/org/apache/cassandra/simulator/test/TrivialSimulationTest.java similarity index 98% rename from test/simulator/test/org/apache/cassandra/simulator/test/SimulationTest.java rename to test/simulator/test/org/apache/cassandra/simulator/test/TrivialSimulationTest.java index 648ec54f3e..d7b9e5b1a3 100644 --- a/test/simulator/test/org/apache/cassandra/simulator/test/SimulationTest.java +++ b/test/simulator/test/org/apache/cassandra/simulator/test/TrivialSimulationTest.java @@ -32,7 +32,7 @@ import org.apache.cassandra.utils.concurrent.CountDownLatch; import static org.apache.cassandra.simulator.cluster.ClusterActions.InitialConfiguration.initializeAll; -public class SimulationTest extends SimulationTestBase +public class TrivialSimulationTest extends SimulationTestBase { @Test public void trivialTest() throws IOException // for demonstration/experiment purposes diff --git a/test/unit/org/apache/cassandra/AbstractSerializationsTester.java b/test/unit/org/apache/cassandra/AbstractSerializationsTester.java index d79521b196..6503707498 100644 --- a/test/unit/org/apache/cassandra/AbstractSerializationsTester.java +++ b/test/unit/org/apache/cassandra/AbstractSerializationsTester.java @@ -29,11 +29,12 @@ import java.util.Map; public class AbstractSerializationsTester { - protected static final String CUR_VER = System.getProperty("cassandra.version", "4.0"); + protected static final String CUR_VER = System.getProperty("cassandra.version", "4.1"); protected static final Map VERSION_MAP = new HashMap () {{ put("3.0", MessagingService.VERSION_30); put("4.0", MessagingService.VERSION_40); + put("4.1", MessagingService.VERSION_41); }}; protected static final boolean EXECUTE_WRITES = Boolean.getBoolean("cassandra.test-serialization-writes"); diff --git a/test/unit/org/apache/cassandra/Util.java b/test/unit/org/apache/cassandra/Util.java index 8a1a62bfcd..5784eb1f71 100644 --- a/test/unit/org/apache/cassandra/Util.java +++ b/test/unit/org/apache/cassandra/Util.java @@ -88,6 +88,7 @@ import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.FilterFactory; import org.awaitility.Awaitility; +import static org.apache.cassandra.utils.Clock.Global.nanoTime; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; diff --git a/test/unit/org/apache/cassandra/config/DatabaseDescriptorRefTest.java b/test/unit/org/apache/cassandra/config/DatabaseDescriptorRefTest.java index 735664c36d..da06cb1d06 100644 --- a/test/unit/org/apache/cassandra/config/DatabaseDescriptorRefTest.java +++ b/test/unit/org/apache/cassandra/config/DatabaseDescriptorRefTest.java @@ -82,6 +82,8 @@ public class DatabaseDescriptorRefTest "org.apache.cassandra.config.Config$FlushCompression", "org.apache.cassandra.config.Config$InternodeCompression", "org.apache.cassandra.config.Config$MemtableAllocationType", + "org.apache.cassandra.config.Config$PaxosOnLinearizabilityViolation", + "org.apache.cassandra.config.Config$PaxosStatePurging", "org.apache.cassandra.config.Config$PaxosVariant", "org.apache.cassandra.config.Config$RepairCommandPoolFullStrategy", "org.apache.cassandra.config.Config$UserFunctionTimeoutPolicy", diff --git a/test/unit/org/apache/cassandra/config/LoadOldYAMLBackwardCompatibilityTest.java b/test/unit/org/apache/cassandra/config/LoadOldYAMLBackwardCompatibilityTest.java index d57009ef6a..39d37ce496 100644 --- a/test/unit/org/apache/cassandra/config/LoadOldYAMLBackwardCompatibilityTest.java +++ b/test/unit/org/apache/cassandra/config/LoadOldYAMLBackwardCompatibilityTest.java @@ -49,7 +49,7 @@ public class LoadOldYAMLBackwardCompatibilityTest assertEquals(DurationSpec.inMilliseconds(10000), config.range_request_timeout); assertEquals(DurationSpec.inMilliseconds(2000), config.write_request_timeout); assertEquals(DurationSpec.inMilliseconds(5000), config.counter_write_request_timeout); - assertEquals(DurationSpec.inMilliseconds(1000), config.cas_contention_timeout); + assertEquals(DurationSpec.inMilliseconds(1800), config.cas_contention_timeout); assertEquals(DurationSpec.inMilliseconds(60000), config.truncate_request_timeout); assertEquals(DurationSpec.inSeconds(300), config.streaming_keep_alive_period); assertEquals(DurationSpec.inMilliseconds(500), config.slow_query_log_timeout); diff --git a/test/unit/org/apache/cassandra/config/ParseAndConvertUnitsTest.java b/test/unit/org/apache/cassandra/config/ParseAndConvertUnitsTest.java index 0421e3e435..e760826a89 100644 --- a/test/unit/org/apache/cassandra/config/ParseAndConvertUnitsTest.java +++ b/test/unit/org/apache/cassandra/config/ParseAndConvertUnitsTest.java @@ -45,7 +45,7 @@ public class ParseAndConvertUnitsTest assertEquals(DurationSpec.inMilliseconds(10000), config.range_request_timeout); assertEquals(DurationSpec.inMilliseconds(2000), config.write_request_timeout); assertEquals(DurationSpec.inMilliseconds(5000), config.counter_write_request_timeout); - assertEquals(DurationSpec.inMilliseconds(1000), config.cas_contention_timeout); + assertEquals(DurationSpec.inMilliseconds(1800), config.cas_contention_timeout); assertEquals(DurationSpec.inMilliseconds(60000), config.truncate_request_timeout); assertEquals(DurationSpec.inSeconds(300), config.streaming_keep_alive_period); assertEquals(DurationSpec.inMilliseconds(500), config.slow_query_log_timeout); diff --git a/test/unit/org/apache/cassandra/cql3/PstmtPersistenceTest.java b/test/unit/org/apache/cassandra/cql3/PstmtPersistenceTest.java index 8d96f09ac4..2d0ea3b1ea 100644 --- a/test/unit/org/apache/cassandra/cql3/PstmtPersistenceTest.java +++ b/test/unit/org/apache/cassandra/cql3/PstmtPersistenceTest.java @@ -32,7 +32,6 @@ import org.apache.cassandra.db.marshal.UTF8Type; import org.apache.cassandra.schema.SchemaConstants; import org.apache.cassandra.schema.SchemaKeyspaceTables; import org.apache.cassandra.service.ClientState; -import org.apache.cassandra.service.QueryState; import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.MD5Digest; diff --git a/test/unit/org/apache/cassandra/db/filter/ColumnFilterTest.java b/test/unit/org/apache/cassandra/db/filter/ColumnFilterTest.java index d28d0dc552..b23aa0731f 100644 --- a/test/unit/org/apache/cassandra/db/filter/ColumnFilterTest.java +++ b/test/unit/org/apache/cassandra/db/filter/ColumnFilterTest.java @@ -97,6 +97,7 @@ public class ColumnFilterTest DatabaseDescriptor.setSeedProvider(Arrays::asList); DatabaseDescriptor.setEndpointSnitch(new SimpleSnitch()); DatabaseDescriptor.setDefaultFailureDetector(); + DatabaseDescriptor.setPartitionerUnsafe(new Murmur3Partitioner()); Gossiper.instance.start(0); } diff --git a/test/unit/org/apache/cassandra/db/partition/PartitionUpdateTest.java b/test/unit/org/apache/cassandra/db/partition/PartitionUpdateTest.java index a4555c888b..771facf132 100644 --- a/test/unit/org/apache/cassandra/db/partition/PartitionUpdateTest.java +++ b/test/unit/org/apache/cassandra/db/partition/PartitionUpdateTest.java @@ -58,7 +58,7 @@ public class PartitionUpdateTest extends CQLTester builder.newRow().add("s", 1); builder.newRow(1).add("a", 2); int size1 = builder.build().dataSize(); - Assert.assertEquals(44, size1); + Assert.assertEquals(94, size1); builder = UpdateBuilder.create(cfm, "key0"); builder.newRow(1).add("a", 2); diff --git a/test/unit/org/apache/cassandra/index/internal/CassandraIndexTest.java b/test/unit/org/apache/cassandra/index/internal/CassandraIndexTest.java index 2c46bdb0f1..84eb8d0746 100644 --- a/test/unit/org/apache/cassandra/index/internal/CassandraIndexTest.java +++ b/test/unit/org/apache/cassandra/index/internal/CassandraIndexTest.java @@ -569,7 +569,7 @@ public class CassandraIndexTest extends CQLTester Awaitility.await() .atMost(1, TimeUnit.MINUTES) .pollDelay(1, TimeUnit.SECONDS) - .untilAsserted(() -> assertRows(execute(selectBuiltIndexesQuery))); + .untilAsserted(() -> assertRows(execute(selectBuiltIndexesQuery), row("system", "PaxosUncommittedIndex", null))); String indexName = "build_remove_test_idx"; String tableName = createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY (a, b))"); @@ -577,18 +577,18 @@ public class CassandraIndexTest extends CQLTester waitForIndex(KEYSPACE, tableName, indexName); // check that there are no other rows in the built indexes table - assertRows(execute(selectBuiltIndexesQuery), row(KEYSPACE, indexName, null)); + assertRows(execute(selectBuiltIndexesQuery), row(KEYSPACE, indexName, null), row("system", "PaxosUncommittedIndex", null)); // rebuild the index and verify the built status table getCurrentColumnFamilyStore().rebuildSecondaryIndex(indexName); waitForIndex(KEYSPACE, tableName, indexName); // check that there are no other rows in the built indexes table - assertRows(execute(selectBuiltIndexesQuery), row(KEYSPACE, indexName, null)); + assertRows(execute(selectBuiltIndexesQuery), row(KEYSPACE, indexName, null), row("system", "PaxosUncommittedIndex", null)); // check that dropping the index removes it from the built indexes table dropIndex("DROP INDEX %s." + indexName); - assertRows(execute(selectBuiltIndexesQuery)); + assertRows(execute(selectBuiltIndexesQuery), row("system", "PaxosUncommittedIndex", null)); } diff --git a/test/unit/org/apache/cassandra/io/sstable/metadata/MetadataSerializerTest.java b/test/unit/org/apache/cassandra/io/sstable/metadata/MetadataSerializerTest.java index 6e1bd408dd..480c27fead 100644 --- a/test/unit/org/apache/cassandra/io/sstable/metadata/MetadataSerializerTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/metadata/MetadataSerializerTest.java @@ -39,9 +39,6 @@ import org.apache.cassandra.io.sstable.Descriptor; import org.apache.cassandra.io.sstable.format.SSTableFormat; import org.apache.cassandra.io.sstable.format.Version; import org.apache.cassandra.io.sstable.format.big.BigFormat; -import org.apache.cassandra.io.util.DataOutputStreamPlus; -import org.apache.cassandra.io.util.FileUtils; -import org.apache.cassandra.io.util.RandomAccessReader; import org.apache.cassandra.utils.Throwables; import static org.junit.Assert.assertEquals; diff --git a/test/unit/org/apache/cassandra/locator/ReplicaPlansTest.java b/test/unit/org/apache/cassandra/locator/ReplicaPlansTest.java index 4d0dd47ba6..8231b03a48 100644 --- a/test/unit/org/apache/cassandra/locator/ReplicaPlansTest.java +++ b/test/unit/org/apache/cassandra/locator/ReplicaPlansTest.java @@ -89,7 +89,7 @@ public class ReplicaPlansTest Keyspace ks = ks(ImmutableSet.of(EP1, EP2, EP3), ImmutableMap.of("DC1", "3", "DC2", "3")); EndpointsForToken natural = EndpointsForToken.of(token, full(EP1), full(EP2), full(EP3), full(EP4), full(EP5), full(EP6)); EndpointsForToken pending = EndpointsForToken.empty(token); - ReplicaPlan.ForTokenWrite plan = ReplicaPlans.forWrite(ks, ConsistencyLevel.EACH_QUORUM, natural, pending, Predicates.alwaysTrue(), ReplicaPlans.writeNormal); + ReplicaPlan.ForWrite plan = ReplicaPlans.forWrite(ks, ConsistencyLevel.EACH_QUORUM, natural, pending, Predicates.alwaysTrue(), ReplicaPlans.writeNormal); assertEquals(natural, plan.liveAndDown); assertEquals(natural, plan.live); assertEquals(natural, plan.contacts()); @@ -99,7 +99,7 @@ public class ReplicaPlansTest Keyspace ks = ks(ImmutableSet.of(EP1, EP2, EP3), ImmutableMap.of("DC1", "3", "DC2", "3")); EndpointsForToken natural = EndpointsForToken.of(token, full(EP1), full(EP2), trans(EP3), full(EP4), full(EP5), trans(EP6)); EndpointsForToken pending = EndpointsForToken.empty(token); - ReplicaPlan.ForTokenWrite plan = ReplicaPlans.forWrite(ks, ConsistencyLevel.EACH_QUORUM, natural, pending, Predicates.alwaysTrue(), ReplicaPlans.writeNormal); + ReplicaPlan.ForWrite plan = ReplicaPlans.forWrite(ks, ConsistencyLevel.EACH_QUORUM, natural, pending, Predicates.alwaysTrue(), ReplicaPlans.writeNormal); assertEquals(natural, plan.liveAndDown); assertEquals(natural, plan.live); EndpointsForToken expectContacts = EndpointsForToken.of(token, full(EP1), full(EP2), full(EP4), full(EP5)); diff --git a/test/unit/org/apache/cassandra/metrics/DecayingEstimatedHistogramReservoirTest.java b/test/unit/org/apache/cassandra/metrics/DecayingEstimatedHistogramReservoirTest.java index 25e64f5d3c..ef57dfd474 100644 --- a/test/unit/org/apache/cassandra/metrics/DecayingEstimatedHistogramReservoirTest.java +++ b/test/unit/org/apache/cassandra/metrics/DecayingEstimatedHistogramReservoirTest.java @@ -29,9 +29,10 @@ import org.junit.Assert; import org.junit.Ignore; import org.junit.Test; -import com.codahale.metrics.Clock; import com.codahale.metrics.Snapshot; import org.apache.cassandra.utils.EstimatedHistogram; +import org.apache.cassandra.utils.MonotonicClock; +import org.apache.cassandra.utils.MonotonicClockTranslation; import org.apache.cassandra.utils.Pair; import org.quicktheories.core.Gen; @@ -514,11 +515,11 @@ public class DecayingEstimatedHistogramReservoirTest DecayingEstimatedHistogramReservoir histogram = new DecayingEstimatedHistogramReservoir(clock); - clock.addMillis(DecayingEstimatedHistogramReservoir.LANDMARK_RESET_INTERVAL_IN_MS - 1_000L); + clock.addNanos(DecayingEstimatedHistogramReservoir.LANDMARK_RESET_INTERVAL_IN_NS - TimeUnit.SECONDS.toNanos(1L)); - while (clock.getTime() < DecayingEstimatedHistogramReservoir.LANDMARK_RESET_INTERVAL_IN_MS + 1_000L) + while (clock.now() < DecayingEstimatedHistogramReservoir.LANDMARK_RESET_INTERVAL_IN_NS + TimeUnit.SECONDS.toNanos(1L)) { - clock.addMillis(900); + clock.addNanos(TimeUnit.MILLISECONDS.toNanos(900)); for (int i = 0; i < 1_000_000; i++) { histogram.update(1000); @@ -540,7 +541,7 @@ public class DecayingEstimatedHistogramReservoirTest DecayingEstimatedHistogramReservoir histogram = new DecayingEstimatedHistogramReservoir(clock); DecayingEstimatedHistogramReservoir another = new DecayingEstimatedHistogramReservoir(clock); - clock.addMillis(DecayingEstimatedHistogramReservoir.LANDMARK_RESET_INTERVAL_IN_MS - 1_000L); + clock.addNanos(DecayingEstimatedHistogramReservoir.LANDMARK_RESET_INTERVAL_IN_NS - TimeUnit.SECONDS.toNanos(1L)); histogram.update(1000); clock.addMillis(100); @@ -584,27 +585,52 @@ public class DecayingEstimatedHistogramReservoirTest assertTrue("Expected less than [" + Math.round(expectedValue * 1.2) + "] but actual is [" + actualValue + "]", actualValue < Math.round(expectedValue * 1.2)); } - public class TestClock extends Clock { + public class TestClock implements MonotonicClock + { private long tick = 0; + public void addNanos(long nanos) + { + tick += nanos; + } + public void addMillis(long millis) { - tick += millis * 1_000_000L; + tick += TimeUnit.MILLISECONDS.toNanos(millis); } public void addSeconds(long seconds) { - tick += seconds * 1_000_000_000L; + tick += TimeUnit.SECONDS.toNanos(seconds); } - public long getTick() + public long now() { return tick; } - public long getTime() + @Override + public long error() { - return tick / 1_000_000L; - }; + return 0; + } + + @Override + public MonotonicClockTranslation translate() + { + throw new UnsupportedOperationException(); + } + + @Override + public boolean isAfter(long instant) + { + throw new UnsupportedOperationException(); + } + + @Override + public boolean isAfter(long now, long instant) + { + throw new UnsupportedOperationException(); + } } } diff --git a/test/unit/org/apache/cassandra/net/WriteCallbackInfoTest.java b/test/unit/org/apache/cassandra/net/WriteCallbackInfoTest.java index 6b21f17599..2bc24dc840 100644 --- a/test/unit/org/apache/cassandra/net/WriteCallbackInfoTest.java +++ b/test/unit/org/apache/cassandra/net/WriteCallbackInfoTest.java @@ -39,7 +39,8 @@ import org.apache.cassandra.service.paxos.Commit; import org.apache.cassandra.utils.ByteBufferUtil; import static org.apache.cassandra.locator.ReplicaUtils.full; -import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID; +import static org.apache.cassandra.service.paxos.Ballot.Flag.NONE; +import static org.apache.cassandra.service.paxos.BallotGenerator.Global.nextBallot; public class WriteCallbackInfoTest { @@ -80,7 +81,7 @@ public class WriteCallbackInfoTest { TableMetadata metadata = MockSchema.newTableMetadata("", ""); Object payload = verb == Verb.PAXOS_COMMIT_REQ - ? new Commit(nextTimeUUID(), new PartitionUpdate.Builder(metadata, ByteBufferUtil.EMPTY_BYTE_BUFFER, RegularAndStaticColumns.NONE, 1).build()) + ? new Commit(nextBallot(NONE), new PartitionUpdate.Builder(metadata, ByteBufferUtil.EMPTY_BYTE_BUFFER, RegularAndStaticColumns.NONE, 1).build()) : new Mutation(PartitionUpdate.simpleBuilder(metadata, "").build()); RequestCallbacks.WriteCallbackInfo wcbi = new RequestCallbacks.WriteCallbackInfo(Message.out(verb, payload), full(testEp), null, cl, allowHints); diff --git a/test/unit/org/apache/cassandra/repair/RepairJobTest.java b/test/unit/org/apache/cassandra/repair/RepairJobTest.java index 5d7b6273e2..27977e6277 100644 --- a/test/unit/org/apache/cassandra/repair/RepairJobTest.java +++ b/test/unit/org/apache/cassandra/repair/RepairJobTest.java @@ -46,6 +46,7 @@ import org.junit.Test; import org.apache.cassandra.SchemaLoader; import org.apache.cassandra.concurrent.ExecutorFactory; import org.apache.cassandra.concurrent.ExecutorPlus; +import org.apache.cassandra.db.ConsistencyLevel; import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.dht.ByteOrderedPartitioner; import org.apache.cassandra.dht.IPartitioner; @@ -61,6 +62,10 @@ import org.apache.cassandra.repair.messages.RepairMessage; import org.apache.cassandra.repair.messages.SyncRequest; import org.apache.cassandra.schema.KeyspaceParams; import org.apache.cassandra.service.ActiveRepairService; +import org.apache.cassandra.service.paxos.Paxos; +import org.apache.cassandra.service.paxos.cleanup.PaxosCleanupRequest; +import org.apache.cassandra.service.paxos.cleanup.PaxosCleanupResponse; +import org.apache.cassandra.service.paxos.cleanup.PaxosCleanupSession; import org.apache.cassandra.streaming.PreviewKind; import org.apache.cassandra.streaming.SessionSummary; import org.apache.cassandra.utils.ByteBufferUtil; @@ -72,12 +77,18 @@ import org.apache.cassandra.utils.Throwables; import org.apache.cassandra.utils.TimeUUID; import org.apache.cassandra.utils.asserts.SyncTaskListAssert; +import static java.util.Collections.emptySet; +import static org.apache.cassandra.repair.RepairParallelism.SEQUENTIAL; +import static org.apache.cassandra.streaming.PreviewKind.NONE; import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID; import static org.apache.cassandra.utils.asserts.SyncTaskAssert.assertThat; import static org.apache.cassandra.utils.asserts.SyncTaskListAssert.assertThat; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.apache.cassandra.net.Verb.PAXOS2_CLEANUP_START_PREPARE_REQ; +import static org.apache.cassandra.net.Verb.PAXOS2_CLEANUP_REQ; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -114,9 +125,11 @@ public class RepairJobTest public MeasureableRepairSession(TimeUUID parentRepairSession, TimeUUID id, CommonRange commonRange, String keyspace, RepairParallelism parallelismDegree, boolean isIncremental, boolean pullRepair, - PreviewKind previewKind, boolean optimiseStreams, String... cfnames) + PreviewKind previewKind, boolean optimiseStreams, boolean repairPaxos, boolean paxosOnly, + String... cfnames) { - super(parentRepairSession, id, commonRange, keyspace, parallelismDegree, isIncremental, pullRepair, previewKind, optimiseStreams, cfnames); + super(parentRepairSession, id, commonRange, keyspace, parallelismDegree, isIncremental, pullRepair, + previewKind, optimiseStreams, repairPaxos, paxosOnly, cfnames); } protected ExecutorPlus createExecutor() @@ -176,9 +189,9 @@ public class RepairJobTest ActiveRepairService.UNREPAIRED_SSTABLE, false, PreviewKind.NONE); this.session = new MeasureableRepairSession(parentRepairSession, nextTimeUUID(), - new CommonRange(neighbors, Collections.emptySet(), FULL_RANGE), - KEYSPACE, RepairParallelism.SEQUENTIAL, false, false, - PreviewKind.NONE, false, CF); + new CommonRange(neighbors, emptySet(), FULL_RANGE), + KEYSPACE, SEQUENTIAL, false, false, + NONE, false, true, false, CF); this.job = new RepairJob(session, CF); this.sessionJobDesc = new RepairJobDesc(session.parentRepairSession, session.getId(), @@ -212,6 +225,7 @@ public class RepairJobTest job.run(); + Thread.sleep(1000); RepairResult result = job.get(TEST_TIMEOUT_S, TimeUnit.SECONDS); // Since there are no differences, there should be nothing to sync. @@ -845,32 +859,49 @@ public class RepairJobTest { MessagingService.instance().inboundSink.add(message -> message.verb().isResponse()); MessagingService.instance().outboundSink.add((message, to) -> { - if (message == null || !(message.payload instanceof RepairMessage)) + if (message == null || !(message.payload instanceof RepairMessage)) + return false; + + if (message.verb() == PAXOS2_CLEANUP_START_PREPARE_REQ) + { + Message messageIn = message.responseWith(Paxos.newBallot(null, ConsistencyLevel.SERIAL)); + MessagingService.instance().inboundSink.accept(messageIn); + return false; + } + + if (message.verb() == PAXOS2_CLEANUP_REQ) + { + PaxosCleanupRequest request = (PaxosCleanupRequest) message.payload; + PaxosCleanupSession.finishSession(to, new PaxosCleanupResponse(request.session, true, null)); + return false; + } + + if (!(message.payload instanceof RepairMessage)) + return false; + + // So different Thread's messages don't overwrite each other. + synchronized (MESSAGE_LOCK) + { + messageCapture.add(message); + } + + switch (message.verb()) + { + case SNAPSHOT_MSG: + MessagingService.instance().callbacks.removeAndRespond(message.id(), to, message.emptyResponse()); + break; + case VALIDATION_REQ: + session.validationComplete(sessionJobDesc, to, mockTrees.get(to)); + break; + case SYNC_REQ: + SyncRequest syncRequest = (SyncRequest) message.payload; + session.syncComplete(sessionJobDesc, new SyncNodePair(syncRequest.src, syncRequest.dst), + true, Collections.emptyList()); + break; + default: + break; + } return false; - - // So different Thread's messages don't overwrite each other. - synchronized (MESSAGE_LOCK) - { - messageCapture.add(message); - } - - switch (message.verb()) - { - case SNAPSHOT_MSG: - MessagingService.instance().callbacks.removeAndRespond(message.id(), to, message.emptyResponse()); - break; - case VALIDATION_REQ: - session.validationComplete(sessionJobDesc, to, mockTrees.get(to)); - break; - case SYNC_REQ: - SyncRequest syncRequest = (SyncRequest) message.payload; - session.syncComplete(sessionJobDesc, new SyncNodePair(syncRequest.src, syncRequest.dst), - true, Collections.emptyList()); - break; - default: - break; - } - return false; }); } } diff --git a/test/unit/org/apache/cassandra/repair/RepairSessionTest.java b/test/unit/org/apache/cassandra/repair/RepairSessionTest.java index 96314a4e4b..4bddeac598 100644 --- a/test/unit/org/apache/cassandra/repair/RepairSessionTest.java +++ b/test/unit/org/apache/cassandra/repair/RepairSessionTest.java @@ -68,7 +68,7 @@ public class RepairSessionTest new CommonRange(endpoints, Collections.emptySet(), Arrays.asList(repairRange)), "Keyspace1", RepairParallelism.SEQUENTIAL, false, false, - PreviewKind.NONE, false, "Standard1"); + PreviewKind.NONE, false, false, false, "Standard1"); // perform convict session.convict(remote, Double.MAX_VALUE); diff --git a/test/unit/org/apache/cassandra/repair/messages/RepairMessageSerializerTest.java b/test/unit/org/apache/cassandra/repair/messages/RepairMessageSerializerTest.java index 88dd5d6dc6..f2e0b5b0cb 100644 --- a/test/unit/org/apache/cassandra/repair/messages/RepairMessageSerializerTest.java +++ b/test/unit/org/apache/cassandra/repair/messages/RepairMessageSerializerTest.java @@ -20,7 +20,6 @@ package org.apache.cassandra.repair.messages; import java.io.IOException; -import com.google.common.collect.Sets; import org.junit.Assert; import org.junit.Test; diff --git a/test/unit/org/apache/cassandra/service/ActiveRepairServiceTest.java b/test/unit/org/apache/cassandra/service/ActiveRepairServiceTest.java index 3b8a57a52e..031ed57ee9 100644 --- a/test/unit/org/apache/cassandra/service/ActiveRepairServiceTest.java +++ b/test/unit/org/apache/cassandra/service/ActiveRepairServiceTest.java @@ -36,7 +36,6 @@ import java.util.concurrent.TimeUnit; import com.google.common.collect.ImmutableList; import com.google.common.collect.Sets; -import com.google.common.util.concurrent.Uninterruptibles; import org.apache.cassandra.utils.TimeUUID; import org.apache.cassandra.utils.concurrent.Condition; diff --git a/test/unit/org/apache/cassandra/service/PaxosStateTest.java b/test/unit/org/apache/cassandra/service/PaxosStateTest.java new file mode 100644 index 0000000000..f4fb8d027b --- /dev/null +++ b/test/unit/org/apache/cassandra/service/PaxosStateTest.java @@ -0,0 +1,177 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.service; + +import java.nio.ByteBuffer; + +import com.google.common.collect.Iterables; + +import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.service.paxos.Ballot; +import org.apache.cassandra.service.paxos.BallotGenerator; +import org.apache.cassandra.service.paxos.v1.PrepareVerbHandler; +import org.apache.cassandra.service.paxos.v1.ProposeVerbHandler; + +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.function.Supplier; + +import org.apache.cassandra.dht.Murmur3Partitioner; +import org.apache.cassandra.exceptions.ReadTimeoutException; +import org.apache.cassandra.service.paxos.PaxosOperationLock; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +import org.apache.cassandra.SchemaLoader; +import org.apache.cassandra.Util; +import org.apache.cassandra.db.*; +import org.apache.cassandra.db.rows.Row; +import org.apache.cassandra.db.partitions.PartitionUpdate; +import org.apache.cassandra.gms.Gossiper; +import org.apache.cassandra.service.paxos.Commit; +import org.apache.cassandra.service.paxos.PaxosState; +import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.utils.FBUtilities; + +import static org.apache.cassandra.service.paxos.Ballot.Flag.NONE; +import static org.apache.cassandra.service.paxos.BallotGenerator.Global.atUnixMicros; +import static org.apache.cassandra.utils.Clock.Global.nanoTime; +import static org.junit.Assert.*; + +public class PaxosStateTest +{ + @BeforeClass + public static void setUpClass() throws Throwable + { + SchemaLoader.loadSchema(); + SchemaLoader.schemaDefinition("PaxosStateTest"); + } + + @AfterClass + public static void stopGossiper() + { + Gossiper.instance.stop(); + } + + @Test + public void testCommittingAfterTruncation() throws Exception + { + ColumnFamilyStore cfs = Keyspace.open("PaxosStateTestKeyspace1").getColumnFamilyStore("Standard1"); + String key = "key" + nanoTime(); + ByteBuffer value = ByteBufferUtil.bytes(0); + RowUpdateBuilder builder = new RowUpdateBuilder(cfs.metadata(), FBUtilities.timestampMicros(), key); + builder.clustering("a").add("val", value); + PartitionUpdate update = Iterables.getOnlyElement(builder.build().getPartitionUpdates()); + + // CFS should be empty initially + assertNoDataPresent(cfs, Util.dk(key)); + + // Commit the proposal & verify the data is present + Commit beforeTruncate = newProposal(0, update); + PaxosState.commitDirect(beforeTruncate); + assertDataPresent(cfs, Util.dk(key), "val", value); + + // Truncate then attempt to commit again, mutation should + // be ignored as the proposal predates the truncation + cfs.truncateBlocking(); + PaxosState.commitDirect(beforeTruncate); + assertNoDataPresent(cfs, Util.dk(key)); + + // Now try again with a ballot created after the truncation + long timestamp = SystemKeyspace.getTruncatedAt(update.metadata().id) + 1; + Commit afterTruncate = newProposal(timestamp, update); + PaxosState.commitDirect(afterTruncate); + assertDataPresent(cfs, Util.dk(key), "val", value); + } + + private Commit newProposal(long ballotMicros, PartitionUpdate update) + { + return Commit.newProposal(atUnixMicros(ballotMicros, NONE), update); + } + + private void assertDataPresent(ColumnFamilyStore cfs, DecoratedKey key, String name, ByteBuffer value) + { + Row row = Util.getOnlyRowUnfiltered(Util.cmd(cfs, key).build()); + assertEquals(0, ByteBufferUtil.compareUnsigned(value, + row.getCell(cfs.metadata().getColumn(ByteBufferUtil.bytes(name))).buffer())); + } + + private void assertNoDataPresent(ColumnFamilyStore cfs, DecoratedKey key) + { + Util.assertEmpty(Util.cmd(cfs, key).build()); + } + + @Test + public void testPrepareProposePaxos() throws Throwable + { + ColumnFamilyStore cfs = Keyspace.open("PaxosStateTestKeyspace1").getColumnFamilyStore("Standard1"); + String key = "key" + nanoTime(); + ByteBuffer value = ByteBufferUtil.bytes(0); + RowUpdateBuilder builder = new RowUpdateBuilder(cfs.metadata(), FBUtilities.timestampMicros(), key); + builder.clustering("a").add("val", value); + PartitionUpdate update = Iterables.getOnlyElement(builder.build().getPartitionUpdates()); + + // CFS should be empty initially + assertNoDataPresent(cfs, Util.dk(key)); + + Ballot ballot = atUnixMicros(1000 * System.currentTimeMillis(), NONE); + + Commit commit = Commit.newPrepare(Util.dk(key), cfs.metadata(), ballot); + + assertTrue("paxos prepare stage failed", PrepareVerbHandler.doPrepare(commit).promised); + assertTrue("paxos propose stage failed", ProposeVerbHandler.doPropose(commit)); + } + + public void testPaxosLock() throws ExecutionException, InterruptedException, ExecutionException + { + DecoratedKey key = new BufferDecoratedKey(Murmur3Partitioner.MINIMUM, ByteBufferUtil.EMPTY_BYTE_BUFFER); + TableMetadata metadata = Keyspace.open("PaxosStateTestKeyspace1").getColumnFamilyStore("Standard1").metadata.get(); + Supplier locker = () -> PaxosState.lock(key, metadata, System.nanoTime() + TimeUnit.SECONDS.toNanos(1L), ConsistencyLevel.SERIAL, false); + ExecutorService executor = Executors.newFixedThreadPool(1); + Future future; + try (PaxosOperationLock lock = locker.get()) + { + try + { + try (PaxosOperationLock lock2 = locker.get()) + { + Assert.fail(); + } + } + catch (ReadTimeoutException rte) + { + } + + future = executor.submit(() -> { + try (PaxosOperationLock lock2 = locker.get()) + { + } + }); + } + finally + { + executor.shutdown(); + } + future.get(); + } +} diff --git a/test/unit/org/apache/cassandra/service/WriteResponseHandlerTest.java b/test/unit/org/apache/cassandra/service/WriteResponseHandlerTest.java index 5285eb073f..7a9bbf33c8 100644 --- a/test/unit/org/apache/cassandra/service/WriteResponseHandlerTest.java +++ b/test/unit/org/apache/cassandra/service/WriteResponseHandlerTest.java @@ -268,7 +268,7 @@ public class WriteResponseHandlerTest private static AbstractWriteResponseHandler createWriteResponseHandler(ConsistencyLevel cl, ConsistencyLevel ideal, long queryStartTime) { return ks.getReplicationStrategy().getWriteResponseHandler(ReplicaPlans.forWrite(ks, cl, targets, pending, Predicates.alwaysTrue(), ReplicaPlans.writeAll), - null, WriteType.SIMPLE, queryStartTime, ideal); + null, WriteType.SIMPLE, null, queryStartTime, ideal); } private static Message createDummyMessage(int target) diff --git a/test/unit/org/apache/cassandra/service/WriteResponseHandlerTransientTest.java b/test/unit/org/apache/cassandra/service/WriteResponseHandlerTransientTest.java index 2d3a23635d..f912c722be 100644 --- a/test/unit/org/apache/cassandra/service/WriteResponseHandlerTransientTest.java +++ b/test/unit/org/apache/cassandra/service/WriteResponseHandlerTransientTest.java @@ -150,27 +150,27 @@ public class WriteResponseHandlerTransientTest EndpointsForToken natural = EndpointsForToken.of(dummy.getToken(), full(EP1), full(EP2), trans(EP3), full(EP5)); EndpointsForToken pending = EndpointsForToken.of(dummy.getToken(), full(EP4), trans(EP6)); ReplicaLayout.ForTokenWrite layout = new ReplicaLayout.ForTokenWrite(ks.getReplicationStrategy(), natural, pending); - ReplicaPlan.ForTokenWrite replicaPlan = ReplicaPlans.forWrite(ks, ConsistencyLevel.QUORUM, layout, layout, ReplicaPlans.writeAll); + ReplicaPlan.ForWrite replicaPlan = ReplicaPlans.forWrite(ks, ConsistencyLevel.QUORUM, layout, layout, ReplicaPlans.writeAll); Assert.assertTrue(Iterables.elementsEqual(EndpointsForRange.of(full(EP4), trans(EP6)), replicaPlan.pending())); } - private static ReplicaPlan.ForTokenWrite expected(EndpointsForToken natural, EndpointsForToken selected) + private static ReplicaPlan.ForWrite expected(EndpointsForToken natural, EndpointsForToken selected) { - return new ReplicaPlan.ForTokenWrite(ks, ks.getReplicationStrategy(), ConsistencyLevel.QUORUM, EndpointsForToken.empty(dummy.getToken()), natural, natural, selected); + return new ReplicaPlan.ForWrite(ks, ks.getReplicationStrategy(), ConsistencyLevel.QUORUM, EndpointsForToken.empty(dummy.getToken()), natural, natural, selected); } - private static ReplicaPlan.ForTokenWrite getSpeculationContext(EndpointsForToken natural, Predicate livePredicate) + private static ReplicaPlan.ForWrite getSpeculationContext(EndpointsForToken natural, Predicate livePredicate) { ReplicaLayout.ForTokenWrite liveAndDown = new ReplicaLayout.ForTokenWrite(ks.getReplicationStrategy(), natural, EndpointsForToken.empty(dummy.getToken())); ReplicaLayout.ForTokenWrite live = new ReplicaLayout.ForTokenWrite(ks.getReplicationStrategy(), natural.filter(r -> livePredicate.test(r.endpoint())), EndpointsForToken.empty(dummy.getToken())); return ReplicaPlans.forWrite(ks, ConsistencyLevel.QUORUM, liveAndDown, live, ReplicaPlans.writeNormal); } - private static void assertSpeculationReplicas(ReplicaPlan.ForTokenWrite expected, EndpointsForToken replicas, Predicate livePredicate) + private static void assertSpeculationReplicas(ReplicaPlan.ForWrite expected, EndpointsForToken replicas, Predicate livePredicate) { - ReplicaPlan.ForTokenWrite actual = getSpeculationContext(replicas, livePredicate); + ReplicaPlan.ForWrite actual = getSpeculationContext(replicas, livePredicate); assertEquals(expected.pending(), actual.pending()); assertEquals(expected.live(), actual.live()); assertEquals(expected.contacts(), actual.contacts()); diff --git a/test/unit/org/apache/cassandra/service/paxos/AbstractPaxosRepairTest.java b/test/unit/org/apache/cassandra/service/paxos/AbstractPaxosRepairTest.java new file mode 100644 index 0000000000..9721879d56 --- /dev/null +++ b/test/unit/org/apache/cassandra/service/paxos/AbstractPaxosRepairTest.java @@ -0,0 +1,174 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.paxos; + +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; + +import com.google.common.collect.Iterables; +import org.junit.Assert; +import org.junit.Test; + +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.dht.Murmur3Partitioner; +import org.apache.cassandra.service.paxos.AbstractPaxosRepair.State; +import org.apache.cassandra.utils.ByteBufferUtil; + +import static org.apache.cassandra.service.paxos.AbstractPaxosRepair.DONE; + +/** + * test the state change logic of AbstractPaxosRepair + */ +public class AbstractPaxosRepairTest +{ + private static DecoratedKey dk(int k) + { + return Murmur3Partitioner.instance.decorateKey(ByteBufferUtil.bytes(k)); + } + + private static final DecoratedKey DK1 = dk(1); + + private static State STARTED = new State(); + + private static class PaxosTestRepair extends AbstractPaxosRepair + { + public PaxosTestRepair() + { + super(Murmur3Partitioner.instance.decorateKey(ByteBufferUtil.bytes(1)), null); + } + + public State restart(State state, long waitUntil) + { + return STARTED; + } + + public void setState(State update) + { + updateState(state(), null, (i1, i2) -> update); + } + } + + private static class Event + { + final AbstractPaxosRepair repair; + final AbstractPaxosRepair.Result result; + + public Event(AbstractPaxosRepair repair, AbstractPaxosRepair.Result result) + { + this.repair = repair; + this.result = result; + } + + public String toString() + { + return "Event{" + + "repair=" + repair + + ", result=" + result + + '}'; + } + + public boolean equals(Object o) + { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Event event = (Event) o; + return Objects.equals(repair, event.repair) && Objects.equals(result, event.result); + } + + public int hashCode() + { + return Objects.hash(repair, result); + } + } + + private static class Listener implements AbstractPaxosRepair.Listener + { + + final List events = new ArrayList<>(); + + public void onComplete(AbstractPaxosRepair repair, AbstractPaxosRepair.Result result) + { + events.add(new Event(repair, result)); + } + + public Event getOnlyEvent() + { + return Iterables.getOnlyElement(events); + } + } + + @Test + public void stateUpdate() + { + // listeners shoulnd't be called on state updates + PaxosTestRepair repair = new PaxosTestRepair(); + Listener listener = new Listener(); + repair.addListener(listener); + Assert.assertNull(repair.state()); + repair.start(); + Assert.assertSame(STARTED, repair.state()); + Assert.assertTrue(listener.events.isEmpty()); + } + + @Test + public void resultUpdate() + { + // listeners should be called on state updates + PaxosTestRepair repair = new PaxosTestRepair(); + Listener listener = new Listener(); + repair.addListener(listener); + repair.start(); + Assert.assertTrue(listener.events.isEmpty()); + + repair.setState(DONE); + Assert.assertEquals(new Event(repair, DONE), listener.getOnlyEvent()); + } + + @Test + public void stateUpdateException() + { + // state should be set to failure and listeners called on exception + Throwable e = new Throwable(); + + PaxosTestRepair repair = new PaxosTestRepair(); + Listener listener = new Listener(); + repair.addListener(listener); + repair.start(); + + repair.updateState(repair.state(), null, (i1, i2) -> {throw e;}); + Assert.assertEquals(new Event(repair, new AbstractPaxosRepair.Failure(e)), listener.getOnlyEvent()); + + } + + @Test + public void postResultListenerAttachment() + { + // listener should be called immediately if the repair is already complete + PaxosTestRepair repair = new PaxosTestRepair(); + repair.start(); + + repair.setState(DONE); + + Listener listener = new Listener(); + Assert.assertTrue(listener.events.isEmpty()); + repair.addListener(listener); + Assert.assertEquals(new Event(repair, DONE), listener.getOnlyEvent()); + } +} diff --git a/test/unit/org/apache/cassandra/service/paxos/ContentionStrategyTest.java b/test/unit/org/apache/cassandra/service/paxos/ContentionStrategyTest.java new file mode 100644 index 0000000000..9bb0cb4aa5 --- /dev/null +++ b/test/unit/org/apache/cassandra/service/paxos/ContentionStrategyTest.java @@ -0,0 +1,430 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.service.paxos; + +import java.util.List; +import java.util.Random; +import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.BiFunction; +import java.util.function.Consumer; +import java.util.function.DoubleSupplier; +import java.util.function.LongBinaryOperator; + +import com.google.common.collect.ImmutableList; +import org.junit.Assert; +import org.junit.Test; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import net.nicoulaj.compilecommand.annotations.Inline; +import org.apache.cassandra.config.DatabaseDescriptor; + +import static org.apache.cassandra.service.paxos.ContentionStrategy.*; +import static org.apache.cassandra.service.paxos.ContentionStrategy.WaitRandomizerFactory.*; +import static org.apache.cassandra.service.paxos.ContentionStrategyTest.WaitRandomizerType.*; + +public class ContentionStrategyTest +{ + private static final Logger logger = LoggerFactory.getLogger(ContentionStrategyTest.class); + + static + { + DatabaseDescriptor.daemonInitialization(); + } + + private static final long MAX = maxQueryTimeoutMicros()/2; + + private static final WaitParseValidator DEFAULT_WAIT_RANDOMIZER_VALIDATOR = new WaitParseValidator(defaultWaitRandomizer(), QEXP, 1.5); + private static final BoundParseValidator DEFAULT_MIN_VALIDATOR = new BoundParseValidator(defaultMinWait(), true, assertBound(0, MAX, 0, selectors.maxReadWrite(0f).getClass(), 0.50, 0, modifiers.multiply(0f).getClass(), 0.66)); + private static final BoundParseValidator DEFAULT_MAX_VALIDATOR = new BoundParseValidator(defaultMaxWait(), false, assertBound(10000, 100000, 100000, selectors.maxReadWrite(0f).getClass(), 0.95, 0, modifiers.multiplyByAttemptsExp(0f).getClass(), 1.8)); + private static final BoundParseValidator DEFAULT_MIN_DELTA_VALIDATOR = new BoundParseValidator(defaultMinDelta(), true, assertBound(5000, MAX, 5000, selectors.maxReadWrite(0f).getClass(), 0.50, 0, modifiers.multiply(0f).getClass(), 0.5)); + + private static List VALIDATE = ImmutableList.of( + new BoundParseValidator("p95(rw)", false, assertBound(0, MAX, MAX, selectors.maxReadWrite(0f).getClass(), 0.95, 0, modifiers.identity().getClass(), 1)), + new BoundParseValidator("5ms<=p50(rw)*0.66", false, assertBound(5000, MAX, MAX, selectors.maxReadWrite(0f).getClass(), 0.50, 0, modifiers.multiply(0).getClass(), 0.66)), + new BoundParseValidator("5us <= p50(r)*1.66*attempts", true, assertBound(5, MAX, 5, selectors.read(0f).getClass(), 0.50, 0, modifiers.multiplyByAttempts(0f).getClass(), 1.66)), + new BoundParseValidator("0<=p50(w)*0.66^attempts", true, assertBound(0, MAX, 0, selectors.write(0f).getClass(), 0.50, 0, modifiers.multiplyByAttemptsExp(0f).getClass(), 0.66)), + new BoundParseValidator("125us", true, assertBound(125, 125, 125, selectors.constant(0).getClass(), 0.0f, 125, modifiers.identity().getClass(), 1)), + new BoundParseValidator("5us <= p95(r)*1.8^attempts <= 100us", true, assertBound(5, 100, 5, selectors.read(0f).getClass(), 0.95, 0, modifiers.multiplyByAttemptsExp(0f).getClass(), 1.8)), + DEFAULT_MIN_VALIDATOR, DEFAULT_MAX_VALIDATOR, DEFAULT_MIN_DELTA_VALIDATOR + ); + + private static List VALIDATE_RANDOMIZER = ImmutableList.of( + new WaitParseValidator("quantizedexponential(0.5)", QEXP, 0.5), + new WaitParseValidator("exponential(2.5)", EXP, 2.5), + new WaitParseValidator("exp(10)", EXP, 10), + new WaitParseValidator("uniform", UNIFORM, 0), + DEFAULT_WAIT_RANDOMIZER_VALIDATOR + ); + + static class BoundParseValidator + { + final String spec; + final boolean isMin; + final Consumer validator; + + BoundParseValidator(String spec, boolean isMin, Consumer validator) + { + this.spec = spec; + this.isMin = isMin; + this.validator = validator; + } + + void validate(Bound bound) + { + validator.accept(bound); + } + } + + enum WaitRandomizerType + { + UNIFORM(Uniform.class, (p, f) -> f.uniform()), + EXP(Exponential.class, (p, f) -> f.exponential(p)), + QEXP(QuantizedExponential.class, (p, f) -> f.quantizedExponential(p)); + + final Class clazz; + final BiFunction getter; + + WaitRandomizerType(Class clazz, BiFunction getter) + { + this.clazz = clazz; + this.getter = getter; + } + } + + + static class WaitParseValidator + { + final String spec; + final WaitRandomizerType type; + final double power; + + WaitParseValidator(String spec, WaitRandomizerType type, double power) + { + this.spec = spec; + this.type = type; + this.power = power; + } + + void validate(WaitRandomizer randomizer) + { + Assert.assertSame(type.clazz, randomizer.getClass()); + if (AbstractExponential.class.isAssignableFrom(type.clazz)) + Assert.assertEquals(power, ((AbstractExponential) randomizer).power, 0.00001); + } + } + + private static class WaitRandomizerOutputValidator + { + static void validate(WaitRandomizerType type, long seed, int trials, int samplesPerTrial) + { + Random random = new Random(seed); + WaitRandomizer randomizer = type.getter.apply(2d, new WaitRandomizerFactory() + { + @Override public LongBinaryOperator uniformLongSupplier() { return (min, max) -> min + random.nextInt((int) (max - min)); } + @Override public DoubleSupplier uniformDoubleSupplier() { return random::nextDouble; } + }); + + for (int i = 0 ; i < trials ; ++i) + { + int min = random.nextInt(1 << 20); + int max = min + 1024 + random.nextInt(1 << 20); + double minMean = minMean(type, min, max); + double maxMean = maxMean(type, min, max); + double sampleMean = sampleMean(samplesPerTrial, min, max, randomizer); + Assert.assertTrue(minMean <= sampleMean); + Assert.assertTrue(maxMean >= sampleMean); + } + } + + private static double minMean(WaitRandomizerType type, int min, int max) + { + switch (type) + { + case UNIFORM: return min + (max - min) * (4d/10); + case EXP: case QEXP: return min + (max - min) * (6d/10); + default: throw new IllegalStateException(); + } + } + + private static double maxMean(WaitRandomizerType type, int min, int max) + { + switch (type) + { + case UNIFORM: return min + (max - min) * (6d/10); + case EXP: case QEXP: return min + (max - min) * (8d/10); + default: throw new IllegalStateException(); + } + } + + private static double sampleMean(int samples, int min, int max, WaitRandomizer randomizer) + { + double sum = 0; + int attempts = 1; + for (int i = 0 ; i < samples ; ++i) + { + long wait = randomizer.wait(min, max, attempts = (attempts & 15) + 1); + Assert.assertTrue(wait >= min); + Assert.assertTrue(wait <= max); + sum += wait; + } + double mean = sum / samples; + Assert.assertTrue(mean >= min); + Assert.assertTrue(mean <= max); + return mean; + } + } + + private static Consumer assertBound( + long min, long max, long onFailure, + Class selectorClass, + double selectorPercentile, + long selectorConst, + Class modifierClass, + double modifierVal + ) + { + return bound -> { + Assert.assertEquals(min, bound.min); + Assert.assertEquals(max, bound.max); + Assert.assertEquals(onFailure, bound.onFailure); + Assert.assertSame(selectorClass, bound.selector.getClass()); + if (selectorClass == selectors.constant(0).getClass()) + { + LatencySupplier fail = v -> { throw new UnsupportedOperationException(); }; + Assert.assertEquals(selectorConst, bound.selector.select(fail, fail)); + } + else + { + AtomicReference percentile = new AtomicReference<>(); + LatencySupplier set = v -> { percentile.set(v); return 0; }; + bound.selector.select(set, set); + Assert.assertNotNull(percentile.get()); + Assert.assertEquals(selectorPercentile, percentile.get(), 0.00001); + } + Assert.assertSame(modifierClass, bound.modifier.getClass()); + Assert.assertEquals(1000000L * modifierVal, bound.modifier.modify(1000000, 1), 0.00001); + }; + } + + private static void assertParseFailure(String spec) + { + + try + { + Bound bound = parseBound(spec, false); + Assert.fail("expected parse failure, but got " + bound); + } + catch (IllegalArgumentException e) + { + // expected + } + } + + @Test + public void strategyParseTest() + { + for (BoundParseValidator min : VALIDATE.stream().filter(v -> v.isMin).toArray(BoundParseValidator[]::new)) + { + for (BoundParseValidator max : VALIDATE.stream().filter(v -> !v.isMin).toArray(BoundParseValidator[]::new)) + { + for (BoundParseValidator minDelta : VALIDATE.stream().filter(v -> v.isMin).toArray(BoundParseValidator[]::new)) + { + for (WaitParseValidator random : VALIDATE_RANDOMIZER) + { + { + ParsedStrategy parsed = parseStrategy("min=" + min.spec + ",max=" + max.spec + ",delta=" + minDelta.spec + ",random=" + random.spec); + Assert.assertEquals(parsed.min, min.spec); + min.validate(parsed.strategy.min); + Assert.assertEquals(parsed.max, max.spec); + max.validate(parsed.strategy.max); + Assert.assertEquals(parsed.minDelta, minDelta.spec); + minDelta.validate(parsed.strategy.minDelta); + Assert.assertEquals(parsed.waitRandomizer, random.spec); + random.validate(parsed.strategy.waitRandomizer); + } + ParsedStrategy parsed = parseStrategy("random=" + random.spec); + Assert.assertEquals(parsed.min, DEFAULT_MIN_VALIDATOR.spec); + DEFAULT_MIN_VALIDATOR.validate(parsed.strategy.min); + Assert.assertEquals(parsed.max, DEFAULT_MAX_VALIDATOR.spec); + DEFAULT_MAX_VALIDATOR.validate(parsed.strategy.max); + Assert.assertEquals(parsed.minDelta, DEFAULT_MIN_DELTA_VALIDATOR.spec); + DEFAULT_MIN_DELTA_VALIDATOR.validate(parsed.strategy.minDelta); + Assert.assertEquals(parsed.waitRandomizer, random.spec); + random.validate(parsed.strategy.waitRandomizer); + } + ParsedStrategy parsed = parseStrategy("delta=" + minDelta.spec); + Assert.assertEquals(parsed.min, DEFAULT_MIN_VALIDATOR.spec); + DEFAULT_MIN_VALIDATOR.validate(parsed.strategy.min); + Assert.assertEquals(parsed.max, DEFAULT_MAX_VALIDATOR.spec); + DEFAULT_MAX_VALIDATOR.validate(parsed.strategy.max); + Assert.assertEquals(parsed.minDelta, minDelta.spec); + minDelta.validate(parsed.strategy.minDelta); + } + ParsedStrategy parsed = parseStrategy("max=" + max.spec); + Assert.assertEquals(parsed.min, DEFAULT_MIN_VALIDATOR.spec); + DEFAULT_MIN_VALIDATOR.validate(parsed.strategy.min); + Assert.assertEquals(parsed.max, max.spec); + max.validate(parsed.strategy.max); + Assert.assertEquals(parsed.minDelta, DEFAULT_MIN_DELTA_VALIDATOR.spec); + DEFAULT_MIN_DELTA_VALIDATOR.validate(parsed.strategy.minDelta); + } + ParsedStrategy parsed = parseStrategy("min=" + min.spec); + Assert.assertEquals(parsed.min, min.spec); + min.validate(parsed.strategy.min); + Assert.assertEquals(parsed.max, DEFAULT_MAX_VALIDATOR.spec); + DEFAULT_MAX_VALIDATOR.validate(parsed.strategy.max); + Assert.assertEquals(parsed.minDelta, DEFAULT_MIN_DELTA_VALIDATOR.spec); + DEFAULT_MIN_DELTA_VALIDATOR.validate(parsed.strategy.minDelta); + } + } + + @Test + public void testParseRoundTrip() + { + LatencySelectorFactory selectorFactory = new LatencySelectorFactory() + { + LatencySelectorFactory delegate = ContentionStrategy.selectors; + public LatencySelector constant(long latency) { return selector(delegate.constant(latency), String.format("%dms", latency)); } + public LatencySelector read(double percentile) { return selector(delegate.read(percentile), String.format("p%d(r)", (int) (percentile * 100))); } + public LatencySelector write(double percentile) { return selector(delegate.write(percentile), String.format("p%d(w)", (int) (percentile * 100))); } + public LatencySelector maxReadWrite(double percentile) { return selector(delegate.maxReadWrite(percentile), String.format("p%d(rw)", (int) percentile * 100)); } + + private LatencySelector selector(LatencySelector selector, String str) { + return new LatencySelector() + { + public long select(LatencySupplier read, LatencySupplier write) + { + return selector.select(read, write); + } + + public String toString() + { + return str; + } + }; + } + }; + + LatencyModifierFactory modifierFactory = new LatencyModifierFactory() + { + LatencyModifierFactory delegate = ContentionStrategy.modifiers; + public LatencyModifier identity() { return modifier(delegate.identity(), ""); } + public LatencyModifier multiply(double constant) { return modifier(delegate.multiply(constant), String.format(" * %.2f", constant)); } + public LatencyModifier multiplyByAttempts(double multiply) { return modifier(delegate.multiplyByAttempts(multiply), String.format(" * %.2f * attempts", multiply)); } + public LatencyModifier multiplyByAttemptsExp(double base) { return modifier(delegate.multiplyByAttemptsExp(base), String.format(" * %.2f ^ attempts", base)); } + + private LatencyModifier modifier(LatencyModifier modifier, String str) { + return new LatencyModifier() + { + @Inline + public long modify(long latency, int attempts) + { + return modifier.modify(latency, attempts); + } + + public String toString() + { + return str; + } + }; + } + }; + + LatencyModifier[] latencyModifiers = new LatencyModifier[]{ + modifierFactory.multiply(0.5), + modifierFactory.multiplyByAttempts(0.5), + modifierFactory.multiplyByAttemptsExp(0.5) + }; + + LatencySelector[] latencySelectors = new LatencySelector[]{ + selectorFactory.read(0.5), + selectorFactory.write(0.5), + selectorFactory.maxReadWrite(0.99) + }; + + for (boolean min : new boolean[] { true, false}) + { + String left = min ? "10ms <= " : ""; + for (boolean max : new boolean[] { true, false}) + { + String right = max ? " <= 10ms" : ""; + + for (LatencySelector selector : latencySelectors) + { + for (LatencyModifier modifier : latencyModifiers) + { + String mid = String.format("%s%s", selector, modifier); + String input = left + mid + right; + Bound bound = parseBound(input, false, selectorFactory, modifierFactory); + Assert.assertTrue(String.format("Bound: %d" , bound.min), !min || bound.min == 10000); + Assert.assertTrue(String.format("Bound: %d" , bound.max), !max || bound.max == 10000); + Assert.assertEquals(selector.toString(), bound.selector.toString()); + Assert.assertEquals(modifier.toString(), bound.modifier.toString()); + } + } + } + } + } + + @Test + public void boundParseTest() + { + VALIDATE.forEach(v -> v.validate(parseBound(v.spec, v.isMin))); + } + + @Test + public void waitRandomizerParseTest() + { + VALIDATE_RANDOMIZER.forEach(v -> v.validate(parseWaitRandomizer(v.spec))); + } + + @Test + public void waitRandomizerSampleTest() + { + waitRandomizerSampleTest(2); + } + + private void waitRandomizerSampleTest(int count) + { + while (count-- > 0) + { + long seed = ThreadLocalRandom.current().nextLong(); + logger.info("Seed {}", seed); + for (WaitRandomizerType type : WaitRandomizerType.values()) + { + WaitRandomizerOutputValidator.validate(type, seed, 100, 1000000); + } + } + } + + @Test + public void boundParseFailureTest() + { + assertParseFailure("10ms <= p95(r) <= 5ms"); + assertParseFailure("10 <= p95(r)"); + assertParseFailure("10 <= 20 <= 30"); + assertParseFailure("p95(r) < 5"); + assertParseFailure("p95(x)"); + assertParseFailure("p95()"); + assertParseFailure("p95"); + assertParseFailure("p50(rw)+0.66"); + } +} diff --git a/test/unit/org/apache/cassandra/service/paxos/PaxosProposeTest.java b/test/unit/org/apache/cassandra/service/paxos/PaxosProposeTest.java new file mode 100644 index 0000000000..78724900aa --- /dev/null +++ b/test/unit/org/apache/cassandra/service/paxos/PaxosProposeTest.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.paxos; + +import java.util.Arrays; +import java.util.concurrent.atomic.AtomicLongFieldUpdater; + +import org.junit.Test; + +import static org.apache.cassandra.service.paxos.PaxosPropose.*; + +public class PaxosProposeTest +{ + static class V + { + static final AtomicLongFieldUpdater updater = AtomicLongFieldUpdater.newUpdater(V.class, "v"); + volatile long v; + public boolean valid() + { + return v == 0; + } + } + @Test + public void testShouldSignal() + { + int[] signalledAtK = new int[12]; + V[] v = new V[] { new V(), new V(), new V(), new V(), new V(), new V(), new V(), new V(), new V(), new V(), new V(), new V() }; + boolean[] signalled = new boolean[12]; + for (int total = 2 ; total < 16 ; ++total) + { + for (int required = (total/2) + 1 ; required < total ; ++required) + { + for (int i = 0 ; i < total ; ++i) + { + for (int j = 0 ; j < total - i ; ++j) + { + Arrays.fill(signalled, false); + Arrays.fill(signalledAtK, Integer.MAX_VALUE); + for (int x = 0 ; x < v.length ; ++x) + v[x].v = 0; + + for (int k = 0 ; k <= total - (i + j) ; ++k) + { + signalled[0] = v[0].valid() && shouldSignal(responses(i, j, k), required, total, true, V.updater, v[0]); + signalled[1] = v[1].valid() && shouldSignal(responses(j, i, k), required, total, true, V.updater, v[1]); + signalled[2] = v[2].valid() && shouldSignal(responses(j, k, i), required, total, true, V.updater, v[2]); + signalled[3] = v[3].valid() && shouldSignal(responses(k, i, j), required, total, true, V.updater, v[3]); + signalled[4] = v[4].valid() && shouldSignal(responses(i, k, j), required, total, true, V.updater, v[4]); + signalled[5] = v[5].valid() && shouldSignal(responses(k, j, i), required, total, true, V.updater, v[5]); + signalled[6] = v[6].valid() && shouldSignal(responses(i, j, k), required, total, false, V.updater, v[6]); + signalled[7] = v[7].valid() && shouldSignal(responses(j, i, k), required, total, false, V.updater, v[7]); + signalled[8] = v[8].valid() && shouldSignal(responses(j, k, i), required, total, false, V.updater, v[8]); + signalled[9] = v[9].valid() && shouldSignal(responses(k, i, j), required, total, false, V.updater, v[9]); + signalled[10] = v[10].valid() && shouldSignal(responses(i, k, j), required, total, false, V.updater, v[10]); + signalled[11] = v[11].valid() && shouldSignal(responses(k, j, i), required, total, false, V.updater, v[11]); + for (int x = 0 ; x < 12 ; ++x) + { + if (signalled[x] && signalledAtK[x] < k) + throw new IllegalStateException(String.format("(%d,%d,%d): (%d,%d,%d,%d)", total, required, x, i, j, k, signalledAtK[x])); + else if (signalled[x]) + signalledAtK[x] = k; + } + } + + for (int x = 0 ; x < 12 ; ++x) + { + if (signalledAtK[x] == Integer.MAX_VALUE) + throw new IllegalStateException(String.format("(%d,%d,%d): (%d, %d)", total, required, x, i, j)); + } + } + } + } + } + } + + private static long responses(int i, int j, int k) + { + return i * ACCEPT_INCREMENT + j * REFUSAL_INCREMENT + k * FAILURE_INCREMENT; + } + +} diff --git a/test/unit/org/apache/cassandra/service/paxos/PaxosRepairHistoryTest.java b/test/unit/org/apache/cassandra/service/paxos/PaxosRepairHistoryTest.java new file mode 100644 index 0000000000..387ec653ce --- /dev/null +++ b/test/unit/org/apache/cassandra/service/paxos/PaxosRepairHistoryTest.java @@ -0,0 +1,531 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.paxos; + +import java.util.*; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import java.util.stream.LongStream; +import java.util.stream.Stream; + +import com.google.common.collect.Lists; + +import org.apache.cassandra.SchemaLoader; +import org.apache.cassandra.db.SystemKeyspace; +import org.apache.cassandra.dht.Murmur3Partitioner.LongToken; +import org.apache.cassandra.utils.FBUtilities; +import org.apache.cassandra.utils.Pair; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.dht.Murmur3Partitioner; +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.Token; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.apache.cassandra.dht.Range.deoverlap; +import static org.apache.cassandra.service.paxos.Ballot.Flag.NONE; +import static org.apache.cassandra.service.paxos.Ballot.none; +import static org.apache.cassandra.service.paxos.BallotGenerator.Global.atUnixMicros; +import static org.apache.cassandra.service.paxos.BallotGenerator.Global.nextBallot; +import static org.apache.cassandra.service.paxos.Commit.latest; +import static org.apache.cassandra.service.paxos.PaxosRepairHistory.trim; + +public class PaxosRepairHistoryTest +{ + static final Logger logger = LoggerFactory.getLogger(PaxosRepairHistoryTest.class); + private static final AtomicInteger tableNum = new AtomicInteger(); + static + { + System.setProperty("cassandra.partitioner", Murmur3Partitioner.class.getName()); + DatabaseDescriptor.daemonInitialization(); + assert DatabaseDescriptor.getPartitioner() instanceof Murmur3Partitioner; + } + + private static final Token MIN_TOKEN = Murmur3Partitioner.instance.getMinimumToken(); + + private static Token t(long t) + { + return new LongToken(t); + } + + private static Ballot b(int b) + { + return Ballot.atUnixMicrosWithLsb(b, 0, NONE); + } + + private static Range r(Token l, Token r) + { + return new Range<>(l, r); + } + + private static Range r(long l, long r) + { + return r(t(l), t(r)); + } + + private static Pair pt(long t, int b) + { + return Pair.create(t(t), b(b)); + } + + private static Pair pt(long t, Ballot b) + { + return Pair.create(t(t), b); + } + + private static Pair pt(Token t, int b) + { + return Pair.create(t, b(b)); + } + + private static PaxosRepairHistory h(Pair... points) + { + int length = points.length + (points[points.length - 1].left == null ? 0 : 1); + Token[] tokens = new Token[length - 1]; + Ballot[] ballots = new Ballot[length]; + for (int i = 0 ; i < length - 1 ; ++i) + { + tokens[i] = points[i].left; + ballots[i] = points[i].right; + } + ballots[length - 1] = length == points.length ? points[length - 1].right : none(); + return new PaxosRepairHistory(tokens, ballots); + } + + static + { + assert t(100).equals(t(100)); + assert b(111).equals(b(111)); + } + + private static class Builder + { + PaxosRepairHistory history = PaxosRepairHistory.EMPTY; + + Builder add(Ballot ballot, Range... ranges) + { + history = PaxosRepairHistory.add(history, Lists.newArrayList(ranges), ballot); + return this; + } + + Builder clear() + { + history = PaxosRepairHistory.EMPTY; + return this; + } + } + + static Builder builder() + { + return new Builder(); + } + + private static void checkSystemTableIO(PaxosRepairHistory history) + { + Assert.assertEquals(history, PaxosRepairHistory.fromTupleBufferList(history.toTupleBufferList())); + String tableName = "test" + tableNum.getAndIncrement(); + SystemKeyspace.savePaxosRepairHistory("test", tableName, history, false); + Assert.assertEquals(history, SystemKeyspace.loadPaxosRepairHistory("test", tableName)); + } + + @BeforeClass + public static void init() throws Exception + { + SchemaLoader.prepareServer(); + } + + @Test + public void testAdd() + { + Builder builder = builder(); + Assert.assertEquals(h(pt(10, none()), pt(20, 5), pt(30, none()), pt(40, 5)), + builder.add(b(5), r(10, 20), r(30, 40)).history); + + Assert.assertEquals(none(), builder.history.ballotForToken(t(0))); + Assert.assertEquals(none(), builder.history.ballotForToken(t(10))); + Assert.assertEquals(b(5), builder.history.ballotForToken(t(11))); + Assert.assertEquals(b(5), builder.history.ballotForToken(t(20))); + Assert.assertEquals(none(), builder.history.ballotForToken(t(21))); + + builder.clear(); + Assert.assertEquals(h(pt(10, none()), pt(20, 5), pt(30, none()), pt(40, 6)), + builder.add(b(5), r(10, 20)).add(b(6), r(30, 40)).history); + builder.clear(); + Assert.assertEquals(h(pt(10, none()), pt(20, 5), pt(30, 6), pt(40, 5)), + builder.add(b(5), r(10, 40)).add(b(6), r(20, 30)).history); + + builder.clear(); + Assert.assertEquals(h(pt(10, none()), pt(20, 6), pt(30, 5)), + builder.add(b(6), r(10, 20)).add(b(5), r(15, 30)).history); + + builder.clear(); + Assert.assertEquals(h(pt(10, none()), pt(20, 5), pt(30, 6)), + builder.add(b(5), r(10, 25)).add(b(6), r(20, 30)).history); + } + + @Test + public void testTrim() + { + Assert.assertEquals(h(pt(10, none()), pt(20, 5), pt(30, none()), pt(40, 5), pt(50, none()), pt(60, 5)), + trim(h(pt(0, none()), pt(70, 5)), Lists.newArrayList(r(10, 20), r(30, 40), r(50, 60)))); + + Assert.assertEquals(h(pt(10, none()), pt(20, 5)), + trim(h(pt(0, none()), pt(20, 5)), Lists.newArrayList(r(10, 30)))); + + Assert.assertEquals(h(pt(10, none()), pt(20, 5)), + trim(h(pt(10, none()), pt(30, 5)), Lists.newArrayList(r(0, 20)))); + } + + @Test + public void testFullRange() + { + // test full range is collapsed + Builder builder = builder(); + Assert.assertEquals(h(pt(null, 5)), + builder.add(b(5), r(MIN_TOKEN, MIN_TOKEN)).history); + + Assert.assertEquals(b(5), builder.history.ballotForToken(MIN_TOKEN)); + Assert.assertEquals(b(5), builder.history.ballotForToken(t(0))); + } + + @Test + public void testWrapAroundRange() + { + Builder builder = builder(); + Assert.assertEquals(h(pt(-100, 5), pt(100, none()), pt(null, 5)), + builder.add(b(5), r(100, -100)).history); + + Assert.assertEquals(b(5), builder.history.ballotForToken(MIN_TOKEN)); + Assert.assertEquals(b(5), builder.history.ballotForToken(t(-101))); + Assert.assertEquals(b(5), builder.history.ballotForToken(t(-100))); + Assert.assertEquals(none(), builder.history.ballotForToken(t(-99))); + Assert.assertEquals(none(), builder.history.ballotForToken(t(0))); + Assert.assertEquals(none(), builder.history.ballotForToken(t(100))); + Assert.assertEquals(b(5), builder.history.ballotForToken(t(101))); + } + + private static Token[] tks(long ... tks) + { + return LongStream.of(tks).mapToObj(LongToken::new).toArray(Token[]::new); + } + + private static Ballot[] uuids(String ... uuids) + { + return Stream.of(uuids).map(Ballot::fromString).toArray(Ballot[]::new); + } + + @Test + public void testRegression() + { + Assert.assertEquals(none(), trim( + new PaxosRepairHistory( + tks(-9223372036854775807L, -3952873730080618203L, -1317624576693539401L, 1317624576693539401L, 6588122883467697005L), + uuids("1382954c-1dd2-11b2-8fb2-f45d70d6d6d8", "138260a4-1dd2-11b2-abb2-c13c36b179e1", "1382951a-1dd2-11b2-1dd8-b7e242b38dbe", "138294fc-1dd2-11b2-83c4-43fb3a552386", "13829510-1dd2-11b2-f353-381f2ed963fa", "1382954c-1dd2-11b2-8fb2-f45d70d6d6d8")), + Collections.singleton(new Range<>(new LongToken(-1317624576693539401L), new LongToken(1317624576693539401L)))) + .ballotForToken(new LongToken(-4208619967696141037L))); + } + + @Test + public void testInequality() + { + Collection> ranges = Collections.singleton(new Range<>(Murmur3Partitioner.MINIMUM, Murmur3Partitioner.MINIMUM)); + PaxosRepairHistory a = PaxosRepairHistory.add(PaxosRepairHistory.EMPTY, ranges, none()); + PaxosRepairHistory b = PaxosRepairHistory.add(PaxosRepairHistory.EMPTY, ranges, nextBallot(NONE)); + Assert.assertNotEquals(a, b); + } + + @Test + public void testRandomTrims() + { + ExecutorService executor = Executors.newFixedThreadPool(FBUtilities.getAvailableProcessors()); + List> results = new ArrayList<>(); + int count = 1000; + for (int numberOfAdditions : new int[] { 1, 10, 100 }) + { + for (float maxCoveragePerRange : new float[] { 0.01f, 0.1f, 0.5f }) + { + for (float chanceOfMinToken : new float[] { 0.01f, 0.1f }) + { + results.addAll(testRandomTrims(executor, count, numberOfAdditions, 3, maxCoveragePerRange, chanceOfMinToken)); + } + } + } + FBUtilities.waitOnFutures(results); + executor.shutdown(); + } + + private List> testRandomTrims(ExecutorService executor, int tests, int numberOfAdditions, int maxNumberOfRangesPerAddition, float maxCoveragePerRange, float chanceOfMinToken) + { + return ThreadLocalRandom.current() + .longs(tests) + .mapToObj(seed -> executor.submit(() -> testRandomTrims(seed, numberOfAdditions, maxNumberOfRangesPerAddition, maxCoveragePerRange, chanceOfMinToken))) + .collect(Collectors.toList()); + } + + private void testRandomTrims(long seed, int numberOfAdditions, int maxNumberOfRangesPerAddition, float maxCoveragePerRange, float chanceOfMinToken) + { + Random random = new Random(seed); + logger.info("Seed {} ({}, {}, {}, {})", seed, numberOfAdditions, maxNumberOfRangesPerAddition, maxCoveragePerRange, chanceOfMinToken); + PaxosRepairHistory history = RandomPaxosRepairHistory.build(random, numberOfAdditions, maxNumberOfRangesPerAddition, maxCoveragePerRange, chanceOfMinToken); + // generate a random list of ranges that cover the whole ring + long[] tokens = random.longs(16).distinct().toArray(); + if (random.nextBoolean()) + tokens[0] = Long.MIN_VALUE; + Arrays.sort(tokens); + List>> ranges = IntStream.range(0, tokens.length <= 3 ? 1 : 1 + random.nextInt((tokens.length - 1) / 2)) + .mapToObj(ignore -> new ArrayList>()) + .collect(Collectors.toList()); + + for (int i = 1 ; i < tokens.length ; ++i) + ranges.get(random.nextInt(ranges.size())).add(new Range<>(new LongToken(tokens[i - 1]), new LongToken(tokens[i]))); + ranges.get(random.nextInt(ranges.size())).add(new Range<>(new LongToken(tokens[tokens.length - 1]), new LongToken(tokens[0]))); + + List splits = new ArrayList<>(); + for (List> rs : ranges) + { + PaxosRepairHistory trimmed = PaxosRepairHistory.trim(history, rs); + splits.add(trimmed); + if (rs.isEmpty()) + continue; + + Range prev = rs.get(rs.size() - 1); + for (Range range : rs) + { + if (prev.right.equals(range.left)) + { + Assert.assertEquals(history.ballotForToken(((LongToken)range.left).decreaseSlightly()), trimmed.ballotForToken(((LongToken)range.left).decreaseSlightly())); + Assert.assertEquals(history.ballotForToken(range.left), trimmed.ballotForToken(range.left)); + } + else + { + if (!range.left.isMinimum()) + Assert.assertEquals(none(), trimmed.ballotForToken(range.left)); + if (!prev.right.isMinimum()) + Assert.assertEquals(none(), trimmed.ballotForToken(prev.right.increaseSlightly())); + } + Assert.assertEquals(history.ballotForToken(range.left.increaseSlightly()), trimmed.ballotForToken(range.left.increaseSlightly())); + if (!range.left.increaseSlightly().equals(range.right)) + Assert.assertEquals(history.ballotForToken(((LongToken)range.right).decreaseSlightly()), trimmed.ballotForToken(((LongToken)range.right).decreaseSlightly())); + + if (range.right.isMinimum()) + Assert.assertEquals(history.ballotForToken(new LongToken(Long.MAX_VALUE)), trimmed.ballotForToken(new LongToken(Long.MAX_VALUE))); + else + Assert.assertEquals(history.ballotForToken(range.right), trimmed.ballotForToken(range.right)); + prev = range; + } + } + + PaxosRepairHistory merged = PaxosRepairHistory.EMPTY; + for (PaxosRepairHistory split : splits) + merged = PaxosRepairHistory.merge(merged, split); + + Assert.assertEquals(history, merged); + checkSystemTableIO(history); + } + + @Test + public void testRandomAdds() + { + ExecutorService executor = Executors.newFixedThreadPool(FBUtilities.getAvailableProcessors()); + List> results = new ArrayList<>(); + int count = 1000; + for (int numberOfAdditions : new int[] { 1, 10, 100 }) + { + for (float maxCoveragePerRange : new float[] { 0.01f, 0.1f, 0.5f }) + { + for (float chanceOfMinToken : new float[] { 0.01f, 0.1f }) + { + results.addAll(testRandomAdds(executor, count, 3, numberOfAdditions, 3, maxCoveragePerRange, chanceOfMinToken)); + } + } + } + FBUtilities.waitOnFutures(results); + executor.shutdown(); + } + + private List> testRandomAdds(ExecutorService executor, int tests, int numberOfMerges, int numberOfAdditions, int maxNumberOfRangesPerAddition, float maxCoveragePerRange, float chanceOfMinToken) + { + return ThreadLocalRandom.current() + .longs(tests) + .mapToObj(seed -> executor.submit(() -> testRandomAdds(seed, numberOfMerges, numberOfAdditions, maxNumberOfRangesPerAddition, maxCoveragePerRange, chanceOfMinToken))) + .collect(Collectors.toList()); + } + + private void testRandomAdds(long seed, int numberOfMerges, int numberOfAdditions, int maxNumberOfRangesPerAddition, float maxCoveragePerRange, float chanceOfMinToken) + { + Random random = new Random(seed); + String id = String.format("%d, %d, %d, %d, %f, %f", seed, numberOfMerges, numberOfAdditions, maxNumberOfRangesPerAddition, maxCoveragePerRange, chanceOfMinToken); + logger.info(id); + List merge = new ArrayList<>(); + while (numberOfMerges-- > 0) + { + RandomWithCanonical build = new RandomWithCanonical(); + build.addRandom(random, numberOfAdditions, maxNumberOfRangesPerAddition, maxCoveragePerRange, chanceOfMinToken); + merge.add(build); + } + + RandomWithCanonical check = new RandomWithCanonical(); + for (RandomWithCanonical add : merge) + check = check.merge(add); + check.serdeser(); + + for (Token token : check.canonical.keySet()) + { + LongToken tk = (LongToken) token; + Assert.assertEquals(id, check.ballotForToken(tk.decreaseSlightly()), check.test.ballotForToken(tk.decreaseSlightly())); + Assert.assertEquals(id, check.ballotForToken(tk), check.test.ballotForToken(token)); + Assert.assertEquals(id, check.ballotForToken(tk.increaseSlightly()), check.test.ballotForToken(token.increaseSlightly())); + } + + // check some random + { + int count = 1000; + while (count-- > 0) + { + LongToken token = new LongToken(random.nextLong()); + Assert.assertEquals(id, check.ballotForToken(token), check.test.ballotForToken(token)); + } + } + + } + + static class RandomPaxosRepairHistory + { + PaxosRepairHistory test = PaxosRepairHistory.EMPTY; + + void add(Collection> ranges, Ballot ballot) + { + test = PaxosRepairHistory.add(test, ranges, ballot); + } + + void merge(RandomPaxosRepairHistory other) + { + test = PaxosRepairHistory.merge(test, other.test); + } + + void addOneRandom(Random random, int maxRangeCount, float maxCoverage, float minChance) + { + int count = maxRangeCount == 1 ? 1 : 1 + random.nextInt(maxRangeCount - 1); + Ballot ballot = atUnixMicros(random.nextInt(Integer.MAX_VALUE), NONE); + List> ranges = new ArrayList<>(); + while (count-- > 0) + { + long length = (long) (2 * random.nextDouble() * maxCoverage * Long.MAX_VALUE); + if (length == 0) length = 1; + Range range; + if (random.nextFloat() <= minChance) + { + if (random.nextBoolean()) range = new Range<>(Murmur3Partitioner.MINIMUM, new LongToken(Long.MIN_VALUE + length)); + else range = new Range<>(new LongToken(Long.MAX_VALUE - length), Murmur3Partitioner.MINIMUM); + } + else + { + long start = random.nextLong(); + range = new Range<>(new LongToken(start), new LongToken(start + length)); + } + ranges.add(range); + } + ranges.sort(Range::compareTo); + add(deoverlap(ranges), ballot); + } + + void addRandom(Random random, int count, int maxNumberOfRangesPerAddition, float maxCoveragePerAddition, float minTokenChance) + { + while (count-- > 0) + addOneRandom(random, maxNumberOfRangesPerAddition, maxCoveragePerAddition, minTokenChance); + } + + static PaxosRepairHistory build(Random random, int count, int maxNumberOfRangesPerAddition, float maxCoveragePerRange, float chanceOfMinToken) + { + RandomPaxosRepairHistory result = new RandomPaxosRepairHistory(); + result.addRandom(random, count, maxNumberOfRangesPerAddition, maxCoveragePerRange, chanceOfMinToken); + return result.test; + } + } + + static class RandomWithCanonical extends RandomPaxosRepairHistory + { + NavigableMap canonical = new TreeMap<>(); + { + canonical.put(Murmur3Partitioner.MINIMUM, none()); + } + + Ballot ballotForToken(LongToken token) + { + return canonical + .floorEntry(token.token == Long.MIN_VALUE ? token : token.decreaseSlightly()) + .getValue(); + } + + RandomWithCanonical merge(RandomWithCanonical other) + { + RandomWithCanonical result = new RandomWithCanonical(); + result.test = PaxosRepairHistory.merge(test, other.test); + result.canonical = new TreeMap<>(); + result.canonical.putAll(canonical); + for (Map.Entry entry : other.canonical.entrySet()) + { + Token left = entry.getKey(); + Token right = other.canonical.higherKey(left); + if (right == null) right = Murmur3Partitioner.MINIMUM; + result.addCanonical(new Range<>(left, right), entry.getValue()); + } + return result; + } + + void serdeser() + { + PaxosRepairHistory tmp = PaxosRepairHistory.fromTupleBufferList(test.toTupleBufferList()); + Assert.assertEquals(test, tmp); + test = tmp; + } + + void add(Collection> addRanges, Ballot ballot) + { + super.add(addRanges, ballot); + for (Range range : addRanges) + addCanonical(range, ballot); + } + + void addCanonical(Range range, Ballot ballot) + { + canonical.put(range.left, canonical.floorEntry(range.left).getValue()); + if (!range.right.isMinimum()) + canonical.put(range.right, canonical.floorEntry(range.right).getValue()); + + for (Range r : range.unwrap()) + { + (r.right.isMinimum() + ? canonical.subMap(r.left, true, new LongToken(Long.MAX_VALUE), true) + : canonical.subMap(r.left, true, r.right, false) + ).entrySet().forEach(e -> e.setValue(latest(e.getValue(), ballot))); + } + } + } +} diff --git a/test/unit/org/apache/cassandra/service/paxos/PaxosRepairTest.java b/test/unit/org/apache/cassandra/service/paxos/PaxosRepairTest.java new file mode 100644 index 0000000000..e226cf32b3 --- /dev/null +++ b/test/unit/org/apache/cassandra/service/paxos/PaxosRepairTest.java @@ -0,0 +1,172 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.paxos; + +import java.net.UnknownHostException; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +import org.junit.Test; + +import org.apache.cassandra.locator.InetAddressAndPort; + +import org.apache.cassandra.utils.CassandraVersion; + +import static org.apache.cassandra.service.paxos.PaxosRepair.validateVersionCompatibility; +import static org.junit.Assert.*; + +import static org.apache.cassandra.service.paxos.PaxosRepairTest.Requirements.*; + +public class PaxosRepairTest +{ + private static final String DC0 = "DC0"; + private static final String DC1 = "DC1"; + private static final String DC2 = "DC2"; + + private static InetAddressAndPort intToInet(int i) + { + try + { + return InetAddressAndPort.getByAddress(new byte[]{127, 0, 0, (byte)i}); + } + catch (UnknownHostException e) + { + throw new AssertionError(e); + } + } + + // should make reading the tests slightly easier + enum Requirements + { + NORMAL(false, false), + STRICT_QUORUM(true, false), + NO_DC_CHECKS(false, true), + STRICT_QUORUM_AND_NO_DC_CHECKS(true, true); + + final boolean strict_quorum; + final boolean no_dc_check; + + Requirements(boolean strict_quorum, boolean no_dc_check) + { + this.strict_quorum = strict_quorum; + this.no_dc_check = no_dc_check; + } + } + + private static class Topology + { + Set dead = new HashSet<>(); + Map endpointToDc = new HashMap<>(); + + + Topology alive(String dc, int... epNums) + { + for (int e : epNums) + { + InetAddressAndPort ep = intToInet(e); + endpointToDc.put(ep, dc); + dead.remove(ep); + } + return this; + } + + Topology dead(int... eps) + { + for (int i : eps) + { + InetAddressAndPort ep = intToInet(i); + assert endpointToDc.containsKey(ep); + dead.add(ep); + } + return this; + } + + String getDc(InetAddressAndPort ep) + { + assert endpointToDc.containsKey(ep); + return endpointToDc.get(ep); + } + + Set all() + { + return new HashSet<>(endpointToDc.keySet()); + } + + Set live() + { + Set eps = all(); + eps.removeAll(dead); + return eps; + } + + boolean hasSufficientLiveNodes(Requirements requirements) + { + return PaxosRepair.hasSufficientLiveNodesForTopologyChange(all(), live(), this::getDc, requirements.no_dc_check, requirements.strict_quorum); + } + } + + static Topology topology(String dc, int... epNums) + { + Topology t = new Topology(); + t.alive(dc, epNums); + return t; + } + + @Test + public void singleDcHasSufficientLiveNodesTest() + { + assertTrue(topology(DC0, 1).hasSufficientLiveNodes(NORMAL)); + assertTrue(topology(DC0, 1, 2).hasSufficientLiveNodes(NORMAL)); + assertTrue(topology(DC0, 1, 2).dead(1).hasSufficientLiveNodes(NORMAL)); + assertFalse(topology(DC0, 1, 2).dead(1, 2).hasSufficientLiveNodes(NORMAL)); + assertTrue(topology(DC0, 1, 2, 3).hasSufficientLiveNodes(NORMAL)); + assertTrue(topology(DC0, 1, 2, 3).dead(1).hasSufficientLiveNodes(NORMAL)); + assertFalse(topology(DC0, 1, 2, 3).dead(1, 2).hasSufficientLiveNodes(NORMAL)); + } + + @Test + public void multiDcHasSufficientLiveNodesTest() + { + assertTrue(topology(DC0, 1, 2, 3).alive(DC1, 4, 5, 6).hasSufficientLiveNodes(NORMAL)); + assertTrue(topology(DC0, 1, 2, 3).alive(DC1, 4, 5, 6).dead(1, 4).hasSufficientLiveNodes(NORMAL)); + assertFalse(topology(DC0, 1, 2, 3).alive(DC1, 4, 5, 6).dead(4, 5).hasSufficientLiveNodes(NORMAL)); + assertTrue(topology(DC0, 1, 2, 3).alive(DC1, 4, 5, 6).dead(4, 5).hasSufficientLiveNodes(NO_DC_CHECKS)); + assertFalse(topology(DC0, 1, 2, 3).alive(DC1, 4, 5, 6).dead(1, 4, 5).hasSufficientLiveNodes(NORMAL)); + assertTrue(topology(DC0, 1, 2).alive(DC1, 3, 4).alive(DC2, 5, 6).hasSufficientLiveNodes(NORMAL)); + assertTrue(topology(DC0, 1, 2).alive(DC1, 3, 4).alive(DC2, 5, 6).dead(1).hasSufficientLiveNodes(NORMAL)); + assertFalse(topology(DC0, 1, 2).alive(DC1, 3, 4).alive(DC2, 5, 6).dead(1).hasSufficientLiveNodes(STRICT_QUORUM)); + assertFalse(topology(DC0, 1, 2).alive(DC1, 3, 4).alive(DC2, 5, 6).dead(5, 6).hasSufficientLiveNodes(NORMAL)); + assertTrue(topology(DC0, 1, 2).alive(DC1, 3, 4).alive(DC2, 5, 6).dead(5, 6).hasSufficientLiveNodes(STRICT_QUORUM_AND_NO_DC_CHECKS)); + } + + private static CassandraVersion version(String v) + { + return new CassandraVersion(v); + } + + @Test + public void versionValidationTest() + { + assertTrue(validateVersionCompatibility(version("4.1.0"))); + assertTrue(validateVersionCompatibility(version("4.1.0-SNAPSHOT"))); + assertFalse(validateVersionCompatibility(null)); + } +} diff --git a/test/unit/org/apache/cassandra/service/paxos/PaxosStateTest.java b/test/unit/org/apache/cassandra/service/paxos/PaxosStateTest.java new file mode 100644 index 0000000000..fd6afba99f --- /dev/null +++ b/test/unit/org/apache/cassandra/service/paxos/PaxosStateTest.java @@ -0,0 +1,323 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.service.paxos; + +import java.nio.ByteBuffer; +import java.util.function.Function; + +import com.google.common.collect.Iterables; + +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.schema.TableMetadata; + +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +import org.apache.cassandra.SchemaLoader; +import org.apache.cassandra.Util; +import org.apache.cassandra.config.Config; +import org.apache.cassandra.db.*; +import org.apache.cassandra.db.rows.Row; +import org.apache.cassandra.db.partitions.PartitionUpdate; +import org.apache.cassandra.gms.Gossiper; +import org.apache.cassandra.service.paxos.PaxosState.Snapshot; +import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.utils.FBUtilities; + +import static org.apache.cassandra.config.Config.PaxosStatePurging.gc_grace; +import static org.apache.cassandra.config.Config.PaxosStatePurging.legacy; +import static org.apache.cassandra.config.Config.PaxosStatePurging.repaired; +import static org.apache.cassandra.service.paxos.Ballot.Flag.NONE; +import static org.apache.cassandra.service.paxos.BallotGenerator.Global.atUnixMicros; +import static org.apache.cassandra.service.paxos.Commit.*; +import static org.junit.Assert.*; + +public class PaxosStateTest +{ + static TableMetadata metadata; + @BeforeClass + public static void setUpClass() throws Throwable + { + SchemaLoader.loadSchema(); + SchemaLoader.schemaDefinition("PaxosStateTest"); + metadata = Keyspace.open("PaxosStateTestKeyspace1").getColumnFamilyStore("Standard1").metadata.get(); + metadata.withSwapped(metadata.params.unbuild().gcGraceSeconds(3600).build()); + } + + @AfterClass + public static void stopGossiper() + { + Gossiper.instance.stop(); + } + + @Test + public void testCommittingAfterTruncation() throws Exception + { + ColumnFamilyStore cfs = Keyspace.open("PaxosStateTestKeyspace1").getColumnFamilyStore("Standard1"); + String key = "key" + System.nanoTime(); + ByteBuffer value = ByteBufferUtil.bytes(0); + RowUpdateBuilder builder = new RowUpdateBuilder(metadata, FBUtilities.timestampMicros(), key); + builder.clustering("a").add("val", value); + PartitionUpdate update = Iterables.getOnlyElement(builder.build().getPartitionUpdates()); + + // CFS should be empty initially + assertNoDataPresent(cfs, Util.dk(key)); + + // Commit the proposal & verify the data is present + Commit beforeTruncate = newProposal(0, update); + PaxosState.commitDirect(beforeTruncate); + assertDataPresent(cfs, Util.dk(key), "val", value); + + // Truncate then attempt to commit again, mutation should + // be ignored as the proposal predates the truncation + cfs.truncateBlocking(); + PaxosState.commitDirect(beforeTruncate); + assertNoDataPresent(cfs, Util.dk(key)); + + // Now try again with a ballot created after the truncation + long timestamp = SystemKeyspace.getTruncatedAt(update.metadata().id) + 1; + Commit afterTruncate = newProposal(timestamp, update); + PaxosState.commitDirect(afterTruncate); + assertDataPresent(cfs, Util.dk(key), "val", value); + } + + @Test + public void testReadExpired() + { + DatabaseDescriptor.setPaxosStatePurging(gc_grace); + + String key = "key" + System.nanoTime(); + Accepted accepted = newProposal(1, key).accepted(); + PaxosState.legacyPropose(accepted); + + DatabaseDescriptor.setPaxosStatePurging(repaired); + // not expired if read in the past + assertPaxosState(key, accepted, state -> state.current(accepted.ballot).accepted); + // not expired if read with paxos state purging enabled + assertPaxosState(key, accepted, state -> state.current(Integer.MAX_VALUE).accepted); + DatabaseDescriptor.setPaxosStatePurging(gc_grace); + // not expired if read in the past + assertPaxosState(key, accepted, state -> state.current(accepted.ballot).accepted); + // expired if read with paxos state purging disabled + assertPaxosState(key, null, state -> state.current(Integer.MAX_VALUE).accepted); + // clear cache to read from disk + PaxosState.RECENT.clear(); + + Committed committed = accepted.committed(); + Committed empty = emptyProposal(key).accepted().committed(); + PaxosState.commitDirect(committed); + DatabaseDescriptor.setPaxosStatePurging(repaired); + // not expired if read in the past + assertPaxosState(key, committed, state -> state.current(committed.ballot).committed); + // not expired if read with paxos state purging enabled + assertPaxosState(key, committed, state -> state.current(Integer.MAX_VALUE).committed); + DatabaseDescriptor.setPaxosStatePurging(gc_grace); + // not expired if read in the past + assertPaxosState(key, committed, state -> state.current(committed.ballot).committed); + // expired if read with paxos state purging disabled + assertPaxosState(key, empty, state -> state.current(Integer.MAX_VALUE).committed); + DatabaseDescriptor.setPaxosStatePurging(repaired); + } + + @Test + public void testReadTTLd() + { + String key = "key" + System.nanoTime(); + String key2 = key + 'A'; + Accepted accepted = new AcceptedWithTTL(newProposal(1, key), 1); + PaxosState.legacyPropose(accepted); + PaxosState.legacyPropose(new AcceptedWithTTL(newProposal(1, key2), 10000)); + + DatabaseDescriptor.setPaxosStatePurging(repaired); + // not expired if read in the past + assertPaxosState(key, accepted, state -> state.current(0).accepted); + // TTL, so still expired if read with paxos state purging enabled + assertPaxosState(key, null, state -> state.current(Integer.MAX_VALUE).accepted); + DatabaseDescriptor.setPaxosStatePurging(gc_grace); + // not expired if read in the past + assertPaxosState(key, accepted, state -> state.current(0).accepted); + // expired if read with paxos state purging disabled + assertPaxosState(key, null, state -> state.current(Integer.MAX_VALUE).accepted); + DatabaseDescriptor.setPaxosStatePurging(legacy); + // not expired if read in the past + assertPaxosState(key, accepted, state -> state.current(0).accepted); + // expired if read with paxos state purging disabled + assertPaxosState(key, null, state -> state.current(Integer.MAX_VALUE).accepted); + // clear cache to read from disk + PaxosState.RECENT.clear(); + + Committed committed = new CommittedWithTTL(accepted, 1); + Committed empty = emptyProposal(key).accepted().committed(); + PaxosState.commitDirect(committed); + + DatabaseDescriptor.setPaxosStatePurging(repaired); + // not expired if read in the past + assertPaxosState(key, committed, state -> state.current(0).committed); + // not expired if read with paxos state purging enabled + assertPaxosState(key, empty, state -> state.current(Integer.MAX_VALUE).committed); + DatabaseDescriptor.setPaxosStatePurging(gc_grace); + // not expired if read in the past + assertPaxosState(key, committed, state -> state.current(0).committed); + // expired if read with paxos state purging disabled + assertPaxosState(key, empty, state -> state.current(Integer.MAX_VALUE).committed); + DatabaseDescriptor.setPaxosStatePurging(legacy); + // not expired if read in the past + assertPaxosState(key, committed, state -> state.current(0).committed); + // expired if read with paxos state purging disabled + assertPaxosState(key, empty, state -> state.current(Integer.MAX_VALUE).committed); + } + + @Test + public void testWriteTTLd() + { + String key = "key" + System.nanoTime(); + Accepted accepted = newProposal(1, key).accepted(); + + DatabaseDescriptor.setPaxosStatePurging(legacy); // write with TTLs + PaxosState.legacyPropose(accepted); + accepted = new AcceptedWithTTL(accepted, -1); // for equality test + + DatabaseDescriptor.setPaxosStatePurging(repaired); + // not expired if read in the past (or now) + assertPaxosState(key, accepted, state -> state.current(0).accepted); + // TTL, so still expired if read with paxos state purging enabled + assertPaxosState(key, null, state -> state.current(Integer.MAX_VALUE).accepted); + + DatabaseDescriptor.setPaxosStatePurging(gc_grace); + // not expired if read in the past (or now) + assertPaxosState(key, accepted, state -> state.current(0).accepted); + // expired if read with paxos state purging disabled + assertPaxosState(key, null, state -> state.current(Integer.MAX_VALUE).accepted); + + DatabaseDescriptor.setPaxosStatePurging(legacy); + // not expired if read in the past (or now) + assertPaxosState(key, accepted, state -> state.current(0).accepted); + // TTL, so expired in the future + assertPaxosState(key, null, state -> state.current(Integer.MAX_VALUE).accepted); + + PaxosState.RECENT.clear(); + + Committed committed = new Committed(accepted); + Committed empty = emptyProposal(key).accepted().committed(); + DatabaseDescriptor.setPaxosStatePurging(legacy); // write with TTLs + committed = new CommittedWithTTL(committed, -1); // for equality test + PaxosState.commitDirect(committed); + + DatabaseDescriptor.setPaxosStatePurging(repaired); + // not expired if read in the past + assertPaxosState(key, committed, state -> state.current(0).committed); + // not expired if read with paxos state purging enabled + assertPaxosState(key, empty, state -> state.current(Integer.MAX_VALUE).committed); + DatabaseDescriptor.setPaxosStatePurging(gc_grace); + // not expired if read in the past + assertPaxosState(key, committed, state -> state.current(0).committed); + // expired if read with paxos state purging disabled + assertPaxosState(key, empty, state -> state.current(Integer.MAX_VALUE).committed); + DatabaseDescriptor.setPaxosStatePurging(legacy); + // not expired if read in the past + assertPaxosState(key, committed, state -> state.current(0).committed); + // expired if read with paxos state purging disabled + assertPaxosState(key, empty, state -> state.current(Integer.MAX_VALUE).committed); + } + + private static void assertPaxosState(String key, Commit expect, Function test) + { + // TODO: test from cache after write + PaxosState.RECENT.clear(); + // test from disk + try (PaxosState state = PaxosState.get(Util.dk(key), metadata)) + { + Assert.assertEquals(expect, test.apply(state)); + } + // test from cache + try (PaxosState state = PaxosState.get(Util.dk(key), metadata)) + { + Assert.assertEquals(expect, test.apply(state)); + } + } + + private Commit newProposal(long ballotMillis, PartitionUpdate update) + { + return Commit.newProposal(atUnixMicros(ballotMillis, NONE), update); + } + + private Proposal emptyProposal(String k) + { + return Proposal.empty(Ballot.none(), Util.dk(k), Keyspace.open("PaxosStateTestKeyspace1").getColumnFamilyStore("Standard1").metadata.get()); + } + + private Proposal newProposal(long ballotMicros, String k) + { + return Proposal.empty(atUnixMicros(ballotMicros, NONE), Util.dk(k), Keyspace.open("PaxosStateTestKeyspace1").getColumnFamilyStore("Standard1").metadata.get()); + } + + private void assertDataPresent(ColumnFamilyStore cfs, DecoratedKey key, String name, ByteBuffer value) + { + Row row = Util.getOnlyRowUnfiltered(Util.cmd(cfs, key).build()); + assertEquals(0, ByteBufferUtil.compareUnsigned(value, + row.getCell(cfs.metadata.get().getColumn(ByteBufferUtil.bytes(name))).buffer())); + } + + private void assertNoDataPresent(ColumnFamilyStore cfs, DecoratedKey key) + { + Util.assertEmpty(Util.cmd(cfs, key).build()); + } + + private static void assertAcceptedMerge(Accepted expected, Accepted left, Accepted right) + { + Committed empty = Committed.none(expected.update.partitionKey(), expected.update.metadata()); + Snapshot snapshotLeft = new Snapshot(null, null, left, empty); + Snapshot snapshotRight = new Snapshot(null, null, right, empty); + Accepted merged = Snapshot.merge(snapshotLeft, snapshotRight).accepted; + Assert.assertSame(expected, merged); + } + + @Test + public void testAcceptMerging() + { + Accepted accepted = newProposal(1, "1").accepted(); + AcceptedWithTTL acceptedWithTTL1 = new AcceptedWithTTL(accepted, 100); + AcceptedWithTTL acceptedWithTTL2 = new AcceptedWithTTL(accepted, 200); + + assertAcceptedMerge(accepted, accepted, acceptedWithTTL1); + assertAcceptedMerge(accepted, acceptedWithTTL1, accepted); + assertAcceptedMerge(acceptedWithTTL2, acceptedWithTTL1, acceptedWithTTL2); + assertAcceptedMerge(acceptedWithTTL2, acceptedWithTTL2, acceptedWithTTL1); + } + + private static void assertCommittedMerge(Committed expected, Committed left, Committed right) + { + Committed merged = Snapshot.merge(new Snapshot(null, null, null, left), new Snapshot(null, null, null, right)).committed; + Assert.assertSame(expected, merged); + } + + @Test + public void testCommitMerging() + { + Committed committed = newProposal(1, "1").accepted().committed(); + Committed committedWithTTL1 = new CommittedWithTTL(committed, 100); + Committed committedWithTTL2 = new CommittedWithTTL(committed, 200); + + assertCommittedMerge(committed, committed, committedWithTTL1); + assertCommittedMerge(committed, committedWithTTL1, committed); + assertCommittedMerge(committedWithTTL2, committedWithTTL1, committedWithTTL2); + assertCommittedMerge(committedWithTTL2, committedWithTTL2, committedWithTTL1); + } +} diff --git a/test/unit/org/apache/cassandra/service/paxos/cleanup/PaxosTableRepairsTest.java b/test/unit/org/apache/cassandra/service/paxos/cleanup/PaxosTableRepairsTest.java new file mode 100644 index 0000000000..22441fec42 --- /dev/null +++ b/test/unit/org/apache/cassandra/service/paxos/cleanup/PaxosTableRepairsTest.java @@ -0,0 +1,262 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.paxos.cleanup; + +import org.junit.Assert; +import org.junit.Test; + +import org.apache.cassandra.db.ConsistencyLevel; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.dht.Murmur3Partitioner; +import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.service.paxos.AbstractPaxosRepair; +import org.apache.cassandra.service.paxos.Ballot; +import org.apache.cassandra.service.paxos.BallotGenerator; +import org.apache.cassandra.service.paxos.cleanup.PaxosTableRepairs.KeyRepair; +import org.apache.cassandra.utils.ByteBufferUtil; + +import static org.apache.cassandra.service.paxos.Ballot.Flag.NONE; + +public class PaxosTableRepairsTest +{ + private static DecoratedKey dk(int k) + { + return Murmur3Partitioner.instance.decorateKey(ByteBufferUtil.bytes(k)); + } + + private static final DecoratedKey DK1 = dk(1); + private static final DecoratedKey DK2 = dk(2); + + private static class MockRepair extends AbstractPaxosRepair + { + private static State STARTED = new State(); + private boolean reportCompleted = false; + private boolean failOnStart = false; + + public MockRepair(DecoratedKey key) + { + super(key, null); + } + + public State restart(State state, long waitUntil) + { + if (failOnStart) + throw new RuntimeException(); + return STARTED; + } + + void complete() + { + set(DONE); + } + + public boolean isComplete() + { + return reportCompleted || super.isComplete(); + } + } + + private static class MockTableRepairs extends PaxosTableRepairs + { + @Override + MockRepair createRepair(DecoratedKey key, Ballot incompleteBallot, ConsistencyLevel consistency, TableMetadata cfm) + { + return new MockRepair(key); + } + + MockRepair startOrGetOrQueue(DecoratedKey key, int i) + { + return (MockRepair) startOrGetOrQueue(key, BallotGenerator.Global.atUnixMicros(i, NONE), ConsistencyLevel.SERIAL, null, r -> {}); + } + } + + /** + * repairs with different keys shouldn't interfere with each other + */ + @Test + public void testMultipleRepairs() + { + MockTableRepairs repairs = new MockTableRepairs(); + + MockRepair repair1 = repairs.startOrGetOrQueue(DK1, 0); + MockRepair repair2 = repairs.startOrGetOrQueue(DK2, 1); + + Assert.assertTrue(repair1.isStarted()); + Assert.assertTrue(repair2.isStarted()); + Assert.assertTrue(repairs.hasActiveRepairs(DK1)); + Assert.assertTrue(repairs.hasActiveRepairs(DK2)); + + repair1.complete(); + repair2.complete(); + + // completing the repairs should have cleaned repairs map + Assert.assertFalse(repairs.hasActiveRepairs(DK1)); + Assert.assertFalse(repairs.hasActiveRepairs(DK2)); + } + + @Test + public void testRepairQueueing() + { + MockTableRepairs repairs = new MockTableRepairs(); + + MockRepair repair1 = repairs.startOrGetOrQueue(DK1, 0); + MockRepair repair2 = repairs.startOrGetOrQueue(DK1, 1); + MockRepair repair3 = repairs.startOrGetOrQueue(DK1, 2); + + Assert.assertTrue(repair1.isStarted()); + Assert.assertFalse(repair2.isStarted()); + Assert.assertFalse(repair3.isStarted()); + + KeyRepair keyRepair = repairs.getKeyRepairUnsafe(DK1); + Assert.assertEquals(repair1, keyRepair.activeRepair()); + Assert.assertTrue(keyRepair.queueContains(repair2)); + Assert.assertTrue(keyRepair.queueContains(repair3)); + + repair1.complete(); + Assert.assertTrue(repair2.isStarted()); + Assert.assertTrue(repairs.hasActiveRepairs(DK1)); + Assert.assertEquals(repair2, keyRepair.activeRepair()); + Assert.assertTrue(keyRepair.queueContains(repair3)); + + repair2.complete(); + Assert.assertTrue(repair3.isStarted()); + Assert.assertTrue(repairs.hasActiveRepairs(DK1)); + + // completing the final repair should cleanup the map + repair3.complete(); + Assert.assertFalse(repairs.hasActiveRepairs(DK1)); + } + + @Test + public void testRepairCancellation() + { + MockTableRepairs repairs = new MockTableRepairs(); + + MockRepair repair1 = repairs.startOrGetOrQueue(DK1, 0); + MockRepair repair2 = repairs.startOrGetOrQueue(DK1, 1); + MockRepair repair3 = repairs.startOrGetOrQueue(DK1, 2); + + Assert.assertTrue(repair1.isStarted()); + Assert.assertFalse(repair2.isStarted()); + Assert.assertFalse(repair3.isStarted()); + Assert.assertTrue(repairs.hasActiveRepairs(DK1)); + + repairs.clear(); + Assert.assertTrue(repair2.isComplete()); + Assert.assertTrue(repair3.isComplete()); + Assert.assertFalse(repairs.hasActiveRepairs(DK1)); + + MockRepair repair4 = repairs.startOrGetOrQueue(DK1, 0); + Assert.assertTrue(repair4.isStarted()); + Assert.assertTrue(repairs.hasActiveRepairs(DK1)); + repair4.complete(); + } + + @Test + public void testQueuedCancellation() + { + // if a queued repair is cancelled, it should be removed without affecting the active repair + MockTableRepairs repairs = new MockTableRepairs(); + MockRepair repair1 = repairs.startOrGetOrQueue(DK1, 0); + MockRepair repair2 = repairs.startOrGetOrQueue(DK1, 1); + MockRepair repair3 = repairs.startOrGetOrQueue(DK1, 2); + + KeyRepair keyRepair = repairs.getKeyRepairUnsafe(DK1); + Assert.assertEquals(repair1, keyRepair.activeRepair()); + Assert.assertTrue(keyRepair.queueContains(repair2)); + + repair2.cancel(); + Assert.assertEquals(repair1, keyRepair.activeRepair()); + Assert.assertFalse(keyRepair.queueContains(repair2)); + + repair1.complete(); + Assert.assertEquals(repair3, keyRepair.activeRepair()); + Assert.assertFalse(keyRepair.queueContains(repair2)); + } + + @Test + public void testFailureToStart() + { + // if an exception is thrown during repair scheduling, the next repair should be scheduled or things should be cleaned up + MockTableRepairs repairs = new MockTableRepairs(); + MockRepair repair1 = repairs.startOrGetOrQueue(DK1, 0); + MockRepair repair2 = repairs.startOrGetOrQueue(DK1, 1); + MockRepair repair3 = repairs.startOrGetOrQueue(DK1, 2); + + repair2.failOnStart = true; + KeyRepair keyRepair = repairs.getKeyRepairUnsafe(DK1); + Assert.assertEquals(repair1, keyRepair.activeRepair()); + Assert.assertTrue(keyRepair.queueContains(repair2)); + Assert.assertTrue(keyRepair.queueContains(repair3)); + Assert.assertFalse(repair2.isComplete()); + + repair1.complete(); + Assert.assertEquals(repair3, keyRepair.activeRepair()); + Assert.assertFalse(keyRepair.queueContains(repair2)); + Assert.assertTrue(repair2.isComplete()); + } + + @Test + public void testCompletedQueuedRepair() + { + // if a queued repair has been somehow completed (or cancelled) without also being removed, it should be skipped + MockTableRepairs repairs = new MockTableRepairs(); + MockRepair repair1 = repairs.startOrGetOrQueue(DK1, 0); + MockRepair repair2 = repairs.startOrGetOrQueue(DK1, 1); + MockRepair repair3 = repairs.startOrGetOrQueue(DK1, 2); + + repair2.reportCompleted = true; + KeyRepair keyRepair = repairs.getKeyRepairUnsafe(DK1); + Assert.assertEquals(repair1, keyRepair.activeRepair()); + Assert.assertTrue(keyRepair.queueContains(repair2)); + Assert.assertTrue(keyRepair.queueContains(repair3)); + + repair1.complete(); + Assert.assertEquals(repair3, keyRepair.activeRepair()); + Assert.assertFalse(keyRepair.queueContains(repair2)); + } + + @Test + public void testEviction() + { + MockTableRepairs repairs = new MockTableRepairs(); + MockRepair repair1 = repairs.startOrGetOrQueue(DK1, 0); + MockRepair repair2 = repairs.startOrGetOrQueue(DK1, 1); + + repairs.evictHungRepairs(System.nanoTime()); + KeyRepair keyRepair = repairs.getKeyRepairUnsafe(DK1); + Assert.assertTrue(repair1.isComplete()); + Assert.assertEquals(repair2, keyRepair.activeRepair()); + } + + @Test + public void testClearRepairs() + { + MockTableRepairs repairs = new MockTableRepairs(); + MockRepair repair1 = repairs.startOrGetOrQueue(DK1, 0); + MockRepair repair2 = repairs.startOrGetOrQueue(DK1, 1); + + KeyRepair keyRepair = repairs.getKeyRepairUnsafe(DK1); + Assert.assertEquals(repair1, keyRepair.activeRepair()); + + repairs.clear(); + Assert.assertEquals(0, keyRepair.pending()); + Assert.assertNull(repairs.getKeyRepairUnsafe(DK1)); + } +} diff --git a/test/unit/org/apache/cassandra/service/paxos/uncommitted/PaxosBallotTrackerTest.java b/test/unit/org/apache/cassandra/service/paxos/uncommitted/PaxosBallotTrackerTest.java new file mode 100644 index 0000000000..1f9db0851f --- /dev/null +++ b/test/unit/org/apache/cassandra/service/paxos/uncommitted/PaxosBallotTrackerTest.java @@ -0,0 +1,258 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.paxos.uncommitted; + +import java.io.IOException; + +import org.apache.cassandra.cql3.statements.schema.CreateTableStatement; +import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.service.paxos.Ballot; +import org.apache.cassandra.service.paxos.PaxosState.MaybePromise.Outcome; +import org.junit.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.SchemaLoader; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.db.ConsistencyLevel; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.partitions.PartitionUpdate; +import org.apache.cassandra.schema.KeyspaceParams; +import org.apache.cassandra.service.paxos.Commit; +import org.apache.cassandra.service.paxos.Paxos; +import org.apache.cassandra.service.paxos.PaxosState; +import org.apache.cassandra.utils.ByteBufferUtil; + +import static org.apache.cassandra.service.paxos.PaxosState.MaybePromise.Outcome.REJECT; +import static org.apache.cassandra.service.paxos.PaxosState.ballotTracker; +import static org.apache.cassandra.service.paxos.uncommitted.PaxosUncommittedTests.PAXOS_CFS; + +public class PaxosBallotTrackerTest +{ + private static final Logger logger = LoggerFactory.getLogger(PaxosBallotTrackerTest.class); + + protected static String ks; + protected static final String tbl = "tbl"; + protected static TableMetadata cfm; + + // which stage the ballot is tested at + enum Stage { PREPARE, PROPOSE, COMMIT } + enum Order + { + FIRST, // first ballot + SUBSEQUENT, // newest ballot + SUPERSEDED // ballot + } + + @BeforeClass + public static void setUpClass() throws Exception + { + SchemaLoader.prepareServer(); + + ks = "coordinatorsessiontest"; + cfm = CreateTableStatement.parse("CREATE TABLE tbl (k INT PRIMARY KEY, v INT)", ks).build(); + SchemaLoader.createKeyspace(ks, KeyspaceParams.simple(1), cfm); + } + + @Before + public void setUp() + { + PAXOS_CFS.truncateBlocking(); + PaxosState.unsafeReset(); + } + + private static DecoratedKey dk(int v) + { + return DatabaseDescriptor.getPartitioner().decorateKey(ByteBufferUtil.bytes(v)); + } + + private static void testHighBound(Stage stage, Order order) + { + logger.info("testHighBound for {} {} ", stage, order); + Ballot ballot1 = Paxos.ballotForConsistency(1001, ConsistencyLevel.SERIAL); + Ballot ballot2 = Paxos.ballotForConsistency(1002, ConsistencyLevel.SERIAL); + + Ballot opBallot; + Ballot expected; + switch (order) + { + case FIRST: + opBallot = ballot1; + expected = ballot1; + break; + case SUBSEQUENT: + ballotTracker().updateHighBoundUnsafe(Ballot.none(), ballot1); + Assert.assertEquals(ballot1, ballotTracker().getHighBound()); + opBallot = ballot2; + expected = ballot2; + break; + case SUPERSEDED: + ballotTracker().updateHighBoundUnsafe(Ballot.none(), ballot2); + Assert.assertEquals(ballot2, ballotTracker().getHighBound()); + opBallot = ballot1; + expected = ballot2; + break; + default: + throw new AssertionError(); + } + + DecoratedKey key = dk(1); + Commit.Proposal commit = new Commit.Proposal(opBallot, PaxosRowsTest.nonEmptyUpdate(opBallot, cfm, key)); + + switch (stage) + { + case PREPARE: + try (PaxosState state = PaxosState.get(commit)) + { + state.promiseIfNewer(commit.ballot, true); + } + break; + case PROPOSE: + try (PaxosState state = PaxosState.get(commit)) + { + state.acceptIfLatest(commit); + } + break; + case COMMIT: + PaxosState.commitDirect(commit); + break; + default: + throw new AssertionError(); + } + + Assert.assertEquals(expected, ballotTracker().getHighBound()); + } + + /** + * Tests that the ballot high bound is set correctly for all update types + */ + @Test + public void highBound() + { + for (Stage stage: Stage.values()) + { + for (Order order: Order.values()) + { + setUp(); + testHighBound(stage, order); + } + } + } + + @Test + public void lowBoundSet() throws IOException + { + PaxosBallotTracker ballotTracker = ballotTracker(); + Ballot ballot1 = Paxos.ballotForConsistency(1001, ConsistencyLevel.SERIAL); + Ballot ballot2 = Paxos.ballotForConsistency(1002, ConsistencyLevel.SERIAL); + Ballot ballot3 = Paxos.ballotForConsistency(1003, ConsistencyLevel.SERIAL); + + Assert.assertEquals(Ballot.none(), ballotTracker.getLowBound()); + + ballotTracker.updateLowBound(ballot2); + Assert.assertEquals(ballot2, ballotTracker.getLowBound()); + + ballotTracker.updateLowBound(ballot1); + Assert.assertEquals(ballot2, ballotTracker.getLowBound()); + + ballotTracker.updateLowBound(ballot3); + Assert.assertEquals(ballot3, ballotTracker.getLowBound()); + } + + @Test + public void lowBoundPrepare() throws IOException + { + PaxosBallotTracker ballotTracker = ballotTracker(); + Ballot ballot1 = Paxos.ballotForConsistency(1001, ConsistencyLevel.SERIAL); + Ballot ballot2 = Paxos.ballotForConsistency(1002, ConsistencyLevel.SERIAL); + Ballot ballot3 = Paxos.ballotForConsistency(1003, ConsistencyLevel.SERIAL); + Ballot ballot4 = Paxos.ballotForConsistency(1004, ConsistencyLevel.SERIAL); + + ballotTracker.updateLowBound(ballot1); + Assert.assertNotNull(ballotTracker.getLowBound()); + + DecoratedKey key = dk(1); + try (PaxosState state = PaxosState.get(key, cfm)) + { + PaxosState.MaybePromise promise = state.promiseIfNewer(ballot2, true); + Assert.assertEquals(Outcome.PROMISE, promise.outcome()); + Assert.assertNull(promise.supersededBy()); + } + + // set the lower bound into the 'future', and prepare with an earlier ballot + ballotTracker.updateLowBound(ballot4); + try (PaxosState state = PaxosState.get(key, cfm)) + { + PaxosState.MaybePromise promise = state.promiseIfNewer(ballot3, true); + Assert.assertEquals(REJECT, promise.outcome()); + Assert.assertEquals(ballot4, promise.supersededBy()); + } + } + + @Test + public void lowBoundAccept() throws IOException + { + PaxosBallotTracker ballotTracker = ballotTracker(); + Ballot ballot1 = Paxos.ballotForConsistency(1001, ConsistencyLevel.SERIAL); + Ballot ballot2 = Paxos.ballotForConsistency(1002, ConsistencyLevel.SERIAL); + Ballot ballot3 = Paxos.ballotForConsistency(1003, ConsistencyLevel.SERIAL); + Ballot ballot4 = Paxos.ballotForConsistency(1004, ConsistencyLevel.SERIAL); + + ballotTracker.updateLowBound(ballot1); + Assert.assertNotNull(ballotTracker.getLowBound()); + + DecoratedKey key = dk(1); + try (PaxosState state = PaxosState.get(key, cfm)) + { + Ballot result = state.acceptIfLatest(new Commit.Proposal(ballot2, PartitionUpdate.emptyUpdate(cfm, key))); + Assert.assertNull(result); + } + + // set the lower bound into the 'future', and prepare with an earlier ballot + ballotTracker.updateLowBound(ballot4); + try (PaxosState state = PaxosState.get(key, cfm)) + { + Ballot result = state.acceptIfLatest(new Commit.Proposal(ballot3, PartitionUpdate.emptyUpdate(cfm, key))); + Assert.assertEquals(ballot4, result); + } + } + + /** + * updating the lower bound should persist it to disk + */ + @Test + public void persistentLowBound() throws IOException + { + PaxosBallotTracker ballotTracker = ballotTracker(); + Ballot ballot1 = Paxos.ballotForConsistency(1001, ConsistencyLevel.SERIAL); + Assert.assertEquals(Ballot.none(), ballotTracker.getLowBound()); + + // a new tracker shouldn't load a ballot + PaxosBallotTracker tracker2 = PaxosBallotTracker.load(ballotTracker.getDirectory()); + Assert.assertEquals(Ballot.none(), tracker2.getLowBound()); + + // updating the lower bound should flush it to disk + ballotTracker.updateLowBound(ballot1); + Assert.assertEquals(ballot1, ballotTracker.getLowBound()); + + // then loading a new tracker should find the lower bound + PaxosBallotTracker tracker3 = PaxosBallotTracker.load(ballotTracker.getDirectory()); + Assert.assertEquals(ballot1, tracker3.getLowBound()); + } +} diff --git a/test/unit/org/apache/cassandra/service/paxos/uncommitted/PaxosMockUpdateSupplier.java b/test/unit/org/apache/cassandra/service/paxos/uncommitted/PaxosMockUpdateSupplier.java new file mode 100644 index 0000000000..518a71d9a8 --- /dev/null +++ b/test/unit/org/apache/cassandra/service/paxos/uncommitted/PaxosMockUpdateSupplier.java @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.paxos.uncommitted; + +import java.util.*; + +import com.google.common.collect.Iterables; +import com.google.common.collect.Iterators; + +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.Memtable; +import org.apache.cassandra.db.PartitionPosition; +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.Token; +import org.apache.cassandra.schema.TableId; +import org.apache.cassandra.service.paxos.Ballot; +import org.apache.cassandra.utils.CloseableIterator; + +public class PaxosMockUpdateSupplier implements PaxosUncommittedTracker.UpdateSupplier +{ + private final Map> states = new HashMap<>(); + + private NavigableMap mapFor(TableId tableId) + { + return states.computeIfAbsent(tableId, key -> new TreeMap<>()); + } + + private void updateTo(TableId tableId, PaxosKeyState newState) + { + NavigableMap map = mapFor(tableId); + PaxosKeyState current = map.get(newState.key); + if (current != null && PaxosKeyState.BALLOT_COMPARATOR.compare(current, newState) > 0) + return; + + map.put(newState.key, newState); + } + + void inProgress(TableId tableId, DecoratedKey key, Ballot ballot) + { + updateTo(tableId, new PaxosKeyState(tableId, key, ballot, false)); + } + + void committed(TableId tableId, DecoratedKey key, Ballot ballot) + { + updateTo(tableId, new PaxosKeyState(tableId, key, ballot, true)); + } + + public CloseableIterator repairIterator(TableId tableId, Collection> ranges) + { + Iterator iterator = Iterators.filter(mapFor(tableId).values().iterator(), k -> Iterables.any(ranges, r -> r.contains(k.key.getToken()))); + + return new CloseableIterator() + { + public void close() {} + + public boolean hasNext() + { + return iterator.hasNext(); + } + + public PaxosKeyState next() + { + return iterator.next(); + } + }; + } + + public CloseableIterator flushIterator(Memtable memtable) + { + ArrayList keyStates = new ArrayList<>(); + for (Map.Entry> statesEntry : states.entrySet()) + { + for (Map.Entry entry : statesEntry.getValue().entrySet()) + { + keyStates.add(entry.getValue()); + } + } + states.clear(); + + Iterator iterator = keyStates.iterator(); + + return new CloseableIterator() + { + public void close() {} + + public boolean hasNext() + { + return iterator.hasNext(); + } + + public PaxosKeyState next() + { + return iterator.next(); + } + }; + } +} diff --git a/test/unit/org/apache/cassandra/service/paxos/uncommitted/PaxosRowsTest.java b/test/unit/org/apache/cassandra/service/paxos/uncommitted/PaxosRowsTest.java new file mode 100644 index 0000000000..96fe154d01 --- /dev/null +++ b/test/unit/org/apache/cassandra/service/paxos/uncommitted/PaxosRowsTest.java @@ -0,0 +1,165 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.paxos.uncommitted; + +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; + +import com.google.common.collect.Iterators; +import com.google.common.collect.Lists; +import org.junit.*; + +import org.apache.cassandra.SchemaLoader; +import org.apache.cassandra.cql3.ColumnIdentifier; +import org.apache.cassandra.cql3.statements.schema.CreateTableStatement; +import org.apache.cassandra.db.*; +import org.apache.cassandra.db.marshal.UUIDType; +import org.apache.cassandra.db.partitions.PartitionUpdate; +import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator; +import org.apache.cassandra.db.rows.BTreeRow; +import org.apache.cassandra.db.rows.BufferCell; +import org.apache.cassandra.db.rows.Cell; +import org.apache.cassandra.db.rows.Row; +import org.apache.cassandra.db.rows.UnfilteredRowIterator; +import org.apache.cassandra.schema.ColumnMetadata; +import org.apache.cassandra.schema.TableId; +import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.service.paxos.Ballot; +import org.apache.cassandra.service.paxos.Commit; +import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.utils.CloseableIterator; +import org.apache.cassandra.utils.FBUtilities; +import org.apache.cassandra.utils.btree.BTree; + +import static org.apache.cassandra.service.paxos.uncommitted.PaxosUncommittedTests.PAXOS_CFM; +import static org.apache.cassandra.service.paxos.uncommitted.PaxosUncommittedTests.PAXOS_CFS; +import static org.apache.cassandra.service.paxos.uncommitted.PaxosUncommittedTests.createBallots; +import static org.apache.cassandra.service.paxos.uncommitted.PaxosUncommittedTests.dk; + +public class PaxosRowsTest +{ + protected static String ks; + protected static final String tbl = "tbl"; + protected static TableMetadata metadata; + protected static TableId tableId; + + static Commit emptyCommitFor(Ballot ballot, DecoratedKey key) + { + return new Commit(ballot, PartitionUpdate.emptyUpdate(metadata, key)); + } + + static Commit nonEmptyCommitFor(Ballot ballot, DecoratedKey key) + { + return new Commit(ballot, nonEmptyUpdate(ballot, metadata, key)); + } + + static PartitionUpdate nonEmptyUpdate(Ballot ballot, TableMetadata cfm, DecoratedKey key) + { + ColumnMetadata valueColumn = cfm.getColumn(new ColumnIdentifier("v", false)); + return PartitionUpdate.singleRowUpdate(cfm, key, BTreeRow.create(Clustering.EMPTY, LivenessInfo.EMPTY, Row.Deletion.LIVE, BTree.singleton(new BufferCell(valueColumn, ballot.unixMicros(), Cell.NO_TTL, Cell.NO_DELETION_TIME, ByteBufferUtil.bytes(1), null)))); + } + + static Row paxosRowFor(DecoratedKey key) + { + SinglePartitionReadCommand command = SinglePartitionReadCommand.create(PAXOS_CFM, + FBUtilities.nowInSeconds(), + key, + new BufferClustering(UUIDType.instance.decompose(tableId.asUUID()))); + try (ReadExecutionController opGroup = command.executionController(); + UnfilteredPartitionIterator iterator = command.executeLocally(opGroup); + UnfilteredRowIterator partition = Iterators.getOnlyElement(iterator)) + { + return (Row) Iterators.getOnlyElement(partition); + } + } + + @BeforeClass + public static void setUpClass() throws Exception + { + SchemaLoader.prepareServer(); + SystemKeyspace.finishStartup(); + + ks = "coordinatorsessiontest"; + metadata = CreateTableStatement.parse("CREATE TABLE tbl (k INT PRIMARY KEY, v INT)", ks).build(); + tableId = metadata.id; + } + + @Before + public void setUp() throws Exception + { + PAXOS_CFS.truncateBlocking(); + } + + @Test + public void testRowInterpretation() + { + DecoratedKey key = dk(5); + Ballot[] ballots = createBallots(3); + + SystemKeyspace.savePaxosWritePromise(key, metadata, ballots[0]); + Assert.assertEquals(new PaxosKeyState(tableId, key, ballots[0], false), PaxosRows.getCommitState(key, paxosRowFor(key), null)); + SystemKeyspace.savePaxosProposal(emptyCommitFor(ballots[0], key)); + Assert.assertEquals(new PaxosKeyState(tableId, key, ballots[0], true), PaxosRows.getCommitState(key, paxosRowFor(key), null)); + + SystemKeyspace.savePaxosWritePromise(key, metadata, ballots[1]); + Assert.assertEquals(new PaxosKeyState(tableId, key, ballots[1], false), PaxosRows.getCommitState(key, paxosRowFor(key), null)); + SystemKeyspace.savePaxosProposal(nonEmptyCommitFor(ballots[1], key)); + Assert.assertEquals(new PaxosKeyState(tableId, key, ballots[1], false), PaxosRows.getCommitState(key, paxosRowFor(key), null)); + SystemKeyspace.savePaxosCommit(nonEmptyCommitFor(ballots[1], key)); + Assert.assertEquals(new PaxosKeyState(tableId, key, ballots[1], true), PaxosRows.getCommitState(key, paxosRowFor(key), null)); + + // test cfid filter mismatch + Assert.assertNull(PaxosRows.getCommitState(key, paxosRowFor(key), TableId.fromUUID(UUID.randomUUID()))); + + SystemKeyspace.savePaxosCommit(emptyCommitFor(ballots[2], key)); + Assert.assertEquals(new PaxosKeyState(tableId, key, ballots[2], true), PaxosRows.getCommitState(key, paxosRowFor(key), null)); + } + + @Test + public void testIterator() + { + Ballot[] ballots = createBallots(10); + List expected = new ArrayList<>(ballots.length); + for (int i=0; i iterator = PaxosRows.toIterator(partitions, metadata.id, true)) + { + Assert.assertEquals(expected, Lists.newArrayList(iterator)); + } + } +} diff --git a/test/unit/org/apache/cassandra/service/paxos/uncommitted/PaxosStateTrackerTest.java b/test/unit/org/apache/cassandra/service/paxos/uncommitted/PaxosStateTrackerTest.java new file mode 100644 index 0000000000..ee0878cef9 --- /dev/null +++ b/test/unit/org/apache/cassandra/service/paxos/uncommitted/PaxosStateTrackerTest.java @@ -0,0 +1,242 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.paxos.uncommitted; + +import java.io.IOException; +import java.util.Collections; + +import com.google.common.collect.Sets; +import com.google.common.io.Files; +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import org.apache.cassandra.SchemaLoader; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.cql3.ColumnIdentifier; +import org.apache.cassandra.db.Clustering; +import org.apache.cassandra.db.SystemKeyspace; +import org.apache.cassandra.db.marshal.Int32Type; +import org.apache.cassandra.db.partitions.PartitionUpdate; +import org.apache.cassandra.db.rows.BTreeRow; +import org.apache.cassandra.db.rows.BufferCell; +import org.apache.cassandra.db.rows.Cell; +import org.apache.cassandra.db.rows.Row; +import org.apache.cassandra.dht.IPartitioner; +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.Token; +import org.apache.cassandra.io.util.File; +import org.apache.cassandra.io.util.FileUtils; +import org.apache.cassandra.schema.ColumnMetadata; +import org.apache.cassandra.schema.KeyspaceParams; +import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.service.paxos.Ballot; +import org.apache.cassandra.service.paxos.Commit; +import org.apache.cassandra.service.paxos.PaxosRepairHistory; +import org.apache.cassandra.utils.ByteBufferUtil; + +import static org.apache.cassandra.service.paxos.uncommitted.PaxosStateTracker.stateDirectory; +import static org.apache.cassandra.service.paxos.uncommitted.PaxosUncommittedTests.*; +import static org.apache.cassandra.service.paxos.uncommitted.UncommittedTableDataTest.assertIteratorContents; +import static org.apache.cassandra.service.paxos.uncommitted.UncommittedTableDataTest.uncommitted; + + +public class PaxosStateTrackerTest +{ + private File directory1 = null; + private File directory2 = null; + private File[] directories = null; + protected static String ks; + protected static TableMetadata cfm1; + protected static TableMetadata cfm2; + + @BeforeClass + public static void setUpClass() throws Exception + { + SchemaLoader.prepareServer(); + + ks = "coordinatorsessiontest"; + cfm1 = TableMetadata.builder(ks, "tbl1").addPartitionKeyColumn("k", Int32Type.instance).addRegularColumn("v", Int32Type.instance).build(); + cfm2 = TableMetadata.builder(ks, "tbl2").addPartitionKeyColumn("k", Int32Type.instance).addRegularColumn("v", Int32Type.instance).build(); + SchemaLoader.createKeyspace(ks, KeyspaceParams.simple(1), cfm1, cfm2); + } + + @Before + public void setUp() throws Exception + { + PAXOS_CFS.truncateBlocking(); + PAXOS_REPAIR_CFS.truncateBlocking(); + + if (directory1 != null) + FileUtils.deleteRecursive(directory1); + if (directory2 != null) + FileUtils.deleteRecursive(directory2); + + directory1 = new File(Files.createTempDir()); + directory2 = new File(Files.createTempDir()); + directories = new File[]{directory1, directory2}; + } + + private static class SystemProp implements AutoCloseable + { + private final String prop; + private final String prev; + + public SystemProp(String prop, String prev) + { + this.prop = prop; + this.prev = prev; + } + + public void close() + { + if (prev == null) + System.clearProperty(prop); + else + System.setProperty(prop, prev); + } + + public static SystemProp set(String prop, String val) + { + String prev = System.getProperty(prop); + System.setProperty(prop, val); + return new SystemProp(prop, prev); + } + + public static SystemProp set(String prop, boolean val) + { + return set(prop, Boolean.toString(val)); + } + } + + private static PartitionUpdate update(TableMetadata cfm, int k, Ballot ballot) + { + ColumnMetadata col = cfm.getColumn(new ColumnIdentifier("v", false)); + Cell cell = BufferCell.live(col, ballot.unixMicros(), ByteBufferUtil.bytes(0)); + Row row = BTreeRow.singleCellRow(Clustering.EMPTY, cell); + return PartitionUpdate.singleRowUpdate(cfm, dk(k), row); + } + + private static Commit commit(TableMetadata cfm, int k, Ballot ballot) + { + return new Commit(ballot, update(cfm, k, ballot)); + } + + private static void savePaxosRepair(TableMetadata cfm, Range range, Ballot lowBound) + { + PaxosRepairHistory current = SystemKeyspace.loadPaxosRepairHistory(cfm.keyspace, cfm.name); + PaxosRepairHistory updated = PaxosRepairHistory.add(current, Collections.singleton(range), lowBound); + SystemKeyspace.savePaxosRepairHistory(cfm.keyspace, cfm.name, updated, true); + } + + private static void savePaxosRepair(TableMetadata cfm, int left, int right, Ballot lowBound) + { + IPartitioner partitioner = DatabaseDescriptor.getPartitioner(); + Range range = new Range<>(partitioner.getToken(ByteBufferUtil.bytes(left)), partitioner.getToken(ByteBufferUtil.bytes(right))); + savePaxosRepair(cfm, range, lowBound); + } + + private static void initDirectory(File directory) throws IOException + { + PaxosStateTracker.create(new File[]{directory}).ballots().flush(); + } + + @Test + public void autoRebuild() throws Throwable + { + Ballot[] ballots = createBallots(6); + + // save a promise, proposal, and commit to each table + SystemKeyspace.savePaxosWritePromise(dk(0), cfm1, ballots[0]); + SystemKeyspace.savePaxosWritePromise(dk(1), cfm2, ballots[1]); + SystemKeyspace.savePaxosProposal(commit(cfm1, 2, ballots[2])); + SystemKeyspace.savePaxosProposal(commit(cfm2, 3, ballots[3])); + SystemKeyspace.savePaxosCommit(commit(cfm1, 4, ballots[4])); + SystemKeyspace.savePaxosCommit(commit(cfm2, 5, ballots[5])); + + PaxosStateTracker tracker = PaxosStateTracker.create(directories); + Assert.assertTrue(tracker.isRebuildNeeded()); + Assert.assertEquals(Sets.newHashSet(), tracker.uncommitted().tableIds()); + + tracker.maybeRebuild(); + + Assert.assertEquals(stateDirectory(directory1), tracker.uncommitted().getDirectory()); + + Assert.assertEquals(Sets.newHashSet(cfm1.id, cfm2.id), tracker.uncommitted().tableIds()); + + UncommittedTableData tableData1 = tracker.uncommitted().getTableState(cfm1.id); + assertIteratorContents(tableData1.iterator(ALL_RANGES), kl(uncommitted(0, ballots[0]), + uncommitted(2, ballots[2]))); + + UncommittedTableData tableData2 = tracker.uncommitted().getTableState(cfm2.id); + assertIteratorContents(tableData2.iterator(ALL_RANGES), kl(uncommitted(1, ballots[1]), + uncommitted(3, ballots[3]))); + } + + @Test + public void manualRebuild() throws Throwable + { + initDirectory(directory1); + { + PaxosStateTracker tracker = PaxosStateTracker.create(directories); + Assert.assertFalse(tracker.isRebuildNeeded()); + Assert.assertEquals(Ballot.none(), tracker.ballots().getLowBound()); + } + + Ballot[] ballots = createBallots(4); + savePaxosRepair(cfm1, 0, 10, ballots[0]); + savePaxosRepair(cfm1, 10, 20, ballots[1]); + SystemKeyspace.savePaxosWritePromise(dk(0), cfm1, ballots[2]); + SystemKeyspace.savePaxosProposal(commit(cfm1, 2, ballots[3])); + + try (SystemProp forceRebuild = SystemProp.set(PaxosStateTracker.FORCE_REBUILD_PROP, true)) + { + PaxosStateTracker tracker = PaxosStateTracker.create(directories); + Assert.assertTrue(tracker.isRebuildNeeded()); + Assert.assertEquals(Ballot.none(), tracker.ballots().getLowBound()); + tracker.maybeRebuild(); + + UncommittedTableData tableData1 = tracker.uncommitted().getTableState(cfm1.id); + assertIteratorContents(tableData1.iterator(ALL_RANGES), kl(uncommitted(0, ballots[2]), + uncommitted(2, ballots[3]))); + Assert.assertEquals(ballots[1], tracker.ballots().getLowBound()); + Assert.assertEquals(ballots[3], tracker.ballots().getHighBound()); + } + } + + // test we can find paxos data in any directory + @Test + public void testMultiDirectories() throws Throwable + { + initDirectory(directory2); + PaxosStateTracker tracker = PaxosStateTracker.create(directories); + Assert.assertFalse(tracker.isRebuildNeeded()); + Assert.assertEquals(stateDirectory(directory2), tracker.uncommitted().getDirectory()); + } + + // test multiple paxos directories throws exception + @Test(expected=IllegalStateException.class) + public void testConflictingDirectories() throws Throwable + { + initDirectory(directory1); + initDirectory(directory2); + PaxosStateTracker tracker = PaxosStateTracker.create(directories); + } +} diff --git a/test/unit/org/apache/cassandra/service/paxos/uncommitted/PaxosUncommittedTests.java b/test/unit/org/apache/cassandra/service/paxos/uncommitted/PaxosUncommittedTests.java new file mode 100644 index 0000000000..c693d14df1 --- /dev/null +++ b/test/unit/org/apache/cassandra/service/paxos/uncommitted/PaxosUncommittedTests.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.paxos.uncommitted; + +import java.util.*; + +import com.google.common.collect.Lists; + +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.db.*; +import org.apache.cassandra.db.commitlog.CommitLog; +import org.apache.cassandra.dht.*; +import org.apache.cassandra.schema.SchemaConstants; +import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.service.paxos.Ballot; +import org.apache.cassandra.utils.ByteBufferUtil; + +import static org.apache.cassandra.schema.SchemaConstants.SYSTEM_KEYSPACE_NAME; +import static org.apache.cassandra.service.paxos.Ballot.Flag.GLOBAL; +import static org.apache.cassandra.service.paxos.BallotGenerator.Global.nextBallot; + +class PaxosUncommittedTests +{ + static + { + DatabaseDescriptor.daemonInitialization(); + CommitLog.instance.start(); + } + + static final IPartitioner PARTITIONER = new ByteOrderedPartitioner(); + static final Token MIN_TOKEN = PARTITIONER.getMinimumToken(); + static final Range FULL_RANGE = new Range<>(MIN_TOKEN, MIN_TOKEN); + static final Collection> ALL_RANGES = Collections.singleton(FULL_RANGE); + static final ColumnFamilyStore PAXOS_CFS = Keyspace.open(SYSTEM_KEYSPACE_NAME).getColumnFamilyStore(SystemKeyspace.PAXOS); + static final TableMetadata PAXOS_CFM = PAXOS_CFS.metadata.get(); + static final ColumnFamilyStore PAXOS_REPAIR_CFS = Keyspace.open(SYSTEM_KEYSPACE_NAME).getColumnFamilyStore(SystemKeyspace.PAXOS_REPAIR_HISTORY); + static final TableMetadata PAXOS_REPAIR_CFM = PAXOS_REPAIR_CFS.metadata(); + + static Ballot[] createBallots(int num) + { + Ballot[] ballots = new Ballot[num]; + for (int i=0; i kl(Iterator iter) + { + return Lists.newArrayList(iter); + } + + static List kl(PaxosKeyState... states) + { + return Lists.newArrayList(states); + } + + static Token tk(int v) + { + return dk(v).getToken(); + } + + static Range r(Token start, Token stop) + { + return new Range<>(start != null ? start : MIN_TOKEN, stop != null ? stop : MIN_TOKEN); + } + + static Range r(int start, int stop) + { + return r(PARTITIONER.getToken(ByteBufferUtil.bytes(start)), PARTITIONER.getToken(ByteBufferUtil.bytes(stop))); + } + +} diff --git a/test/unit/org/apache/cassandra/service/paxos/uncommitted/PaxosUncommittedTrackerIntegrationTest.java b/test/unit/org/apache/cassandra/service/paxos/uncommitted/PaxosUncommittedTrackerIntegrationTest.java new file mode 100644 index 0000000000..4b9dbe6379 --- /dev/null +++ b/test/unit/org/apache/cassandra/service/paxos/uncommitted/PaxosUncommittedTrackerIntegrationTest.java @@ -0,0 +1,141 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.paxos.uncommitted; + +import com.google.common.collect.Iterators; +import com.google.common.collect.Lists; +import org.junit.*; + +import org.apache.cassandra.SchemaLoader; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.cql3.statements.schema.CreateTableStatement; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.schema.KeyspaceParams; +import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.service.paxos.Ballot; +import org.apache.cassandra.service.paxos.PaxosState; +import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.utils.CloseableIterator; + +import static org.apache.cassandra.service.paxos.Ballot.Flag.NONE; +import static org.apache.cassandra.service.paxos.BallotGenerator.Global.nextBallot; +import static org.apache.cassandra.service.paxos.Commit.*; +import static org.apache.cassandra.service.paxos.uncommitted.PaxosUncommittedTests.ALL_RANGES; +import static org.apache.cassandra.service.paxos.uncommitted.PaxosUncommittedTests.PAXOS_CFS; + +public class PaxosUncommittedTrackerIntegrationTest +{ + protected static String ks; + protected static final String tbl = "tbl"; + protected static TableMetadata cfm; + + @BeforeClass + public static void setUpClass() throws Exception + { + SchemaLoader.prepareServer(); + + ks = "coordinatorsessiontest"; + cfm = CreateTableStatement.parse("CREATE TABLE tbl (k INT PRIMARY KEY, v INT)", ks).build(); + SchemaLoader.createKeyspace(ks, KeyspaceParams.simple(1), cfm); + } + + @Before + public void setUp() throws Exception + { + PAXOS_CFS.truncateBlocking(); + } + + private static DecoratedKey dk(int v) + { + return DatabaseDescriptor.getPartitioner().decorateKey(ByteBufferUtil.bytes(v)); + } + + @Test + public void commitCycle() + { + PaxosUncommittedTracker tracker = PaxosState.uncommittedTracker(); + PaxosBallotTracker ballotTracker = PaxosState.ballotTracker(); + Assert.assertNull(tracker.getTableState(cfm.id)); + Assert.assertEquals(Ballot.none(), ballotTracker.getLowBound()); + Assert.assertEquals(Ballot.none(), ballotTracker.getHighBound()); + + try (CloseableIterator iterator = tracker.uncommittedKeyIterator(cfm.id, ALL_RANGES)) + { + Assert.assertFalse(iterator.hasNext()); + } + + DecoratedKey key = dk(1); + Ballot ballot = nextBallot(NONE); + Proposal proposal = new Proposal(ballot, PaxosRowsTest.nonEmptyUpdate(ballot, cfm, key)); + + try (PaxosState state = PaxosState.get(key, cfm)) + { + state.promiseIfNewer(proposal.ballot, true); + } + + try (CloseableIterator iterator = tracker.uncommittedKeyIterator(cfm.id, ALL_RANGES)) + { + Assert.assertEquals(key, Iterators.getOnlyElement(iterator).getKey()); + } + + try (PaxosState state = PaxosState.get(key, cfm)) + { + state.acceptIfLatest(proposal); + } + + try (CloseableIterator iterator = tracker.uncommittedKeyIterator(cfm.id, ALL_RANGES)) + { + Assert.assertEquals(key, Iterators.getOnlyElement(iterator).getKey()); + } + + PaxosState.commitDirect(proposal.agreed()); + try (CloseableIterator iterator = tracker.uncommittedKeyIterator(cfm.id, ALL_RANGES)) + { + Assert.assertFalse(iterator.hasNext()); + } + } + + @Test + public void inMemoryCommit() + { + PaxosUncommittedTracker tracker = PaxosState.uncommittedTracker(); + + DecoratedKey key = dk(1); + Ballot ballot = nextBallot(NONE); + Proposal proposal = new Proposal(ballot, PaxosRowsTest.nonEmptyUpdate(ballot, cfm, key)); + + try (PaxosState state = PaxosState.get(key, cfm)) + { + state.promiseIfNewer(proposal.ballot, true); + state.acceptIfLatest(proposal); + } + try (CloseableIterator iterator = tracker.uncommittedKeyIterator(cfm.id, ALL_RANGES)) + { + Assert.assertEquals(key, Iterators.getOnlyElement(iterator).getKey()); + } + + PAXOS_CFS.forceBlockingFlush(); + + PaxosState.commitDirect(proposal.agreed()); + try (CloseableIterator iterator = tracker.uncommittedKeyIterator(cfm.id, ALL_RANGES)) + { + Assert.assertEquals(Lists.newArrayList(), Lists.newArrayList(iterator)); + } + } +} diff --git a/test/unit/org/apache/cassandra/service/paxos/uncommitted/PaxosUncommittedTrackerTest.java b/test/unit/org/apache/cassandra/service/paxos/uncommitted/PaxosUncommittedTrackerTest.java new file mode 100644 index 0000000000..6af323f866 --- /dev/null +++ b/test/unit/org/apache/cassandra/service/paxos/uncommitted/PaxosUncommittedTrackerTest.java @@ -0,0 +1,248 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.paxos.uncommitted; + +import java.util.*; + +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; +import com.google.common.io.Files; +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import org.apache.cassandra.SchemaLoader; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.marshal.Int32Type; +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.Token; +import org.apache.cassandra.io.util.File; +import org.apache.cassandra.io.util.FileUtils; +import org.apache.cassandra.schema.KeyspaceParams; +import org.apache.cassandra.schema.TableId; +import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.service.paxos.Ballot; +import org.apache.cassandra.utils.CloseableIterator; + +import static org.apache.cassandra.service.paxos.uncommitted.PaxosUncommittedTests.*; + +public class PaxosUncommittedTrackerTest +{ + private static final String KS = "ks"; + private static final String TBL = "tbl"; + private static TableId cfid; + private File directory = null; + private PaxosUncommittedTracker tracker; + private PaxosMockUpdateSupplier updates; + private UncommittedTableData state; + + @BeforeClass + public static void setupClass() + { + SchemaLoader.prepareServer(); + TableMetadata tableMetadata = TableMetadata.builder("ks", "tbl") + .addPartitionKeyColumn("k", Int32Type.instance) + .addRegularColumn("v", Int32Type.instance) + .build(); + cfid = tableMetadata.id; + SchemaLoader.createKeyspace(KS, KeyspaceParams.simple(1), tableMetadata); + } + + @Before + public void setUp() + { + if (directory != null) + FileUtils.deleteRecursive(directory); + + directory = new File(Files.createTempDir()); + + tracker = new PaxosUncommittedTracker(directory); + tracker.start(); + updates = new PaxosMockUpdateSupplier(); + tracker.unsafSetUpdateSupplier(updates); + state = tracker.getOrCreateTableState(cfid); + } + + private static List uncommittedList(PaxosUncommittedTracker tracker, Collection> ranges) + { + try (CloseableIterator iterator = tracker.uncommittedKeyIterator(cfid, ranges)) + { + return Lists.newArrayList(iterator); + } + } + + private static List uncommittedList(PaxosUncommittedTracker tracker, Range range) + { + return uncommittedList(tracker, Collections.singleton(range)); + } + + private static List uncommittedList(PaxosUncommittedTracker tracker) + { + return uncommittedList(tracker, FULL_RANGE); + } + + @Test + public void inmemory() throws Exception + { + Assert.assertEquals(0, state.numFiles()); + int size = 5; + List expected = new ArrayList<>(size); + int key = 0; + for (Ballot ballot : createBallots(size)) + { + DecoratedKey dk = dk(key++); + updates.inProgress(cfid, dk, ballot); + expected.add(new PaxosKeyState(cfid, dk, ballot, false)); + } + + Assert.assertEquals(expected, uncommittedList(tracker)); + Assert.assertEquals(0, state.numFiles()); + } + + @Test + public void onDisk() throws Exception + { + Assert.assertEquals(0, state.numFiles()); + int size = 5; + List expected = new ArrayList<>(size); + int key = 0; + for (Ballot ballot : createBallots(size)) + { + DecoratedKey dk = dk(key++); + updates.inProgress(cfid, dk, ballot); + expected.add(new PaxosKeyState(cfid, dk, ballot, false)); + } + tracker.flushUpdates(null); + + Assert.assertEquals(expected, uncommittedList(tracker)); + + Assert.assertEquals(1, state.numFiles()); + Assert.assertEquals(expected, kl(state.iterator(Collections.singleton(FULL_RANGE)))); + } + + @Test + public void mixed() throws Exception + { + Assert.assertEquals(0, state.numFiles()); + int size = 10; + PaxosKeyState[] expectedArr = new PaxosKeyState[size]; + List inMemory = new ArrayList<>(size / 2); + List onDisk = new ArrayList<>(size / 2); + Ballot[] ballots = createBallots(size); + + for (int i=0; i expected = kl(expectedArr); + + Assert.assertEquals(expected, uncommittedList(tracker)); + + Assert.assertEquals(1, state.numFiles()); + Assert.assertEquals(onDisk, kl(state.iterator(Collections.singleton(FULL_RANGE)))); + } + + @Test + public void committed() + { + UncommittedTableData tableData = UncommittedTableData.load(directory, cfid); + Assert.assertEquals(0, tableData.numFiles()); + Ballot ballot = createBallots(1)[0]; + + DecoratedKey dk = dk(1); + updates.inProgress(cfid, dk, ballot); + + Assert.assertEquals(kl(new PaxosKeyState(cfid, dk, ballot, false)), uncommittedList(tracker)); + + updates.committed(cfid, dk, ballot); + Assert.assertTrue(uncommittedList(tracker).isEmpty()); + } + + /** + * Test that commits don't resolve in progress transactions with more recent ballots + */ + @Test + public void pastCommit() + { + Ballot[] ballots = createBallots(2); + DecoratedKey dk = dk(1); + Assert.assertTrue(ballots[1].uuidTimestamp() > ballots[0].uuidTimestamp()); + + updates.inProgress(cfid, dk, ballots[1]); + updates.committed(cfid, dk, ballots[0]); + + Assert.assertEquals(kl(new PaxosKeyState(cfid, dk, ballots[1], false)), uncommittedList(tracker)); + } + + @Test + public void tokenRange() throws Exception + { + Assert.assertEquals(0, state.numFiles()); + int size = 10; + PaxosKeyState[] expectedArr = new PaxosKeyState[size]; + Ballot[] ballots = createBallots(size); + + for (int i=0; i expected = kl(expectedArr); + + Assert.assertEquals(expected.subList(0, 5), uncommittedList(tracker, r(null, tk(4)))); + Assert.assertEquals(expected.subList(3, 7), uncommittedList(tracker, r(2, 6))); + Assert.assertEquals(expected.subList(8, 10), uncommittedList(tracker, r(tk(7), null))); + Assert.assertEquals(Lists.newArrayList(Iterables.concat(expected.subList(1, 5), expected.subList(6, 9))), + uncommittedList(tracker, Lists.newArrayList(r(0, 4), r(5, 8)))); + } +} diff --git a/test/unit/org/apache/cassandra/service/paxos/uncommitted/UncommittedTableDataTest.java b/test/unit/org/apache/cassandra/service/paxos/uncommitted/UncommittedTableDataTest.java new file mode 100644 index 0000000000..70b1f254c1 --- /dev/null +++ b/test/unit/org/apache/cassandra/service/paxos/uncommitted/UncommittedTableDataTest.java @@ -0,0 +1,650 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.paxos.uncommitted; + +import java.io.IOException; +import java.nio.charset.Charset; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.UUID; +import java.util.stream.Collectors; + +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; +import com.google.common.io.Files; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.Token; +import org.apache.cassandra.io.util.File; +import org.apache.cassandra.io.util.FileUtils; +import org.apache.cassandra.schema.TableId; +import org.apache.cassandra.service.paxos.Ballot; +import org.apache.cassandra.service.paxos.PaxosRepairHistory; +import org.apache.cassandra.service.paxos.uncommitted.UncommittedTableData.Data; +import org.apache.cassandra.service.paxos.uncommitted.UncommittedTableData.FilterFactory; +import org.apache.cassandra.service.paxos.uncommitted.UncommittedTableData.FlushWriter; +import org.apache.cassandra.service.paxos.uncommitted.UncommittedTableData.Merge; +import org.apache.cassandra.utils.CloseableIterator; + +import static org.apache.cassandra.service.paxos.uncommitted.PaxosUncommittedTests.*; + +public class UncommittedTableDataTest +{ + private static final String KS = "ks"; + private static final String TBL = "tbl"; + private static final TableId CFID = TableId.fromUUID(UUID.randomUUID()); + + private static final String TBL2 = "tbl2"; + private static final TableId CFID2 = TableId.fromUUID(UUID.randomUUID()); + + private File directory = null; + + private static class MockDataFile + { + final File data; + final File crc; + + public MockDataFile(File data, File crc) + { + this.data = data; + this.crc = crc; + } + + boolean exists() + { + return data.exists() && crc.exists(); + } + + boolean isDeleted() + { + return !data.exists() && !crc.exists(); + } + } + + private static final FilterFactory NOOP_FACTORY = new FilterFactory() + { + List> getReplicatedRanges() + { + return new ArrayList<>(ALL_RANGES); + } + + PaxosRepairHistory getPaxosRepairHistory() + { + return PaxosRepairHistory.EMPTY; + } + }; + + private static UncommittedTableData load(File directory, TableId cfid) + { + return UncommittedTableData.load(directory, cfid, NOOP_FACTORY); + } + + MockDataFile mockFile(String table, TableId cfid, long generation, boolean temp) + { + String fname = UncommittedDataFile.fileName(KS, table, cfid, generation) + (temp ? UncommittedDataFile.TMP_SUFFIX : ""); + File data = new File(directory, fname); + File crc = new File(directory, UncommittedDataFile.crcName(fname)); + try + { + Files.write("data", data.toJavaIOFile(), Charset.defaultCharset()); + Files.write("crc", crc.toJavaIOFile(), Charset.defaultCharset()); + } + catch (IOException e) + { + throw new RuntimeException(e); + } + + MockDataFile file = new MockDataFile(data, crc); + Assert.assertTrue(file.exists()); + return file; + } + + MockDataFile mockFile(long generation, boolean temp) + { + return mockFile(TBL, CFID, generation, temp); + } + + @Before + public void setUp() throws Exception + { + if (directory != null) + FileUtils.deleteRecursive(directory); + + directory = new File(Files.createTempDir()); + } + + static PaxosKeyState uncommitted(int key, Ballot ballot) + { + return new PaxosKeyState(CFID, dk(key), ballot, false); + } + + static PaxosKeyState committed(int key, Ballot ballot) + { + return new PaxosKeyState(CFID, dk(key), ballot, true); + } + + private static FlushWriter startFlush(UncommittedTableData tableData, List updates) throws IOException + { + FlushWriter writer = tableData.flushWriter(); + writer.appendAll(updates); + return writer; + } + + private static FlushWriter startFlush(UncommittedTableData tableData, PaxosKeyState... updates) throws IOException + { + return startFlush(tableData, Lists.newArrayList(updates)); + } + + private static void flush(UncommittedTableData tableData, List updates) throws IOException + { + startFlush(tableData, updates).finish(); + } + + private static void flush(UncommittedTableData tableData, PaxosKeyState... states) throws IOException + { + flush(tableData, Lists.newArrayList(states)); + } + + private void mergeWithUpdates(UncommittedTableData tableData, List toAdd) throws IOException + { + flush(tableData, toAdd); + tableData.createMergeTask().run(); + } + + static void assertIteratorContents(CloseableIterator iterator, Iterable expected) + { + try (CloseableIterator iter = iterator) + { + Assert.assertEquals(Lists.newArrayList(expected), Lists.newArrayList(iter)); + } + } + + static void assertFileContents(UncommittedDataFile file, int generation, List expected) + { + Assert.assertEquals(generation, file.generation()); + assertIteratorContents(file.iterator(ALL_RANGES), expected); + } + + static void assertIteratorContents(UncommittedTableData tableData, int generation, List expected) + { + Assert.assertEquals(generation, tableData.nextGeneration() - 1); + assertIteratorContents(tableData.iterator(ALL_RANGES), expected); + } + + + /** + * Test various merge scenarios + */ + @Test + public void testMergeWriter() throws IOException + { + Ballot[] ballots = createBallots(5); + UncommittedTableData tableData = load(directory, CFID); + + mergeWithUpdates(tableData, kl(new PaxosKeyState(CFID, dk(3), ballots[1], false), + new PaxosKeyState(CFID, dk(5), ballots[1], false), + new PaxosKeyState(CFID, dk(7), ballots[1], false), + new PaxosKeyState(CFID, dk(9), ballots[1], false))); + + assertIteratorContents(tableData, 1, kl(new PaxosKeyState(CFID, dk(3), ballots[1], false), + new PaxosKeyState(CFID, dk(5), ballots[1], false), + new PaxosKeyState(CFID, dk(7), ballots[1], false), + new PaxosKeyState(CFID, dk(9), ballots[1], false))); + + // add a commit from the past for key 3, update key 5, and commit key 7 + mergeWithUpdates(tableData, kl(new PaxosKeyState(CFID, dk(3), ballots[0], true), + new PaxosKeyState(CFID, dk(5), ballots[2], false), + new PaxosKeyState(CFID, dk(7), ballots[2], true))); + + // key 7 should be gone because committed keys aren't written out + assertIteratorContents(tableData, 3, kl(new PaxosKeyState(CFID, dk(3), ballots[1], false), + new PaxosKeyState(CFID, dk(5), ballots[2], false), + new PaxosKeyState(CFID, dk(9), ballots[1], false))); + + // add a new key and update an adjacent one + mergeWithUpdates(tableData, kl(new PaxosKeyState(CFID, dk(4), ballots[3], false), + new PaxosKeyState(CFID, dk(5), ballots[3], false))); + assertIteratorContents(tableData, 5, kl(new PaxosKeyState(CFID, dk(3), ballots[1], false), + new PaxosKeyState(CFID, dk(4), ballots[3], false), + new PaxosKeyState(CFID, dk(5), ballots[3], false), + new PaxosKeyState(CFID, dk(9), ballots[1], false))); + + // add 2 new keys + mergeWithUpdates(tableData, kl(new PaxosKeyState(CFID, dk(6), ballots[4], false), + new PaxosKeyState(CFID, dk(7), ballots[4], false))); + assertIteratorContents(tableData, 7, kl(new PaxosKeyState(CFID, dk(3), ballots[1], false), + new PaxosKeyState(CFID, dk(4), ballots[3], false), + new PaxosKeyState(CFID, dk(5), ballots[3], false), + new PaxosKeyState(CFID, dk(6), ballots[4], false), + new PaxosKeyState(CFID, dk(7), ballots[4], false), + new PaxosKeyState(CFID, dk(9), ballots[1], false))); + } + + @Test + public void committedOpsArentWritten() throws Exception + { + Ballot[] ballots = createBallots(2); + + UncommittedTableData tableData = load(directory, CFID); + mergeWithUpdates(tableData, kl(new PaxosKeyState(CFID, dk(1), ballots[0], false))); + assertIteratorContents(tableData, 1, kl(new PaxosKeyState(CFID, dk(1), ballots[0], false))); + + mergeWithUpdates(tableData, kl(new PaxosKeyState(CFID, dk(1), ballots[1], true))); + assertIteratorContents(tableData, 3, kl()); + } + + @Test + public void testIterator() throws Exception + { + Ballot[] ballots = createBallots(10); + List expected = new ArrayList<>(ballots.length); + UncommittedTableData tableData = load(directory, CFID); + + for (int i=0; i updates = kl(uncommitted(3, ballots[1]), + uncommitted(5, ballots[1]), + committed(7, ballots[1]), + uncommitted(9, ballots[1])); + flush(tableData, updates); + + Data data = tableData.data(); + UncommittedDataFile updateFile = Iterables.getOnlyElement(data.files); + assertFileContents(updateFile, 0, updates); + assertIteratorContents(tableData.iterator(ALL_RANGES), updates); + } + + @Test + public void merge() throws Throwable + { + Ballot[] ballots = createBallots(5); + UncommittedTableData tableData = load(directory, CFID); + flush(tableData, + uncommitted(3, ballots[1]), + committed(7, ballots[1])); + flush(tableData, + uncommitted(5, ballots[1]), + uncommitted(9, ballots[1])); + + List updateFiles = tableData.data().files.stream().map(UncommittedDataFile::file).collect(Collectors.toList()); + Assert.assertTrue(Iterables.all(updateFiles, File::exists)); + + tableData.createMergeTask().run(); + + Assert.assertFalse(Iterables.any(updateFiles, File::exists)); + + Data data = tableData.data(); + Assert.assertEquals(1, data.files.size()); + + List expected = kl(uncommitted(3, ballots[1]), + uncommitted(5, ballots[1]), + uncommitted(9, ballots[1])); + assertFileContents(Iterables.getOnlyElement(data.files), 2, expected); + assertIteratorContents(tableData.iterator(ALL_RANGES), expected); + } + + /** + * nothing should break when a merge results in an empty file + */ + @Test + public void emptyMerge() throws Throwable + { + Ballot[] ballots = createBallots(5); + UncommittedTableData tableData = load(directory, CFID); + flush(tableData, + committed(3, ballots[1]), + committed(7, ballots[1])); + + tableData.createMergeTask().run(); + + Data data = tableData.data(); + + assertFileContents(Iterables.getOnlyElement(data.files), 1, Collections.emptyList()); + assertIteratorContents(tableData.iterator(ALL_RANGES), Collections.emptyList()); + } + + @Test + public void load() throws Throwable + { + Ballot[] ballots = createBallots(5); + UncommittedTableData tableData = load(directory, CFID); + flush(tableData, + uncommitted(3, ballots[1]), + committed(7, ballots[1])); + tableData.createMergeTask().run(); + flush(tableData, + uncommitted(5, ballots[1]), + uncommitted(9, ballots[1])); + List expected = kl(uncommitted(3, ballots[1]), + uncommitted(5, ballots[1]), + uncommitted(9, ballots[1])); + assertIteratorContents(tableData.iterator(ALL_RANGES), expected); + + // cleanup shouldn't touch files for other tables + MockDataFile mockStateFile = mockFile(TBL2, CFID2, 2, false); + MockDataFile mockUpdateFile = mockFile(TBL2, CFID2, 3, false); + MockDataFile mockTempUpdate = mockFile(TBL2, CFID2, 4, true); + + UncommittedTableData tableData2 = load(directory, CFID); + assertIteratorContents(tableData2.iterator(ALL_RANGES), expected); + Assert.assertTrue(mockStateFile.exists() && mockUpdateFile.exists() && mockTempUpdate.exists()); + } + + /** + * Shouldn't break if there wasn't a merge before shutdown + */ + @Test + public void loadWithoutStateFile() throws Throwable + { + Ballot[] ballots = createBallots(5); + UncommittedTableData tableData = load(directory, CFID); + + List updates = kl(uncommitted(3, ballots[1]), + uncommitted(5, ballots[1]), + committed(7, ballots[1]), + uncommitted(9, ballots[1])); + flush(tableData, updates); + assertIteratorContents(tableData.iterator(ALL_RANGES), updates); + + UncommittedTableData data2 = load(directory, CFID); + assertIteratorContents(data2.iterator(ALL_RANGES), updates); + } + + /** + * Test that incomplete update flushes are cleaned up + */ + @Test + public void updateRecovery() throws Throwable + { + Ballot[] ballots = createBallots(5); + UncommittedTableData tableData = load(directory, CFID); + + List updates = kl(uncommitted(3, ballots[1]), + uncommitted(5, ballots[1]), + committed(7, ballots[1]), + uncommitted(9, ballots[1])); + flush(tableData, updates); + assertIteratorContents(tableData.iterator(ALL_RANGES), updates); + MockDataFile tmpUpdate = mockFile(tableData.nextGeneration(), true); + + UncommittedTableData tableData2 = load(directory, CFID); + Assert.assertEquals(1, tableData2.nextGeneration()); + assertIteratorContents(tableData2.iterator(ALL_RANGES), updates); + Assert.assertTrue(tmpUpdate.isDeleted()); + } + + /** + * Test that incomplete state merges are cleaned up + */ + @Test + public void stateRecovery() throws Throwable + { + Ballot[] ballots = createBallots(5); + UncommittedTableData tableData = load(directory, CFID); + + List updates = kl(uncommitted(3, ballots[1]), + uncommitted(5, ballots[1]), + committed(7, ballots[1]), + uncommitted(9, ballots[1])); + flush(tableData, updates); + assertIteratorContents(tableData.iterator(ALL_RANGES), updates); + MockDataFile tmpUpdate = mockFile(tableData.nextGeneration(), true); + + UncommittedTableData tableData2 = load(directory, CFID); + Assert.assertEquals(1, tableData2.nextGeneration()); + assertIteratorContents(tableData2.iterator(ALL_RANGES), updates); + Assert.assertTrue(tmpUpdate.isDeleted()); + } + + @Test + public void orphanCrc() throws Throwable + { + Ballot[] ballots = createBallots(5); + UncommittedTableData tableData = load(directory, CFID); + + List updates = kl(uncommitted(3, ballots[1]), + uncommitted(5, ballots[1]), + uncommitted(7, ballots[1]), + uncommitted(9, ballots[1])); + flush(tableData, updates); + long updateGeneration = Iterables.getOnlyElement(tableData.data().files).generation(); + tableData.createMergeTask().run(); + assertIteratorContents(tableData.iterator(ALL_RANGES), updates); + + MockDataFile oldUpdate = mockFile(updateGeneration, false); + FileUtils.deleteWithConfirm(oldUpdate.data); + UncommittedTableData tableData2 = load(directory, CFID); + assertIteratorContents(tableData2.iterator(ALL_RANGES), updates); + Assert.assertTrue(oldUpdate.isDeleted()); + } + + @Test + public void referenceCountingTest() throws Throwable + { + Ballot[] ballots = createBallots(5); + UncommittedTableData tableData = load(directory, CFID); + flush(tableData, + uncommitted(3, ballots[1]), + committed(7, ballots[1])); + + // initial state + UncommittedDataFile updateFile = Iterables.getOnlyElement(tableData.data().files); + Assert.assertEquals(0, updateFile.getActiveReaders()); + Assert.assertFalse(updateFile.isMarkedDeleted()); + + // referenced state + CloseableIterator iterator = tableData.iterator(ALL_RANGES); + Assert.assertEquals(1, updateFile.getActiveReaders()); + Assert.assertFalse(updateFile.isMarkedDeleted()); + + // marked deleted state + tableData.createMergeTask().run(); + Assert.assertEquals(1, updateFile.getActiveReaders()); + Assert.assertTrue(updateFile.isMarkedDeleted()); + Assert.assertTrue(updateFile.file().exists()); + + // unreference and delete + iterator.close(); + Assert.assertEquals(0, updateFile.getActiveReaders()); + Assert.assertTrue(updateFile.isMarkedDeleted()); + Assert.assertFalse(updateFile.file().exists()); + } + + /** + * Test that we don't compact update sequences with gaps. ie: we shouldn't compact update generation 4 + * if we can't include generation 3 + */ + @Test + public void outOfOrderFlush() throws Throwable + { + Ballot[] ballots = createBallots(5); + UncommittedTableData tableData = load(directory, CFID); + FlushWriter pendingFlush = startFlush(tableData, + uncommitted(3, ballots[1]), + committed(7, ballots[1])); + Assert.assertNull(tableData.currentMerge()); + + flush(tableData, + uncommitted(5, ballots[1]), + uncommitted(9, ballots[1])); + + // schedule a merge + Merge merge = tableData.createMergeTask(); + Assert.assertFalse(!merge.dependsOnActiveFlushes()); + Assert.assertFalse(merge.isScheduled); + + // completing the first flush should cause the merge to be scheduled + pendingFlush.finish(); + Assert.assertTrue(!merge.dependsOnActiveFlushes()); + Assert.assertTrue(merge.isScheduled); + + while (tableData.currentMerge() != null) + Thread.sleep(1); + + // confirm that the merge has completed + Assert.assertEquals(3, tableData.nextGeneration()); + Data data = tableData.data(); + Assert.assertEquals(2, Iterables.getOnlyElement(data.files).generation()); + assertIteratorContents(tableData.iterator(ALL_RANGES), kl(uncommitted(3, ballots[1]), + uncommitted(5, ballots[1]), + uncommitted(9, ballots[1]))); + } + + @Test + public void abortedFlush() throws Throwable + { + Ballot[] ballots = createBallots(5); + UncommittedTableData tableData = load(directory, CFID); + FlushWriter pendingFlush = startFlush(tableData, + uncommitted(3, ballots[1]), + committed(7, ballots[1])); + Assert.assertNull(tableData.currentMerge()); + + List secondFlushUpdates = kl(uncommitted(5, ballots[1]), + uncommitted(9, ballots[1])); + flush(tableData, secondFlushUpdates); + tableData.createMergeTask(); + + // the second flush should have triggered a merge + Merge merge = tableData.currentMerge(); + Assert.assertFalse(!merge.dependsOnActiveFlushes()); + Assert.assertFalse(merge.isScheduled); + + // completing the first merge should cause the merge to be scheduled + pendingFlush.abort(null); + Assert.assertTrue(!merge.dependsOnActiveFlushes()); + Assert.assertTrue(merge.isScheduled); + + while (tableData.currentMerge() != null) + Thread.sleep(1); + + // confirm that the merge has completed + Assert.assertEquals(3, tableData.nextGeneration()); + Data data = tableData.data(); + Assert.assertEquals(2, Iterables.getOnlyElement(data.files).generation()); + assertIteratorContents(tableData.iterator(ALL_RANGES), secondFlushUpdates); + } + + /** + * keys that aren't locally replicated shouldn't be written on merge + */ + @Test + public void rangePurge() throws Throwable + { + Ballot[] ballots = createBallots(5); + UncommittedTableData tableData = UncommittedTableData.load(directory, CFID, new FilterFactory() { + List> getReplicatedRanges() + { + return Lists.newArrayList(new Range<>(tk(4), tk(7))); + } + + PaxosRepairHistory getPaxosRepairHistory() + { + return PaxosRepairHistory.EMPTY; + } + }); + + flush(tableData, uncommitted(3, ballots[1]), + uncommitted(5, ballots[1]), + uncommitted(7, ballots[1]), + uncommitted(9, ballots[1])); + tableData.createMergeTask().run(); + assertIteratorContents(tableData.iterator(ALL_RANGES), kl(uncommitted(5, ballots[1]), + uncommitted(7, ballots[1]))); + } + + @Test + public void rangePurge2() throws Throwable + { + Ballot[] ballots = createBallots(5); + UncommittedTableData tableData = UncommittedTableData.load(directory, CFID, new FilterFactory() { + List> getReplicatedRanges() + { + return Lists.newArrayList(new Range<>(tk(4), tk(6)), new Range(tk(8), tk(10))); + } + + PaxosRepairHistory getPaxosRepairHistory() + { + return PaxosRepairHistory.EMPTY; + } + }); + + flush(tableData, uncommitted(3, ballots[1]), + uncommitted(5, ballots[1]), + uncommitted(7, ballots[1]), + uncommitted(9, ballots[1])); + tableData.createMergeTask().run(); + assertIteratorContents(tableData.iterator(ALL_RANGES), kl(uncommitted(5, ballots[1]), + uncommitted(9, ballots[1]))); + } + + /** + * ballots below the low bound should be purged + */ + @Test + public void lowBoundPurge() throws Throwable + { + Ballot[] ballots = createBallots(5); + UncommittedTableData tableData = UncommittedTableData.load(directory, CFID, new FilterFactory() { + List> getReplicatedRanges() + { + return Lists.newArrayList(ALL_RANGES); + } + + PaxosRepairHistory getPaxosRepairHistory() + { + return PaxosRepairHistory.add(PaxosRepairHistory.EMPTY, ALL_RANGES, ballots[1]); + } + }); + + flush(tableData, uncommitted(3, ballots[0]), + uncommitted(5, ballots[1]), + uncommitted(7, ballots[2])); + tableData.createMergeTask().run(); + assertIteratorContents(tableData.iterator(ALL_RANGES), kl(uncommitted(5, ballots[1]), + uncommitted(7, ballots[2]))); + } +} diff --git a/test/unit/org/apache/cassandra/service/reads/repair/AbstractReadRepairTest.java b/test/unit/org/apache/cassandra/service/reads/repair/AbstractReadRepairTest.java index 7587993d3e..0ce1b90e8c 100644 --- a/test/unit/org/apache/cassandra/service/reads/repair/AbstractReadRepairTest.java +++ b/test/unit/org/apache/cassandra/service/reads/repair/AbstractReadRepairTest.java @@ -97,7 +97,7 @@ public abstract class AbstractReadRepairTest static Replica replica2; static Replica replica3; static EndpointsForRange replicas; - static ReplicaPlan.ForRead replicaPlan; + static ReplicaPlan.ForRead replicaPlan; static long now = TimeUnit.NANOSECONDS.toMicros(nanoTime()); static DecoratedKey key; @@ -283,17 +283,17 @@ public abstract class AbstractReadRepairTest return replicaPlan(ks, consistencyLevel, replicas, replicas); } - static ReplicaPlan.ForTokenWrite repairPlan(ReplicaPlan.ForRangeRead readPlan) + static ReplicaPlan.ForWrite repairPlan(ReplicaPlan.ForRangeRead readPlan) { - return repairPlan(readPlan, readPlan.candidates()); + return repairPlan(readPlan, readPlan.readCandidates()); } - static ReplicaPlan.ForTokenWrite repairPlan(EndpointsForRange liveAndDown, EndpointsForRange targets) + static ReplicaPlan.ForWrite repairPlan(EndpointsForRange liveAndDown, EndpointsForRange targets) { return repairPlan(replicaPlan(liveAndDown, targets), liveAndDown); } - static ReplicaPlan.ForTokenWrite repairPlan(ReplicaPlan.ForRangeRead readPlan, EndpointsForRange liveAndDown) + static ReplicaPlan.ForWrite repairPlan(ReplicaPlan.ForRangeRead readPlan, EndpointsForRange liveAndDown) { Token token = readPlan.range().left.getToken(); EndpointsForToken pending = EndpointsForToken.empty(token); diff --git a/test/unit/org/apache/cassandra/service/reads/repair/BlockingReadRepairTest.java b/test/unit/org/apache/cassandra/service/reads/repair/BlockingReadRepairTest.java index 0666eb191c..3a938a29ec 100644 --- a/test/unit/org/apache/cassandra/service/reads/repair/BlockingReadRepairTest.java +++ b/test/unit/org/apache/cassandra/service/reads/repair/BlockingReadRepairTest.java @@ -51,7 +51,7 @@ public class BlockingReadRepairTest extends AbstractReadRepairTest private static class InstrumentedReadRepairHandler extends BlockingPartitionRepair { - public InstrumentedReadRepairHandler(Map repairs, ReplicaPlan.ForTokenWrite writePlan) + public InstrumentedReadRepairHandler(Map repairs, ReplicaPlan.ForWrite writePlan) { super(Util.dk("not a real usable value"), repairs, writePlan, e -> targets.contains(e)); } @@ -70,7 +70,7 @@ public class BlockingReadRepairTest extends AbstractReadRepairTest configureClass(ReadRepairStrategy.BLOCKING); } - private static InstrumentedReadRepairHandler createRepairHandler(Map repairs, ReplicaPlan.ForTokenWrite writePlan) + private static InstrumentedReadRepairHandler createRepairHandler(Map repairs, ReplicaPlan.ForWrite writePlan) { return new InstrumentedReadRepairHandler(repairs, writePlan); } @@ -81,7 +81,7 @@ public class BlockingReadRepairTest extends AbstractReadRepairTest return createRepairHandler(repairs, repairPlan(replicas, replicas)); } - private static class InstrumentedBlockingReadRepair, P extends ReplicaPlan.ForRead> + private static class InstrumentedBlockingReadRepair, P extends ReplicaPlan.ForRead> extends BlockingReadRepair implements InstrumentedReadRepair { public InstrumentedBlockingReadRepair(ReadCommand command, ReplicaPlan.Shared replicaPlan, long queryStartNanoTime) @@ -143,7 +143,7 @@ public class BlockingReadRepairTest extends AbstractReadRepairTest repairs.put(replica1, repair1); repairs.put(replica2, repair2); - ReplicaPlan.ForTokenWrite writePlan = repairPlan(replicas, EndpointsForRange.copyOf(Lists.newArrayList(repairs.keySet()))); + ReplicaPlan.ForWrite writePlan = repairPlan(replicas, EndpointsForRange.copyOf(Lists.newArrayList(repairs.keySet()))); InstrumentedReadRepairHandler handler = createRepairHandler(repairs, writePlan); Assert.assertTrue(handler.mutationsSent.isEmpty()); @@ -269,7 +269,7 @@ public class BlockingReadRepairTest extends AbstractReadRepairTest repairs.put(remote1, mutation(cell1)); EndpointsForRange participants = EndpointsForRange.of(replica1, replica2, remote1, remote2); - ReplicaPlan.ForTokenWrite writePlan = repairPlan(replicaPlan(ks, ConsistencyLevel.LOCAL_QUORUM, participants)); + ReplicaPlan.ForWrite writePlan = repairPlan(replicaPlan(ks, ConsistencyLevel.LOCAL_QUORUM, participants)); InstrumentedReadRepairHandler handler = createRepairHandler(repairs, writePlan); handler.sendInitialRepairs(); Assert.assertEquals(2, handler.mutationsSent.size()); diff --git a/test/unit/org/apache/cassandra/service/reads/repair/DiagEventsBlockingReadRepairTest.java b/test/unit/org/apache/cassandra/service/reads/repair/DiagEventsBlockingReadRepairTest.java index 713cef6122..8399c838bc 100644 --- a/test/unit/org/apache/cassandra/service/reads/repair/DiagEventsBlockingReadRepairTest.java +++ b/test/unit/org/apache/cassandra/service/reads/repair/DiagEventsBlockingReadRepairTest.java @@ -85,7 +85,7 @@ public class DiagEventsBlockingReadRepairTest extends AbstractReadRepairTest repairs.put(replica2, repair2); - ReplicaPlan.ForTokenWrite writePlan = repairPlan(replicas, EndpointsForRange.copyOf(Lists.newArrayList(repairs.keySet()))); + ReplicaPlan.ForWrite writePlan = repairPlan(replicas, EndpointsForRange.copyOf(Lists.newArrayList(repairs.keySet()))); DiagnosticPartitionReadRepairHandler handler = createRepairHandler(repairs, writePlan); Assert.assertTrue(handler.updatesByEp.isEmpty()); @@ -121,7 +121,7 @@ public class DiagEventsBlockingReadRepairTest extends AbstractReadRepairTest return new DiagnosticBlockingRepairHandler(command, replicaPlan, queryStartNanoTime); } - private static DiagnosticPartitionReadRepairHandler createRepairHandler(Map repairs, ReplicaPlan.ForTokenWrite writePlan) + private static DiagnosticPartitionReadRepairHandler createRepairHandler(Map repairs, ReplicaPlan.ForWrite writePlan) { return new DiagnosticPartitionReadRepairHandler(key, repairs, writePlan); } @@ -168,7 +168,7 @@ public class DiagEventsBlockingReadRepairTest extends AbstractReadRepairTest } } - private static class DiagnosticPartitionReadRepairHandler, P extends ReplicaPlan.ForRead> + private static class DiagnosticPartitionReadRepairHandler, P extends ReplicaPlan.ForRead> extends BlockingPartitionRepair { private final Map updatesByEp = new HashMap<>(); @@ -179,7 +179,7 @@ public class DiagEventsBlockingReadRepairTest extends AbstractReadRepairTest return e -> candidates.contains(e); } - DiagnosticPartitionReadRepairHandler(DecoratedKey key, Map repairs, ReplicaPlan.ForTokenWrite writePlan) + DiagnosticPartitionReadRepairHandler(DecoratedKey key, Map repairs, ReplicaPlan.ForWrite writePlan) { super(key, repairs, writePlan, isLocal()); DiagnosticEventService.instance().subscribe(PartitionRepairEvent.class, this::onRepairEvent); diff --git a/test/unit/org/apache/cassandra/service/reads/repair/InstrumentedReadRepair.java b/test/unit/org/apache/cassandra/service/reads/repair/InstrumentedReadRepair.java index 721b6f5e4e..f9bbea94a2 100644 --- a/test/unit/org/apache/cassandra/service/reads/repair/InstrumentedReadRepair.java +++ b/test/unit/org/apache/cassandra/service/reads/repair/InstrumentedReadRepair.java @@ -25,10 +25,10 @@ import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.locator.ReplicaPlan; import org.apache.cassandra.service.reads.ReadCallback; -public interface InstrumentedReadRepair, P extends ReplicaPlan.ForRead> +public interface InstrumentedReadRepair, P extends ReplicaPlan.ForRead> extends ReadRepair { Set getReadRecipients(); - ReadCallback getReadCallback(); + ReadCallback getReadCallback(); } diff --git a/test/unit/org/apache/cassandra/service/reads/repair/ReadOnlyReadRepairTest.java b/test/unit/org/apache/cassandra/service/reads/repair/ReadOnlyReadRepairTest.java index 4dfe2bc09e..82bb8de14d 100644 --- a/test/unit/org/apache/cassandra/service/reads/repair/ReadOnlyReadRepairTest.java +++ b/test/unit/org/apache/cassandra/service/reads/repair/ReadOnlyReadRepairTest.java @@ -36,7 +36,7 @@ import org.apache.cassandra.service.reads.ReadCallback; public class ReadOnlyReadRepairTest extends AbstractReadRepairTest { - private static class InstrumentedReadOnlyReadRepair, P extends ReplicaPlan.ForRead> + private static class InstrumentedReadOnlyReadRepair, P extends ReplicaPlan.ForRead> extends ReadOnlyReadRepair implements InstrumentedReadRepair { public InstrumentedReadOnlyReadRepair(ReadCommand command, ReplicaPlan.Shared replicaPlan, long queryStartNanoTime) @@ -92,7 +92,7 @@ public class ReadOnlyReadRepairTest extends AbstractReadRepairTest public void repairPartitionFailure() { ReplicaPlan.SharedForRangeRead readPlan = ReplicaPlan.shared(replicaPlan(replicas, replicas)); - ReplicaPlan.ForTokenWrite writePlan = repairPlan(replicas, replicas); + ReplicaPlan.ForWrite writePlan = repairPlan(replicas, replicas); InstrumentedReadRepair repair = createInstrumentedReadRepair(readPlan); repair.repairPartition(null, Collections.emptyMap(), writePlan); } diff --git a/test/unit/org/apache/cassandra/service/reads/repair/ReadRepairTest.java b/test/unit/org/apache/cassandra/service/reads/repair/ReadRepairTest.java index 82de4f3dce..a6b6918f43 100644 --- a/test/unit/org/apache/cassandra/service/reads/repair/ReadRepairTest.java +++ b/test/unit/org/apache/cassandra/service/reads/repair/ReadRepairTest.java @@ -72,10 +72,10 @@ public class ReadRepairTest static Replica target3; static EndpointsForRange targets; - private static class InstrumentedReadRepairHandler, P extends ReplicaPlan.ForRead> + private static class InstrumentedReadRepairHandler, P extends ReplicaPlan.ForRead> extends BlockingPartitionRepair { - public InstrumentedReadRepairHandler(Map repairs, ReplicaPlan.ForTokenWrite writePlan) + public InstrumentedReadRepairHandler(Map repairs, ReplicaPlan.ForWrite writePlan) { super(Util.dk("not a valid key"), repairs, writePlan, e -> targets.endpoints().contains(e)); } @@ -164,7 +164,7 @@ public class ReadRepairTest private static InstrumentedReadRepairHandler createRepairHandler(Map repairs, EndpointsForRange all, EndpointsForRange targets) { ReplicaPlan.ForRangeRead readPlan = AbstractReadRepairTest.replicaPlan(ks, ConsistencyLevel.LOCAL_QUORUM, all, targets); - ReplicaPlan.ForTokenWrite writePlan = AbstractReadRepairTest.repairPlan(readPlan); + ReplicaPlan.ForWrite writePlan = AbstractReadRepairTest.repairPlan(readPlan); return new InstrumentedReadRepairHandler(repairs, writePlan); } diff --git a/test/unit/org/apache/cassandra/service/reads/repair/TestableReadRepair.java b/test/unit/org/apache/cassandra/service/reads/repair/TestableReadRepair.java index 2529a71ad0..eecd106e06 100644 --- a/test/unit/org/apache/cassandra/service/reads/repair/TestableReadRepair.java +++ b/test/unit/org/apache/cassandra/service/reads/repair/TestableReadRepair.java @@ -37,7 +37,7 @@ import org.apache.cassandra.locator.Replica; import org.apache.cassandra.locator.ReplicaPlan; import org.apache.cassandra.service.reads.DigestResolver; -public class TestableReadRepair, P extends ReplicaPlan.ForRead> +public class TestableReadRepair, P extends ReplicaPlan.ForRead> implements ReadRepair { public final Map sent = new HashMap<>(); @@ -111,7 +111,7 @@ public class TestableReadRepair, P extends ReplicaPlan.Fo } @Override - public void repairPartition(DecoratedKey partitionKey, Map mutations, ReplicaPlan.ForTokenWrite writePlan) + public void repairPartition(DecoratedKey partitionKey, Map mutations, ReplicaPlan.ForWrite writePlan) { for (Map.Entry entry: mutations.entrySet()) sent.put(entry.getKey().endpoint(), entry.getValue()); diff --git a/test/unit/org/apache/cassandra/transport/ErrorMessageTest.java b/test/unit/org/apache/cassandra/transport/ErrorMessageTest.java index cfeddbae05..c0e3f5ed91 100644 --- a/test/unit/org/apache/cassandra/transport/ErrorMessageTest.java +++ b/test/unit/org/apache/cassandra/transport/ErrorMessageTest.java @@ -101,7 +101,7 @@ public class ErrorMessageTest extends EncodeAndDecodeTestBase int contentions = 1; int receivedBlockFor = 3; ConsistencyLevel consistencyLevel = ConsistencyLevel.SERIAL; - CasWriteTimeoutException ex = new CasWriteTimeoutException(WriteType.CAS, consistencyLevel, receivedBlockFor, receivedBlockFor, contentions); + CasWriteTimeoutException ex = new CasWriteTimeoutException(WriteType.CAS, consistencyLevel, 0, receivedBlockFor, contentions); ErrorMessage deserialized = encodeThenDecode(ErrorMessage.fromException(ex), ProtocolVersion.V5); assertTrue(deserialized.error instanceof CasWriteTimeoutException); @@ -110,10 +110,10 @@ public class ErrorMessageTest extends EncodeAndDecodeTestBase assertEquals(WriteType.CAS, deserializedEx.writeType); assertEquals(contentions, deserializedEx.contentions); assertEquals(consistencyLevel, deserializedEx.consistency); - assertEquals(receivedBlockFor, deserializedEx.received); + assertEquals(0, deserializedEx.received); assertEquals(receivedBlockFor, deserializedEx.blockFor); assertEquals(ex.getMessage(), deserializedEx.getMessage()); - assertTrue(deserializedEx.getMessage().contains("CAS operation timed out - encountered contentions")); + assertTrue(deserializedEx.getMessage().contains("CAS operation timed out: received 0 of 3 required responses after 1 contention retries")); } @Test diff --git a/test/unit/org/apache/cassandra/utils/CassandraVersionTest.java b/test/unit/org/apache/cassandra/utils/CassandraVersionTest.java index 21c7430c47..6f344997f1 100644 --- a/test/unit/org/apache/cassandra/utils/CassandraVersionTest.java +++ b/test/unit/org/apache/cassandra/utils/CassandraVersionTest.java @@ -252,6 +252,16 @@ public class CassandraVersionTest v2 = new CassandraVersion("4.0"); assertTrue(v1.compareTo(v2) < 0); assertTrue(v2.compareTo(v1) > 0); + + assertEquals(-1, v1.compareTo(v2)); + + v1 = new CassandraVersion("1.2.3"); + v2 = new CassandraVersion("1.2.3.1"); + assertEquals(-1, v1.compareTo(v2)); + + v1 = new CassandraVersion("1.2.3.1"); + v2 = new CassandraVersion("1.2.3.2"); + assertEquals(-1, v1.compareTo(v2)); } @Test @@ -346,7 +356,7 @@ public class CassandraVersionTest @Test public void testParseIdentifiersPositive() throws Throwable { - String[] result = parseIdentifiers("DUMMY", "+a.b.cde.f_g."); + String[] result = parseIdentifiers("DUMMY", "a.b.cde.f_g."); String[] expected = {"a", "b", "cde", "f_g"}; assertArrayEquals(expected, result); } diff --git a/test/unit/org/apache/cassandra/utils/concurrent/AbstractTestAsyncPromise.java b/test/unit/org/apache/cassandra/utils/concurrent/AbstractTestAsyncPromise.java index 0c297ecd66..51a07fbfa6 100644 --- a/test/unit/org/apache/cassandra/utils/concurrent/AbstractTestAsyncPromise.java +++ b/test/unit/org/apache/cassandra/utils/concurrent/AbstractTestAsyncPromise.java @@ -37,6 +37,10 @@ import org.junit.Assert; import io.netty.util.concurrent.GenericFutureListener; import org.apache.cassandra.config.DatabaseDescriptor; +import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static java.util.concurrent.TimeUnit.SECONDS; +import static org.apache.cassandra.utils.Clock.Global.nanoTime; + public abstract class AbstractTestAsyncPromise extends AbstractTestPromise { static @@ -222,12 +226,12 @@ public abstract class AbstractTestAsyncPromise extends AbstractTestPromise success(promise, Promise::isDone, false); success(promise, Promise::isCancelled, false); async.success(promise, Promise::get, value); - async.success(promise, p -> p.get(1L, TimeUnit.SECONDS), value); + async.success(promise, p -> p.get(1L, SECONDS), value); async.success(promise, Promise::await, promise); async.success(promise, Promise::awaitUninterruptibly, promise); - async.success(promise, p -> p.await(1L, TimeUnit.SECONDS), true); + async.success(promise, p -> p.await(1L, SECONDS), true); async.success(promise, p -> p.await(1000L), true); - async.success(promise, p -> p.awaitUninterruptibly(1L, TimeUnit.SECONDS), true); + async.success(promise, p -> p.awaitUninterruptibly(1L, SECONDS), true); async.success(promise, p -> p.awaitUninterruptibly(1000L), true); async.success(promise, Promise::sync, promise); async.success(promise, Promise::syncUninterruptibly, promise); @@ -389,18 +393,18 @@ public abstract class AbstractTestAsyncPromise extends AbstractTestPromise success(promise, Promise::getNow, null); success(promise, Promise::cause, null); async.failure(promise, p -> p.get(), ExecutionException.class); - async.failure(promise, p -> p.get(1L, TimeUnit.SECONDS), ExecutionException.class); + async.failure(promise, p -> p.get(1L, SECONDS), ExecutionException.class); async.success(promise, Promise::await, promise); async.success(promise, Promise::awaitThrowUncheckedOnInterrupt, promise); async.success(promise, Promise::awaitUninterruptibly, promise); - async.success(promise, p -> p.await(1L, TimeUnit.SECONDS), true); + async.success(promise, p -> p.await(1L, SECONDS), true); async.success(promise, p -> p.await(1000L), true); - async.success(promise, p -> p.awaitUninterruptibly(1L, TimeUnit.SECONDS), true); - async.success(promise, p -> p.awaitThrowUncheckedOnInterrupt(1L, TimeUnit.SECONDS), true); + async.success(promise, p -> p.awaitUninterruptibly(1L, SECONDS), true); + async.success(promise, p -> p.awaitThrowUncheckedOnInterrupt(1L, SECONDS), true); async.success(promise, p -> p.awaitUninterruptibly(1000L), true); - async.success(promise, p -> p.awaitUntil(System.nanoTime() + TimeUnit.SECONDS.toNanos(1L)), true); - async.success(promise, p -> p.awaitUntilUninterruptibly(System.nanoTime() + TimeUnit.SECONDS.toNanos(1L)), true); - async.success(promise, p -> p.awaitUntilThrowUncheckedOnInterrupt(System.nanoTime() + TimeUnit.SECONDS.toNanos(1L)), true); + async.success(promise, p -> p.awaitUntil(nanoTime() + SECONDS.toNanos(1L)), true); + async.success(promise, p -> p.awaitUntilUninterruptibly(nanoTime() + SECONDS.toNanos(1L)), true); + async.success(promise, p -> p.awaitUntilThrowUncheckedOnInterrupt(nanoTime() + SECONDS.toNanos(1L)), true); async.failure(promise, p -> p.sync(), cause); async.failure(promise, p -> p.syncUninterruptibly(), cause); if (tryOrSet) promise.tryFailure(cause); @@ -449,18 +453,18 @@ public abstract class AbstractTestAsyncPromise extends AbstractTestPromise success(promise, Promise::getNow, null); success(promise, Promise::cause, null); async.failure(promise, p -> p.get(), CancellationException.class); - async.failure(promise, p -> p.get(1L, TimeUnit.SECONDS), CancellationException.class); + async.failure(promise, p -> p.get(1L, SECONDS), CancellationException.class); async.success(promise, Promise::await, promise); async.success(promise, Promise::awaitThrowUncheckedOnInterrupt, promise); async.success(promise, Promise::awaitUninterruptibly, promise); - async.success(promise, p -> p.await(1L, TimeUnit.SECONDS), true); + async.success(promise, p -> p.await(1L, SECONDS), true); async.success(promise, p -> p.await(1000L), true); - async.success(promise, p -> p.awaitUninterruptibly(1L, TimeUnit.SECONDS), true); - async.success(promise, p -> p.awaitThrowUncheckedOnInterrupt(1L, TimeUnit.SECONDS), true); + async.success(promise, p -> p.awaitUninterruptibly(1L, SECONDS), true); + async.success(promise, p -> p.awaitThrowUncheckedOnInterrupt(1L, SECONDS), true); async.success(promise, p -> p.awaitUninterruptibly(1000L), true); - async.success(promise, p -> p.awaitUntil(System.nanoTime() + TimeUnit.SECONDS.toNanos(1L)), true); - async.success(promise, p -> p.awaitUntilUninterruptibly(System.nanoTime() + TimeUnit.SECONDS.toNanos(1L)), true); - async.success(promise, p -> p.awaitUntilThrowUncheckedOnInterrupt(System.nanoTime() + TimeUnit.SECONDS.toNanos(1L)), true); + async.success(promise, p -> p.awaitUntil(nanoTime() + SECONDS.toNanos(1L)), true); + async.success(promise, p -> p.awaitUntilUninterruptibly(nanoTime() + SECONDS.toNanos(1L)), true); + async.success(promise, p -> p.awaitUntilThrowUncheckedOnInterrupt(nanoTime() + SECONDS.toNanos(1L)), true); async.failure(promise, p -> p.sync(), CancellationException.class); async.failure(promise, p -> p.syncUninterruptibly(), CancellationException.class); promise.cancel(interruptIfRunning); @@ -482,17 +486,17 @@ public abstract class AbstractTestAsyncPromise extends AbstractTestPromise public void testOneTimeout(Promise promise) { Async async = new Async(); - async.failure(promise, p -> p.get(1L, TimeUnit.MILLISECONDS), TimeoutException.class); - async.success(promise, p -> p.await(1L, TimeUnit.MILLISECONDS), false); - async.success(promise, p -> p.awaitThrowUncheckedOnInterrupt(1L, TimeUnit.MILLISECONDS), false); + async.failure(promise, p -> p.get(1L, MILLISECONDS), TimeoutException.class); + async.success(promise, p -> p.await(1L, MILLISECONDS), false); + async.success(promise, p -> p.awaitThrowUncheckedOnInterrupt(1L, MILLISECONDS), false); async.success(promise, p -> p.await(1L), false); - async.success(promise, p -> p.awaitUninterruptibly(1L, TimeUnit.MILLISECONDS), false); - async.success(promise, p -> p.awaitThrowUncheckedOnInterrupt(1L, TimeUnit.MILLISECONDS), false); + async.success(promise, p -> p.awaitUninterruptibly(1L, MILLISECONDS), false); + async.success(promise, p -> p.awaitThrowUncheckedOnInterrupt(1L, MILLISECONDS), false); async.success(promise, p -> p.awaitUninterruptibly(1L), false); - async.success(promise, p -> p.awaitUntil(System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(1L)), false); - async.success(promise, p -> p.awaitUntilUninterruptibly(System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(1L)), false); - async.success(promise, p -> p.awaitUntilThrowUncheckedOnInterrupt(System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(1L)), false); - Uninterruptibles.sleepUninterruptibly(10L, TimeUnit.MILLISECONDS); + async.success(promise, p -> p.awaitUntil(nanoTime() + MILLISECONDS.toNanos(1L)), false); + async.success(promise, p -> p.awaitUntilUninterruptibly(nanoTime() + MILLISECONDS.toNanos(1L)), false); + async.success(promise, p -> p.awaitUntilThrowUncheckedOnInterrupt(nanoTime() + MILLISECONDS.toNanos(1L)), false); + Uninterruptibles.sleepUninterruptibly(10L, MILLISECONDS); async.verify(); } diff --git a/test/unit/org/apache/cassandra/utils/concurrent/AbstractTestAwaitable.java b/test/unit/org/apache/cassandra/utils/concurrent/AbstractTestAwaitable.java index f0790349bb..8b95eaf52e 100644 --- a/test/unit/org/apache/cassandra/utils/concurrent/AbstractTestAwaitable.java +++ b/test/unit/org/apache/cassandra/utils/concurrent/AbstractTestAwaitable.java @@ -34,6 +34,10 @@ import net.openhft.chronicle.core.util.ThrowingBiConsumer; import net.openhft.chronicle.core.util.ThrowingConsumer; import net.openhft.chronicle.core.util.ThrowingFunction; +import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static java.util.concurrent.TimeUnit.SECONDS; +import static org.apache.cassandra.utils.Clock.Global.nanoTime; + public abstract class AbstractTestAwaitable { protected final ExecutorService exec = Executors.newCachedThreadPool(); @@ -45,9 +49,9 @@ public abstract class AbstractTestAwaitable async.success(awaitable, a -> a.await(), awaitable); async.success(awaitable, a -> a.awaitUninterruptibly(), awaitable); async.success(awaitable, a -> a.awaitThrowUncheckedOnInterrupt(), awaitable); - async.success(awaitable, a -> a.await(1L, TimeUnit.SECONDS), true); - async.success(awaitable, a -> a.awaitUninterruptibly(1L, TimeUnit.SECONDS), true); - async.success(awaitable, a -> a.awaitThrowUncheckedOnInterrupt(1L, TimeUnit.SECONDS), true); + async.success(awaitable, a -> a.await(1L, SECONDS), true); + async.success(awaitable, a -> a.awaitUninterruptibly(1L, SECONDS), true); + async.success(awaitable, a -> a.awaitThrowUncheckedOnInterrupt(1L, SECONDS), true); async.success(awaitable, a -> a.awaitUntil(Long.MAX_VALUE), true); async.success(awaitable, a -> a.awaitUntilUninterruptibly(Long.MAX_VALUE), true); async.success(awaitable, a -> a.awaitUntilThrowUncheckedOnInterrupt(Long.MAX_VALUE), true); @@ -58,13 +62,13 @@ public abstract class AbstractTestAwaitable public void testOneTimeout(A awaitable) { Async async = new Async(); - async.success(awaitable, a -> a.await(1L, TimeUnit.MILLISECONDS), false); - async.success(awaitable, a -> a.awaitUninterruptibly(1L, TimeUnit.MILLISECONDS), false); - async.success(awaitable, a -> a.awaitThrowUncheckedOnInterrupt(1L, TimeUnit.MILLISECONDS), false); - async.success(awaitable, a -> a.awaitUntil(System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(1L)), false); - async.success(awaitable, a -> a.awaitUntilUninterruptibly(System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(1L)), false); - async.success(awaitable, a -> a.awaitUntilThrowUncheckedOnInterrupt(System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(1L)), false); - Uninterruptibles.sleepUninterruptibly(10L, TimeUnit.MILLISECONDS); + async.success(awaitable, a -> a.await(1L, MILLISECONDS), false); + async.success(awaitable, a -> a.awaitUninterruptibly(1L, MILLISECONDS), false); + async.success(awaitable, a -> a.awaitThrowUncheckedOnInterrupt(1L, MILLISECONDS), false); + async.success(awaitable, a -> a.awaitUntil(nanoTime() + MILLISECONDS.toNanos(1L)), false); + async.success(awaitable, a -> a.awaitUntilUninterruptibly(nanoTime() + MILLISECONDS.toNanos(1L)), false); + async.success(awaitable, a -> a.awaitUntilThrowUncheckedOnInterrupt(nanoTime() + MILLISECONDS.toNanos(1L)), false); + Uninterruptibles.sleepUninterruptibly(10L, MILLISECONDS); async.verify(); } @@ -72,13 +76,13 @@ public abstract class AbstractTestAwaitable { Async async = new Async(); async.failure(awaitable, a -> { Thread.currentThread().interrupt(); a.await(); }, InterruptedException.class); - async.failure(awaitable, a -> { Thread.currentThread().interrupt(); a.await(1L, TimeUnit.SECONDS); }, InterruptedException.class); - async.success(awaitable, a -> { Thread.currentThread().interrupt(); return a.awaitUninterruptibly(1L, TimeUnit.SECONDS); }, false); - async.failure(awaitable, a -> { Thread.currentThread().interrupt(); a.awaitThrowUncheckedOnInterrupt(1L, TimeUnit.SECONDS); }, UncheckedInterruptedException.class); - async.failure(awaitable, a -> { Thread.currentThread().interrupt(); a.awaitUntil(System.nanoTime() + TimeUnit.SECONDS.toNanos(1L)); }, InterruptedException.class); - async.success(awaitable, a -> { Thread.currentThread().interrupt(); return a.awaitUntilUninterruptibly(System.nanoTime() + TimeUnit.SECONDS.toNanos(1L)); }, false); - async.failure(awaitable, a -> { Thread.currentThread().interrupt(); a.awaitUntilThrowUncheckedOnInterrupt(System.nanoTime() + TimeUnit.SECONDS.toNanos(1L)); }, UncheckedInterruptedException.class); - Uninterruptibles.sleepUninterruptibly(2L, TimeUnit.SECONDS); + async.failure(awaitable, a -> { Thread.currentThread().interrupt(); a.await(1L, SECONDS); }, InterruptedException.class); + async.success(awaitable, a -> { Thread.currentThread().interrupt(); return a.awaitUninterruptibly(1L, SECONDS); }, false); + async.failure(awaitable, a -> { Thread.currentThread().interrupt(); a.awaitThrowUncheckedOnInterrupt(1L, SECONDS); }, UncheckedInterruptedException.class); + async.failure(awaitable, a -> { Thread.currentThread().interrupt(); a.awaitUntil(nanoTime() + SECONDS.toNanos(1L)); }, InterruptedException.class); + async.success(awaitable, a -> { Thread.currentThread().interrupt(); return a.awaitUntilUninterruptibly(nanoTime() + SECONDS.toNanos(1L)); }, false); + async.failure(awaitable, a -> { Thread.currentThread().interrupt(); a.awaitUntilThrowUncheckedOnInterrupt(nanoTime() + SECONDS.toNanos(1L)); }, UncheckedInterruptedException.class); + Uninterruptibles.sleepUninterruptibly(2L, SECONDS); async.verify(); } @@ -91,7 +95,7 @@ public abstract class AbstractTestAwaitable { try { - waitingOn.get(i).accept(100L, TimeUnit.MILLISECONDS); + waitingOn.get(i).accept(100L, MILLISECONDS); } catch (Throwable t) { diff --git a/test/unit/org/apache/cassandra/utils/concurrent/SemaphoreTest.java b/test/unit/org/apache/cassandra/utils/concurrent/SemaphoreTest.java index c4cb86d90a..d52598b772 100644 --- a/test/unit/org/apache/cassandra/utils/concurrent/SemaphoreTest.java +++ b/test/unit/org/apache/cassandra/utils/concurrent/SemaphoreTest.java @@ -24,12 +24,15 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import org.junit.Assert; import org.junit.Test; +import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static java.util.concurrent.TimeUnit.MINUTES; +import static org.apache.cassandra.utils.Clock.Global.nanoTime; + public class SemaphoreTest { @@ -57,11 +60,11 @@ public class SemaphoreTest Semaphore s = Semaphore.newFairSemaphore(2); List> fs = start(s); s.release(1); - fs.get(0).get(1L, TimeUnit.MINUTES); + fs.get(0).get(1L, MINUTES); s.release(1); - fs.get(1).get(1L, TimeUnit.MINUTES); + fs.get(1).get(1L, MINUTES); s.release(1); - fs.get(2).get(1L, TimeUnit.MINUTES); + fs.get(2).get(1L, MINUTES); s.release(1); Assert.assertEquals(1, s.permits()); } @@ -74,17 +77,17 @@ public class SemaphoreTest Assert.assertTrue(s.tryAcquire(1)); s.drain(); Assert.assertFalse(s.tryAcquire(1)); - Assert.assertFalse(s.tryAcquire(1, 1L, TimeUnit.MILLISECONDS)); + Assert.assertFalse(s.tryAcquire(1, 1L, MILLISECONDS)); Thread.currentThread().interrupt(); try { s.acquireThrowUncheckedOnInterrupt(1); Assert.fail(); } catch (UncheckedInterruptedException ignore) { } Thread.currentThread().interrupt(); - try { s.tryAcquire(1, 1L, TimeUnit.MILLISECONDS); Assert.fail(); } catch (InterruptedException ignore) { } + try { s.tryAcquire(1, 1L, MILLISECONDS); Assert.fail(); } catch (InterruptedException ignore) { } Thread.currentThread().interrupt(); - try { s.tryAcquireUntil(1, System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(1L)); Assert.fail(); } catch (InterruptedException ignore) { } + try { s.tryAcquireUntil(1, nanoTime() + MILLISECONDS.toNanos(1L)); Assert.fail(); } catch (InterruptedException ignore) { } List> fs = new ArrayList<>(); - fs.add(exec.submit(() -> s.tryAcquire(1, 1L, TimeUnit.MINUTES))); + fs.add(exec.submit(() -> s.tryAcquire(1, 1L, MINUTES))); while (s instanceof Semaphore.Standard && ((Semaphore.Standard) s).waiting() == 0) Thread.yield(); - fs.add(exec.submit(() -> s.tryAcquireUntil(1, System.nanoTime() + TimeUnit.MINUTES.toNanos(1L)))); + fs.add(exec.submit(() -> s.tryAcquireUntil(1, System.nanoTime() + MINUTES.toNanos(1L)))); while (s instanceof Semaphore.Standard && ((Semaphore.Standard) s).waiting() == 1) Thread.yield(); fs.add(exec.submit(() -> { s.acquire(1); return true; } )); return fs;